[201634] Add ignored key pair API

src/hope_dedup_engine/apps/api/const.py

@@ -14,3 +14,6 @@
 
 DUPLICATE = "duplicate"
 DUPLICATE_LIST = f"{DUPLICATE}s"
+
+IGNORED_KEYS = "ignore"
+IGNORED_KEYS_LIST = f"{IGNORED_KEYS}s"

src/hope_dedup_engine/apps/api/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 5.0.6 on 2024-06-04 13:30
+# Generated by Django 5.0.6 on 2024-06-11 09:14
 
 import django.db.models.deletion
 import uuid
@@ -117,4 +117,19 @@ class Migration(migrations.Migration):
                 ),
             ],
         ),
+        migrations.CreateModel(
+            name="IgnoredKeyPair",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("first_reference_pk", models.CharField(max_length=100)),
+                ("second_reference_pk", models.CharField(max_length=100)),
+                (
+                    "deduplication_set",
+                    models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="api.deduplicationset"),
+                ),
+            ],
+            options={
+                "unique_together": {("deduplication_set", "first_reference_pk", "second_reference_pk")},
+            },
+        ),
     ]

src/hope_dedup_engine/apps/api/models/deduplication.py

@@ -1,3 +1,4 @@
+from typing import Any, override
 from uuid import uuid4
 
 from django.conf import settings
@@ -54,3 +55,17 @@ class Duplicate(models.Model):
     second_reference_pk = models.CharField(max_length=REFERENCE_PK_LENGTH)
     second_filename = models.CharField(max_length=255)
     score = models.FloatField()
+
+
+class IgnoredKeyPair(models.Model):
+    deduplication_set = models.ForeignKey(DeduplicationSet, on_delete=models.CASCADE)
+    first_reference_pk = models.CharField(max_length=REFERENCE_PK_LENGTH)
+    second_reference_pk = models.CharField(max_length=REFERENCE_PK_LENGTH)
+
+    class Meta:
+        unique_together = "deduplication_set", "first_reference_pk", "second_reference_pk"
+
+    @override
+    def save(self, **kwargs: Any) -> None:
+        self.first_reference_pk, self.second_reference_pk = sorted((self.first_reference_pk, self.second_reference_pk))
+        super().save(**kwargs)

src/hope_dedup_engine/apps/api/serializers.py

@@ -1,7 +1,7 @@
 from rest_framework import serializers
 
 from hope_dedup_engine.apps.api.models import DeduplicationSet
-from hope_dedup_engine.apps.api.models.deduplication import Duplicate, Image
+from hope_dedup_engine.apps.api.models.deduplication import Duplicate, IgnoredKeyPair, Image
 
 
 class DeduplicationSetSerializer(serializers.ModelSerializer):
@@ -38,3 +38,9 @@ def get_filename(self, duplicate: Duplicate) -> str:
 class DuplicateSerializer(serializers.Serializer):
     first = EntrySerializer(prefix="first", source="*")
     second = EntrySerializer(prefix="second", source="*")
+
+
+class IgnoredKeyPairSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = IgnoredKeyPair
+        fields = "__all__"

src/hope_dedup_engine/apps/api/urls.py

@@ -8,9 +8,16 @@
     DEDUPLICATION_SET,
     DEDUPLICATION_SET_LIST,
     DUPLICATE_LIST,
+    IGNORED_KEYS_LIST,
     IMAGE_LIST,
 )
-from hope_dedup_engine.apps.api.views import BulkImageViewSet, DeduplicationSetViewSet, DuplicateViewSet, ImageViewSet
+from hope_dedup_engine.apps.api.views import (
+    BulkImageViewSet,
+    DeduplicationSetViewSet,
+    DuplicateViewSet,
+    IgnoredKeyPairViewSet,
+    ImageViewSet,
+)
 
 router = routers.SimpleRouter()
 router.register(DEDUPLICATION_SET_LIST, DeduplicationSetViewSet, basename=DEDUPLICATION_SET_LIST)
@@ -19,5 +26,6 @@
 deduplication_sets_router.register(IMAGE_LIST, ImageViewSet, basename=IMAGE_LIST)
 deduplication_sets_router.register(BULK_IMAGE_LIST, BulkImageViewSet, basename=BULK_IMAGE_LIST)
 deduplication_sets_router.register(DUPLICATE_LIST, DuplicateViewSet, basename=DUPLICATE_LIST)
+deduplication_sets_router.register(IGNORED_KEYS_LIST, IgnoredKeyPairViewSet, basename=IGNORED_KEYS_LIST)
 
 urlpatterns = [path("", include(router.urls)), path("", include(deduplication_sets_router.urls))]

src/hope_dedup_engine/apps/api/views.py

@@ -20,8 +20,13 @@
 )
 from hope_dedup_engine.apps.api.const import DEDUPLICATION_SET_FILTER, DEDUPLICATION_SET_PARAM
 from hope_dedup_engine.apps.api.models import DeduplicationSet
-from hope_dedup_engine.apps.api.models.deduplication import Duplicate, Image
-from hope_dedup_engine.apps.api.serializers import DeduplicationSetSerializer, DuplicateSerializer, ImageSerializer
+from hope_dedup_engine.apps.api.models.deduplication import Duplicate, IgnoredKeyPair, Image
+from hope_dedup_engine.apps.api.serializers import (
+    DeduplicationSetSerializer,
+    DuplicateSerializer,
+    IgnoredKeyPairSerializer,
+    ImageSerializer,
+)
 from hope_dedup_engine.apps.api.utils import delete_model_data, start_processing
 
 MESSAGE = "message"
@@ -164,3 +169,22 @@ class DuplicateViewSet(nested_viewsets.NestedViewSetMixin, mixins.ListModelMixin
     parent_lookup_kwargs = {
         DEDUPLICATION_SET_PARAM: DEDUPLICATION_SET_FILTER,
     }
+
+
+class IgnoredKeyPairViewSet(
+    nested_viewsets.NestedViewSetMixin, mixins.ListModelMixin, mixins.CreateModelMixin, viewsets.GenericViewSet
+):
+    authentication_classes = (HDETokenAuthentication,)
+    permission_classes = IsAuthenticated, AssignedToExternalSystem, UserAndDeduplicationSetAreOfTheSameSystem
+    serializer_class = IgnoredKeyPairSerializer
+    queryset = IgnoredKeyPair.objects.all()
+    parent_lookup_kwargs = {
+        DEDUPLICATION_SET_PARAM: DEDUPLICATION_SET_FILTER,
+    }
+
+    def perform_create(self, serializer: Serializer) -> None:
+        super().perform_create(serializer)
+        deduplication_set = serializer.instance.deduplication_set
+        deduplication_set.state = DeduplicationSet.State.DIRTY
+        deduplication_set.updated_by = self.request.user
+        deduplication_set.save()

tests/api/api_const.py

@@ -1,11 +1,20 @@
-from hope_dedup_engine.apps.api.const import BULK_IMAGE_LIST, DEDUPLICATION_SET_LIST, DUPLICATE_LIST, IMAGE_LIST
+from hope_dedup_engine.apps.api.const import (
+    BULK_IMAGE_LIST,
+    DEDUPLICATION_SET_LIST,
+    DUPLICATE_LIST,
+    IGNORED_KEYS_LIST,
+    IMAGE_LIST,
+)
 
 JSON = "json"
-DEDUPLICATION_SET_LIST_VIEW = f"{DEDUPLICATION_SET_LIST}-list"
-DEDUPLICATION_SET_DETAIL_VIEW = f"{DEDUPLICATION_SET_LIST}-detail"
+LIST = "list"
+DETAIL = "detail"
+DEDUPLICATION_SET_LIST_VIEW = f"{DEDUPLICATION_SET_LIST}-{LIST}"
+DEDUPLICATION_SET_DETAIL_VIEW = f"{DEDUPLICATION_SET_LIST}-{DETAIL}"
 DEDUPLICATION_SET_PROCESS_VIEW = f"{DEDUPLICATION_SET_LIST}-process"
-IMAGE_LIST_VIEW = f"{IMAGE_LIST}-list"
-IMAGE_DETAIL_VIEW = f"{IMAGE_LIST}-detail"
-BULK_IMAGE_LIST_VIEW = f"{BULK_IMAGE_LIST}-list"
+IMAGE_LIST_VIEW = f"{IMAGE_LIST}-{LIST}"
+IMAGE_DETAIL_VIEW = f"{IMAGE_LIST}-{DETAIL}"
+BULK_IMAGE_LIST_VIEW = f"{BULK_IMAGE_LIST}-{LIST}"
 BULK_IMAGE_CLEAR_VIEW = f"{BULK_IMAGE_LIST}-clear"
-DUPLICATE_LIST_VIEW = f"{DUPLICATE_LIST}-list"
+DUPLICATE_LIST_VIEW = f"{DUPLICATE_LIST}-{LIST}"
+IGNORED_KEYS_LIST_VIEW = f"{IGNORED_KEYS_LIST}-{LIST}"

tests/api/conftest.py

@@ -5,7 +5,13 @@
 from pytest_factoryboy import LazyFixture, register
 from pytest_mock import MockerFixture
 from rest_framework.test import APIClient
-from testutils.factories.api import DeduplicationSetFactory, DuplicateFactory, ImageFactory, TokenFactory
+from testutils.factories.api import (
+    DeduplicationSetFactory,
+    DuplicateFactory,
+    IgnoredKeyPairFactory,
+    ImageFactory,
+    TokenFactory,
+)
 from testutils.factories.user import ExternalSystemFactory, UserFactory
 
 from hope_dedup_engine.apps.api.models import HDEToken
@@ -16,6 +22,7 @@
 register(DeduplicationSetFactory, external_system=LazyFixture("external_system"))
 register(ImageFactory, deduplication_Set=LazyFixture("deduplication_set"))
 register(DuplicateFactory, deduplication_set=LazyFixture("deduplication_set"))
+register(IgnoredKeyPairFactory, deduplication_set=LazyFixture("deduplication_set"))
 
 
 @fixture

tests/api/test_auth.py

@@ -7,6 +7,7 @@
     BULK_IMAGE_LIST_VIEW,
     DEDUPLICATION_SET_DETAIL_VIEW,
     DEDUPLICATION_SET_LIST_VIEW,
+    IGNORED_KEYS_LIST_VIEW,
     IMAGE_DETAIL_VIEW,
     IMAGE_LIST_VIEW,
     JSON,
@@ -32,6 +33,8 @@
     (BULK_IMAGE_LIST_VIEW, HTTPMethod.POST, (PK,)),
     (IMAGE_DETAIL_VIEW, HTTPMethod.DELETE, (PK, PK)),
     (BULK_IMAGE_CLEAR_VIEW, HTTPMethod.DELETE, (PK,)),
+    (IGNORED_KEYS_LIST_VIEW, HTTPMethod.GET, (PK,)),
+    (IGNORED_KEYS_LIST_VIEW, HTTPMethod.POST, (PK,)),
 )
 
 

tests/api/test_ignored_keys_create.py

@@ -0,0 +1,74 @@
+from api_const import IGNORED_KEYS_LIST_VIEW, JSON
+from pytest import mark
+from rest_framework import status
+from rest_framework.reverse import reverse
+from rest_framework.test import APIClient
+from testutils.factories.api import IgnoredKeyPairFactory
+
+from hope_dedup_engine.apps.api.models import DeduplicationSet
+from hope_dedup_engine.apps.api.models.deduplication import IgnoredKeyPair
+from hope_dedup_engine.apps.api.serializers import IgnoredKeyPairSerializer
+from hope_dedup_engine.apps.security.models import User
+
+
+def test_can_create_ignored_key_pair(api_client: APIClient, deduplication_set: DeduplicationSet) -> None:
+    previous_amount = IgnoredKeyPair.objects.filter(deduplication_set=deduplication_set).count()
+    data = IgnoredKeyPairSerializer(IgnoredKeyPairFactory.build()).data
+
+    response = api_client.post(reverse(IGNORED_KEYS_LIST_VIEW, (deduplication_set.pk,)), data=data, format=JSON)
+    assert response.status_code == status.HTTP_201_CREATED
+    assert IgnoredKeyPair.objects.filter(deduplication_set=deduplication_set).count() == previous_amount + 1
+
+
+def test_cannot_create_ignored_key_pair_between_systems(
+    another_system_api_client: APIClient, deduplication_set: DeduplicationSet
+) -> None:
+    previous_amount = IgnoredKeyPair.objects.filter(deduplication_set=deduplication_set).count()
+    data = IgnoredKeyPairSerializer(IgnoredKeyPairFactory.build()).data
+
+    response = another_system_api_client.post(
+        reverse(IGNORED_KEYS_LIST_VIEW, (deduplication_set.pk,)), data=data, format=JSON
+    )
+    assert response.status_code == status.HTTP_403_FORBIDDEN
+    assert IgnoredKeyPair.objects.filter(deduplication_set=deduplication_set).count() == previous_amount
+
+
+INVALID_PK_VALUES = "", None
+
+
+@mark.parametrize("first_pk", INVALID_PK_VALUES)
+@mark.parametrize("second_pk", INVALID_PK_VALUES)
+def test_invalid_values_handling(
+    api_client: APIClient, deduplication_set: DeduplicationSet, first_pk: str | None, second_pk: str | None
+) -> None:
+    data = IgnoredKeyPairSerializer(IgnoredKeyPairFactory.build()).data
+    data["first_reference_pk"] = first_pk
+    data["second_reference_pk"] = second_pk
+    response = api_client.post(reverse(IGNORED_KEYS_LIST_VIEW, (deduplication_set.pk,)), data=data, format=JSON)
+    assert response.status_code == status.HTTP_400_BAD_REQUEST
+    errors = response.json()
+    assert len(errors) == 2
+    assert "first_reference_pk" in errors
+    assert "second_reference_pk" in errors
+
+
+def test_missing_pk_handling(api_client: APIClient, deduplication_set: DeduplicationSet) -> None:
+    data = IgnoredKeyPairSerializer(IgnoredKeyPairFactory.build()).data
+    del data["first_reference_pk"], data["second_reference_pk"]
+
+    response = api_client.post(reverse(IGNORED_KEYS_LIST_VIEW, (deduplication_set.pk,)), data=data, format=JSON)
+    assert response.status_code == status.HTTP_400_BAD_REQUEST
+    errors = response.json()
+    assert "first_reference_pk" in errors
+    assert "second_reference_pk" in errors
+
+
+def test_deduplication_set_is_updated(api_client: APIClient, user: User, deduplication_set: DeduplicationSet) -> None:
+    assert deduplication_set.updated_by is None
+
+    data = IgnoredKeyPairSerializer(IgnoredKeyPairFactory.build()).data
+    response = api_client.post(reverse(IGNORED_KEYS_LIST_VIEW, (deduplication_set.pk,)), data=data, format=JSON)
+
+    assert response.status_code == status.HTTP_201_CREATED
+    deduplication_set.refresh_from_db()
+    assert deduplication_set.updated_by == user

tests/api/test_ignored_keys_list.py

@@ -0,0 +1,24 @@
+from api_const import IGNORED_KEYS_LIST_VIEW
+from rest_framework import status
+from rest_framework.reverse import reverse
+from rest_framework.test import APIClient
+
+from hope_dedup_engine.apps.api.models import DeduplicationSet
+from hope_dedup_engine.apps.api.models.deduplication import IgnoredKeyPair
+
+
+def test_can_list_ignored_key_pairs(
+    api_client: APIClient, deduplication_set: DeduplicationSet, ignored_key_pair: IgnoredKeyPair
+) -> None:
+    response = api_client.get(reverse(IGNORED_KEYS_LIST_VIEW, (deduplication_set.pk,)))
+    assert response.status_code == status.HTTP_200_OK
+    ignored_key_pairs = response.json()
+    assert len(ignored_key_pairs)
+    assert len(ignored_key_pairs) == IgnoredKeyPair.objects.filter(deduplication_set=deduplication_set).count()
+
+
+def test_cannot_list_ignored_key_pairs_between_systems(
+    another_system_api_client: APIClient, deduplication_set: DeduplicationSet
+) -> None:
+    response = another_system_api_client.get(reverse(IGNORED_KEYS_LIST_VIEW, (deduplication_set.pk,)))
+    assert response.status_code == status.HTTP_403_FORBIDDEN

tests/extras/testutils/factories/api.py

@@ -3,7 +3,7 @@
 from testutils.factories import ExternalSystemFactory, UserFactory
 
 from hope_dedup_engine.apps.api.models import DeduplicationSet, HDEToken
-from hope_dedup_engine.apps.api.models.deduplication import Duplicate, Image
+from hope_dedup_engine.apps.api.models.deduplication import Duplicate, IgnoredKeyPair, Image
 
 
 class TokenFactory(DjangoModelFactory):
@@ -43,3 +43,12 @@ class DuplicateFactory(DjangoModelFactory):
 
     class Meta:
         model = Duplicate
+
+
+class IgnoredKeyPairFactory(DjangoModelFactory):
+    deduplication_set = SubFactory(DeduplicationSetFactory)
+    first_reference_pk = fuzzy.FuzzyText()
+    second_reference_pk = fuzzy.FuzzyText()
+
+    class Meta:
+        model = IgnoredKeyPair