diff --git a/.dockerignore b/.dockerignore index 0bac6ee6..e32d7c8d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,6 @@ .git .* +!.flake8 ~* db docs \ No newline at end of file diff --git a/pdm.lock b/pdm.lock index c76715e2..04e7fad1 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "dev"] strategy = ["cross_platform", "inherit_metadata"] lock_version = "4.4.1" -content_hash = "sha256:09cc7ec3f6fe83b640f8b6f04858ad7b042333962ca30f0e1cc9f671d8902687" +content_hash = "sha256:6d792bff9be2788ed319c31f6740d9a08f0610b584684b3335acf01753a70877" [[package]] name = "amqp" @@ -780,6 +780,21 @@ files = [ {file = "djangorestframework-3.15.1.tar.gz", hash = "sha256:f88fad74183dfc7144b2756d0d2ac716ea5b4c7c9840995ac3bfd8ec034333c1"}, ] +[[package]] +name = "drf-nested-routers" +version = "0.94.1" +requires_python = ">=3.8" +summary = "Nested resources for the Django Rest Framework" +groups = ["default"] +dependencies = [ + "Django>=4.2", + "djangorestframework>=3.14.0", +] +files = [ + {file = "drf-nested-routers-0.94.1.tar.gz", hash = "sha256:2b846385ed95c9f17bf4242db3b264ac826b5af00dda6c737d3fe7cc7bf2c7db"}, + {file = "drf_nested_routers-0.94.1-py2.py3-none-any.whl", hash = "sha256:3a8ec45a025c0f39188ec1ec415244beb875a6f4db87911a1f5a606d09b68c9f"}, +] + [[package]] name = "drf-spectacular" version = "0.27.2" @@ -1664,6 +1679,20 @@ files = [ {file = "pytest_factoryboy-2.7.0.tar.gz", hash = "sha256:67fc54ec8669a3feb8ac60094dd57cd71eb0b20b2c319d2957873674c776a77b"}, ] +[[package]] +name = "pytest-mock" +version = "3.14.0" +requires_python = ">=3.8" +summary = "Thin-wrapper around the mock package for easier use with pytest" +groups = ["dev"] +dependencies = [ + "pytest>=6.2.5", +] +files = [ + {file = "pytest-mock-3.14.0.tar.gz", hash = "sha256:2719255a1efeceadbc056d6bf3df3d1c5015530fb40cf347c0f9afac88410bd0"}, + {file = "pytest_mock-3.14.0-py3-none-any.whl", hash = "sha256:0b72c38033392a5f4621342fe11e9219ac11ec9d375f8e2a0c164539e0d70f6f"}, +] + [[package]] name = "pytest-xdist" version = "3.6.1" diff --git a/pyproject.toml b/pyproject.toml index b10e8988..d1851b50 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,7 @@ dependencies = [ "django-flags>=5.0.13", "django-reversion>=5.0.12", "uwsgi>=2.0.25.1", + "drf-nested-routers>=0.94.1", ] [tool.pdm.build] @@ -74,6 +75,7 @@ dev = [ "unittest-xml-reporting", "watchdog", "pytest-factoryboy>=2.7.0", + "pytest-mock>=3.14.0", ] [tool.black] line-length = 120 diff --git a/src/hope_dedup_engine/apps/api/__init__.py b/src/hope_dedup_engine/apps/api/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/hope_dedup_engine/apps/api/apps.py b/src/hope_dedup_engine/apps/api/apps.py new file mode 100644 index 00000000..1856625e --- /dev/null +++ b/src/hope_dedup_engine/apps/api/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class ApiConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "hope_dedup_engine.apps.api" diff --git a/src/hope_dedup_engine/apps/api/auth.py b/src/hope_dedup_engine/apps/api/auth.py new file mode 100644 index 00000000..4a78ffcc --- /dev/null +++ b/src/hope_dedup_engine/apps/api/auth.py @@ -0,0 +1,25 @@ +from rest_framework.authentication import TokenAuthentication +from rest_framework.permissions import BasePermission +from rest_framework.request import Request +from rest_framework.views import View + +from hope_dedup_engine.apps.api.models import DeduplicationSet +from hope_dedup_engine.apps.api.models.auth import HDEToken + + +class AssignedToExternalSystem(BasePermission): + def has_permission(self, request: Request, view: View) -> bool: + return request.user and request.user.external_system + + +class UserAndDeduplicationSetAreOfTheSameSystem(BasePermission): + def has_permission(self, request: Request, view: View) -> bool: + if deduplication_set_pk := view.kwargs.get("deduplication_set_pk") or view.kwargs.get("pk"): + return DeduplicationSet.objects.filter( + external_system=request.user.external_system, pk=deduplication_set_pk + ).exists() + return True + + +class HDETokenAuthentication(TokenAuthentication): + model = HDEToken diff --git a/src/hope_dedup_engine/apps/api/const.py b/src/hope_dedup_engine/apps/api/const.py new file mode 100644 index 00000000..1533c068 --- /dev/null +++ b/src/hope_dedup_engine/apps/api/const.py @@ -0,0 +1,13 @@ +DEDUPLICATION_SET = "deduplication_set" +DEDUPLICATION_SET_LIST = f"{DEDUPLICATION_SET}s" + +PK = "pk" +DEDUPLICATION_SET_PARAM = f"{DEDUPLICATION_SET}_{PK}" +DEDUPLICATION_SET_FILTER = f"{DEDUPLICATION_SET}__{PK}" + +IMAGE = "image" +IMAGE_LIST = f"{IMAGE}s" + +BULK = "bulk" +BULK_IMAGE = f"{IMAGE}_{BULK}" +BULK_IMAGE_LIST = f"{IMAGE_LIST}_{BULK}" diff --git a/src/hope_dedup_engine/apps/api/migrations/0001_initial.py b/src/hope_dedup_engine/apps/api/migrations/0001_initial.py new file mode 100644 index 00000000..563e0da9 --- /dev/null +++ b/src/hope_dedup_engine/apps/api/migrations/0001_initial.py @@ -0,0 +1,123 @@ +# Generated by Django 5.0.6 on 2024-05-20 12:50 + +import django.db.models.deletion +import uuid +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ("security", "0001_initial"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="DeduplicationSet", + fields=[ + ("id", models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)), + ("name", models.CharField(max_length=100)), + ("reference_pk", models.IntegerField()), + ( + "state", + models.IntegerField( + choices=[(0, "Clean"), (1, "Dirty"), (2, "Processing"), (3, "Error")], default=0 + ), + ), + ("deleted", models.BooleanField(default=False, verbose_name="deleted")), + ("error", models.CharField(blank=True, max_length=255, null=True)), + ("created_at", models.DateTimeField(auto_now_add=True, verbose_name="created at")), + ("updated_at", models.DateTimeField(auto_now=True, verbose_name="updated at")), + ( + "created_by", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="+", + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "external_system", + models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="security.externalsystem"), + ), + ( + "updated_by", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="+", + to=settings.AUTH_USER_MODEL, + ), + ), + ], + ), + migrations.CreateModel( + name="HDEToken", + fields=[ + ("key", models.CharField(max_length=40, primary_key=True, serialize=False, verbose_name="Key")), + ("created", models.DateTimeField(auto_now_add=True, verbose_name="Created")), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="auth_tokens", + to=settings.AUTH_USER_MODEL, + verbose_name="User", + ), + ), + ], + options={ + "verbose_name": "Token", + "verbose_name_plural": "Tokens", + "abstract": False, + }, + ), + migrations.CreateModel( + name="Duplicate", + fields=[ + ("id", models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)), + ("filename", models.CharField(max_length=255)), + ( + "deduplication_set", + models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="api.deduplicationset"), + ), + ], + options={ + "abstract": False, + "unique_together": {("deduplication_set", "filename")}, + }, + ), + migrations.CreateModel( + name="Image", + fields=[ + ("id", models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)), + ("filename", models.CharField(max_length=255)), + ("created_at", models.DateTimeField(auto_now_add=True, verbose_name="created at")), + ( + "created_by", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="+", + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "deduplication_set", + models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="api.deduplicationset"), + ), + ], + options={ + "abstract": False, + "unique_together": {("deduplication_set", "filename")}, + }, + ), + ] diff --git a/src/hope_dedup_engine/apps/api/migrations/__init__.py b/src/hope_dedup_engine/apps/api/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/hope_dedup_engine/apps/api/models/__init__.py b/src/hope_dedup_engine/apps/api/models/__init__.py new file mode 100644 index 00000000..7ae749fe --- /dev/null +++ b/src/hope_dedup_engine/apps/api/models/__init__.py @@ -0,0 +1,2 @@ +from hope_dedup_engine.apps.api.models.auth import HDEToken # noqa: F401 +from hope_dedup_engine.apps.api.models.deduplication import DeduplicationSet # noqa: F401 diff --git a/src/hope_dedup_engine/apps/api/models/auth.py b/src/hope_dedup_engine/apps/api/models/auth.py new file mode 100644 index 00000000..1ff0efe0 --- /dev/null +++ b/src/hope_dedup_engine/apps/api/models/auth.py @@ -0,0 +1,11 @@ +from django.conf import settings +from django.db import models +from django.utils.translation import gettext_lazy as _ + +from rest_framework.authtoken.models import Token + + +class HDEToken(Token): + user = models.ForeignKey( + settings.AUTH_USER_MODEL, related_name="auth_tokens", on_delete=models.CASCADE, verbose_name=_("User") + ) diff --git a/src/hope_dedup_engine/apps/api/models/deduplication.py b/src/hope_dedup_engine/apps/api/models/deduplication.py new file mode 100644 index 00000000..14ea5e95 --- /dev/null +++ b/src/hope_dedup_engine/apps/api/models/deduplication.py @@ -0,0 +1,58 @@ +from uuid import uuid4 + +from django.conf import settings +from django.db import models +from django.utils.translation import gettext_lazy as _ + +from hope_dedup_engine.apps.security.models import ExternalSystem + + +class DeduplicationSet(models.Model): + class State(models.IntegerChoices): + CLEAN = 0, _("Clean") # Deduplication set is created or already processed + DIRTY = 1, _("Dirty") # Images are added to deduplication set, but not yet processed + PROCESSING = 2, _("Processing") # Images are being processed + ERROR = 3, _("Error") # Error occurred + + id = models.UUIDField(primary_key=True, default=uuid4) + name = models.CharField(max_length=100) + reference_pk = models.IntegerField() + state = models.IntegerField( + choices=State.choices, + default=State.CLEAN, + ) + deleted = models.BooleanField(_("deleted"), null=False, blank=False, default=False) + external_system = models.ForeignKey(ExternalSystem, on_delete=models.CASCADE) + error = models.CharField(max_length=255, null=True, blank=True) + created_by = models.ForeignKey( + settings.AUTH_USER_MODEL, on_delete=models.CASCADE, null=True, blank=True, related_name="+" + ) + created_at = models.DateTimeField(_("created at"), auto_now_add=True) + updated_by = models.ForeignKey( + settings.AUTH_USER_MODEL, on_delete=models.CASCADE, null=True, blank=True, related_name="+" + ) + updated_at = models.DateTimeField(_("updated at"), auto_now=True) + + +class ImagePath(models.Model): + id = models.UUIDField(primary_key=True, default=uuid4) + deduplication_set = models.ForeignKey(DeduplicationSet, on_delete=models.CASCADE) + filename = models.CharField(max_length=255) + + class Meta: + abstract = True + unique_together = ("deduplication_set", "filename") + + +class Duplicate(ImagePath): + pass + + +class Image(ImagePath): + created_by = models.ForeignKey( + settings.AUTH_USER_MODEL, on_delete=models.CASCADE, null=True, blank=True, related_name="+" + ) + created_at = models.DateTimeField(_("created at"), auto_now_add=True) + # + # class Meta: + # unique_together = ("deduplication_set", "filename") diff --git a/src/hope_dedup_engine/apps/api/serializers.py b/src/hope_dedup_engine/apps/api/serializers.py new file mode 100644 index 00000000..02832d4a --- /dev/null +++ b/src/hope_dedup_engine/apps/api/serializers.py @@ -0,0 +1,20 @@ +from rest_framework import serializers + +from hope_dedup_engine.apps.api.models import DeduplicationSet +from hope_dedup_engine.apps.api.models.deduplication import Image + + +class DeduplicationSetSerializer(serializers.ModelSerializer): + state = serializers.CharField(source="get_state_display", read_only=True) + + class Meta: + model = DeduplicationSet + exclude = ("deleted",) + read_only_fields = "external_system", "created_at", "created_by", "deleted", "updated_at", "updated_by" + + +class ImageSerializer(serializers.ModelSerializer): + class Meta: + model = Image + fields = "__all__" + read_only_fields = "created_by", "created_at" diff --git a/src/hope_dedup_engine/apps/api/urls.py b/src/hope_dedup_engine/apps/api/urls.py new file mode 100644 index 00000000..b50b2d4d --- /dev/null +++ b/src/hope_dedup_engine/apps/api/urls.py @@ -0,0 +1,16 @@ +from django.urls import include, path + +from rest_framework import routers +from rest_framework_nested import routers as nested_routers + +from hope_dedup_engine.apps.api.const import BULK_IMAGE_LIST, DEDUPLICATION_SET, DEDUPLICATION_SET_LIST, IMAGE_LIST +from hope_dedup_engine.apps.api.views import BulkImageViewSet, DeduplicationSetViewSet, ImageViewSet + +router = routers.SimpleRouter() +router.register(DEDUPLICATION_SET_LIST, DeduplicationSetViewSet, basename=DEDUPLICATION_SET_LIST) + +deduplication_sets_router = nested_routers.NestedSimpleRouter(router, DEDUPLICATION_SET_LIST, lookup=DEDUPLICATION_SET) +deduplication_sets_router.register(IMAGE_LIST, ImageViewSet, basename=IMAGE_LIST) +deduplication_sets_router.register(BULK_IMAGE_LIST, BulkImageViewSet, basename=BULK_IMAGE_LIST) + +urlpatterns = [path("", include(router.urls)), path("", include(deduplication_sets_router.urls))] diff --git a/src/hope_dedup_engine/apps/api/utils.py b/src/hope_dedup_engine/apps/api/utils.py new file mode 100644 index 00000000..fbb30c28 --- /dev/null +++ b/src/hope_dedup_engine/apps/api/utils.py @@ -0,0 +1,9 @@ +from hope_dedup_engine.apps.api.models import DeduplicationSet + + +def start_processing(_: DeduplicationSet) -> None: + pass + + +def delete_model_data(_: DeduplicationSet) -> None: + pass diff --git a/src/hope_dedup_engine/apps/api/views.py b/src/hope_dedup_engine/apps/api/views.py new file mode 100644 index 00000000..9c3a229e --- /dev/null +++ b/src/hope_dedup_engine/apps/api/views.py @@ -0,0 +1,151 @@ +from dataclasses import dataclass +from http import HTTPMethod +from typing import Any +from uuid import UUID + +from django.db.models import QuerySet + +from rest_framework import mixins, status, viewsets +from rest_framework.decorators import action +from rest_framework.permissions import IsAuthenticated +from rest_framework.request import Request +from rest_framework.response import Response +from rest_framework.serializers import Serializer +from rest_framework_nested import viewsets as nested_viewsets + +from hope_dedup_engine.apps.api.auth import ( + AssignedToExternalSystem, + HDETokenAuthentication, + UserAndDeduplicationSetAreOfTheSameSystem, +) +from hope_dedup_engine.apps.api.const import DEDUPLICATION_SET_FILTER, DEDUPLICATION_SET_PARAM +from hope_dedup_engine.apps.api.models import DeduplicationSet +from hope_dedup_engine.apps.api.models.deduplication import Image +from hope_dedup_engine.apps.api.serializers import DeduplicationSetSerializer, ImageSerializer +from hope_dedup_engine.apps.api.utils import delete_model_data, start_processing + +MESSAGE = "message" +STARTED = "started" +RETRYING = "retrying" +ALREADY_PROCESSING = "already processing" + + +class DeduplicationSetViewSet( + mixins.ListModelMixin, mixins.CreateModelMixin, mixins.DestroyModelMixin, viewsets.GenericViewSet +): + authentication_classes = (HDETokenAuthentication,) + permission_classes = IsAuthenticated, AssignedToExternalSystem, UserAndDeduplicationSetAreOfTheSameSystem + serializer_class = DeduplicationSetSerializer + + def get_queryset(self) -> QuerySet: + return DeduplicationSet.objects.filter(external_system=self.request.user.external_system, deleted=False) + + def perform_create(self, serializer: Serializer) -> None: + serializer.save(created_by=self.request.user, external_system=self.request.user.external_system) + + def perform_destroy(self, instance: DeduplicationSet) -> None: + instance.updated_by = self.request.user + instance.deleted = True + instance.save() + delete_model_data(instance) + + @action(detail=True, methods=(HTTPMethod.POST,)) + def process(self, request: Request, pk: UUID | None = None) -> Response: + deduplication_set = DeduplicationSet.objects.get(pk=pk) + match deduplication_set.state: + case DeduplicationSet.State.CLEAN | DeduplicationSet.State.ERROR: + start_processing(deduplication_set) + return Response({MESSAGE: RETRYING}) + case DeduplicationSet.State.DIRTY: + start_processing(deduplication_set) + return Response({MESSAGE: STARTED}) + case DeduplicationSet.State.PROCESSING: + return Response({MESSAGE: ALREADY_PROCESSING}, status=status.HTTP_400_BAD_REQUEST) + + +class ImageViewSet( + nested_viewsets.NestedViewSetMixin, + mixins.ListModelMixin, + mixins.CreateModelMixin, + mixins.DestroyModelMixin, + viewsets.GenericViewSet, +): + authentication_classes = (HDETokenAuthentication,) + permission_classes = IsAuthenticated, AssignedToExternalSystem, UserAndDeduplicationSetAreOfTheSameSystem + serializer_class = ImageSerializer + queryset = Image.objects.all() + parent_lookup_kwargs = { + DEDUPLICATION_SET_PARAM: DEDUPLICATION_SET_FILTER, + } + + def perform_create(self, serializer: Serializer) -> None: + super().perform_create(serializer) + deduplication_set = serializer.instance.deduplication_set + deduplication_set.state = DeduplicationSet.State.DIRTY + deduplication_set.updated_by = self.request.user + deduplication_set.save() + + def perform_destroy(self, instance: Image) -> None: + deduplication_set = instance.deduplication_set + super().perform_destroy(instance) + deduplication_set.state = DeduplicationSet.State.DIRTY + deduplication_set.updated_by = self.request.user + deduplication_set.save() + + +@dataclass +class ListDataWrapper: + data: list[dict] + + def __setitem__(self, key: str, value: Any) -> None: + for item in self.data: + item[key] = value + + +class WrapRequestDataMixin: + def initialize_request(self, request: Request, *args: Any, **kwargs: Any) -> Request: + request = super().initialize_request(request, *args, **kwargs) + request._full_data = ListDataWrapper(request.data) + return request + + +class UnwrapRequestDataMixin: + def initialize_request(self, request: Request, *args: Any, **kwargs: Any) -> Request: + request = super().initialize_request(request, *args, **kwargs) + request._full_data = request._full_data.data + return request + + +# drf-nested-routers doesn't work correctly when request data is a list, so we use WrapRequestDataMixin, +# UnwrapRequestDataMixin, and ListDataWrapper to make it work with list of objects +class BulkImageViewSet( + UnwrapRequestDataMixin, + nested_viewsets.NestedViewSetMixin, + WrapRequestDataMixin, + mixins.CreateModelMixin, + viewsets.GenericViewSet, +): + authentication_classes = (HDETokenAuthentication,) + permission_classes = IsAuthenticated, AssignedToExternalSystem, UserAndDeduplicationSetAreOfTheSameSystem + serializer_class = ImageSerializer + queryset = Image.objects.all() + parent_lookup_kwargs = { + DEDUPLICATION_SET_PARAM: DEDUPLICATION_SET_FILTER, + } + + def get_serializer(self, *args: Any, **kwargs: Any) -> Serializer: + return super().get_serializer(*args, **kwargs, many=True) + + def perform_create(self, serializer: Serializer) -> None: + super().perform_create(serializer) + if deduplication_set := serializer.instance[0].deduplication_set if serializer.instance else None: + deduplication_set.updated_by = self.request.user + deduplication_set.save() + + @action(detail=False, methods=(HTTPMethod.DELETE,)) + def clear(self, request: Request, deduplication_set_pk: str) -> Response: + deduplication_set = DeduplicationSet.objects.get(pk=deduplication_set_pk) + Image.objects.filter(deduplication_set=deduplication_set).delete() + deduplication_set.updated_by = request.user + deduplication_set.save() + return Response(status=status.HTTP_204_NO_CONTENT) diff --git a/src/hope_dedup_engine/apps/core/management/commands/createsystem.py b/src/hope_dedup_engine/apps/core/management/commands/createsystem.py new file mode 100644 index 00000000..fbe2707d --- /dev/null +++ b/src/hope_dedup_engine/apps/core/management/commands/createsystem.py @@ -0,0 +1,17 @@ +from django.core.management import BaseCommand + +from hope_dedup_engine.apps.core.models import ExternalSystem + + +class Command(BaseCommand): + help = "Creates external system" + + def add_arguments(self, parser): + parser.add_argument("name") + + def handle(self, *args, **options): + system, created = ExternalSystem.objects.get_or_create(name=(name := options["name"])) + if created: + self.stdout.write(self.style.SUCCESS(f'"{name}" system created.')) + else: + self.stdout.write(self.style.WARNING(f'"{name}" already exists.')) diff --git a/src/hope_dedup_engine/apps/security/migrations/0001_initial.py b/src/hope_dedup_engine/apps/security/migrations/0001_initial.py index ec2b873a..bf34ef36 100644 --- a/src/hope_dedup_engine/apps/security/migrations/0001_initial.py +++ b/src/hope_dedup_engine/apps/security/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 5.0.6 on 2024-05-13 13:48 +# Generated by Django 5.0.6 on 2024-05-17 18:09 import django.contrib.auth.models import django.contrib.auth.validators @@ -17,6 +17,12 @@ class Migration(migrations.Migration): ] operations = [ + migrations.CreateModel( + name="ExternalSystem", + fields=[ + ("name", models.CharField(max_length=100, primary_key=True, serialize=False)), + ], + ), migrations.CreateModel( name="System", fields=[ @@ -94,6 +100,15 @@ class Migration(migrations.Migration): verbose_name="user permissions", ), ), + ( + "external_system", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="security.externalsystem", + ), + ), ], options={ "abstract": False, diff --git a/src/hope_dedup_engine/apps/security/models.py b/src/hope_dedup_engine/apps/security/models.py index e492f22e..8ed7506a 100644 --- a/src/hope_dedup_engine/apps/security/models.py +++ b/src/hope_dedup_engine/apps/security/models.py @@ -8,7 +8,12 @@ class System(models.Model): name = models.CharField(max_length=255) +class ExternalSystem(models.Model): + name = models.CharField(max_length=100, primary_key=True) + + class User(SecurityMixin, AbstractUser): + external_system = models.ForeignKey(ExternalSystem, on_delete=models.SET_NULL, null=True, blank=True) class Meta: abstract = False diff --git a/src/hope_dedup_engine/config/settings.py b/src/hope_dedup_engine/config/settings.py index 764da1e7..22743611 100644 --- a/src/hope_dedup_engine/config/settings.py +++ b/src/hope_dedup_engine/config/settings.py @@ -42,6 +42,7 @@ "django_celery_beat", "drf_spectacular", "drf_spectacular_sidecar", + "hope_dedup_engine.apps.api", ) MIDDLEWARE = ( diff --git a/src/hope_dedup_engine/config/urls.py b/src/hope_dedup_engine/config/urls.py index 0f48fa01..f3d5de4b 100644 --- a/src/hope_dedup_engine/config/urls.py +++ b/src/hope_dedup_engine/config/urls.py @@ -15,6 +15,7 @@ path(r"sentry_debug/", lambda _: 1 / 0), path(r"__debug__/", include(debug_toolbar.urls)), path(r"", include("hope_dedup_engine.web.urls")), + path("", include("hope_dedup_engine.apps.api.urls")), ] if settings.DEBUG: diff --git a/src/hope_dedup_engine/state.py b/src/hope_dedup_engine/state.py index c8d7f77b..0973df8f 100644 --- a/src/hope_dedup_engine/state.py +++ b/src/hope_dedup_engine/state.py @@ -3,14 +3,23 @@ from copy import copy from datetime import datetime, timedelta from threading import local -from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional +from typing import Any, Dict, Iterator, List, Mapping, Optional, Protocol +# TODO: find out what is correct value for this +not_set = None -not_set = object() + +class AnyRequest(Protocol): + COOKIES: Mapping[str, Any] + + +class AnyResponse(Protocol): + def set_cookie(self, name: str, *args: Any) -> None: + pass class State(local): - request: "AnyRequest|None" = None + request: AnyRequest | None = None cookies: Dict[str, List[Any]] = {} def __repr__(self) -> str: diff --git a/tests/api/conftest.py b/tests/api/conftest.py new file mode 100644 index 00000000..2ae5891f --- /dev/null +++ b/tests/api/conftest.py @@ -0,0 +1,54 @@ +from typing import Any +from unittest.mock import MagicMock + +from pytest import fixture +from pytest_factoryboy import LazyFixture, register +from pytest_mock import MockerFixture +from rest_framework.test import APIClient +from testutils.factories.api import DeduplicationSetFactory, ImageFactory, TokenFactory +from testutils.factories.user import ExternalSystemFactory, UserFactory + +from hope_dedup_engine.apps.api.models import HDEToken +from hope_dedup_engine.apps.security.models import User + +register(ExternalSystemFactory) +register(UserFactory) +register(DeduplicationSetFactory, external_system=LazyFixture("external_system")) +register(ImageFactory, deduplication_Set=LazyFixture("deduplication_set")) + + +@fixture +def anonymous_api_client() -> APIClient: + return APIClient() + + +def get_auth_headers(token: HDEToken) -> dict[str, str]: + return {"HTTP_AUTHORIZATION": f"Token {token.key}"} + + +def create_api_client(user: User) -> APIClient: + token = TokenFactory(user=user) + client = APIClient() + client.credentials(**get_auth_headers(token)) + return client + + +@fixture +def api_client(user: User) -> APIClient: + return create_api_client(user) + + +@fixture +def another_system_api_client(db: Any) -> APIClient: + another_system_user = UserFactory() + return create_api_client(another_system_user) + + +@fixture +def delete_model_data(mocker: MockerFixture) -> MagicMock: + return mocker.patch("hope_dedup_engine.apps.api.views.delete_model_data") + + +@fixture +def start_processing(mocker: MockerFixture) -> MagicMock: + return mocker.patch("hope_dedup_engine.apps.api.views.start_processing") diff --git a/tests/api/const.py b/tests/api/const.py new file mode 100644 index 00000000..a85f9896 --- /dev/null +++ b/tests/api/const.py @@ -0,0 +1,10 @@ +from hope_dedup_engine.apps.api.const import BULK_IMAGE_LIST, DEDUPLICATION_SET_LIST, IMAGE_LIST + +JSON = "json" +DEDUPLICATION_SET_LIST_VIEW = f"{DEDUPLICATION_SET_LIST}-list" +DEDUPLICATION_SET_DETAIL_VIEW = f"{DEDUPLICATION_SET_LIST}-detail" +DEDUPLICATION_SET_PROCESS_VIEW = f"{DEDUPLICATION_SET_LIST}-process" +IMAGE_LIST_VIEW = f"{IMAGE_LIST}-list" +IMAGE_DETAIL_VIEW = f"{IMAGE_LIST}-detail" +BULK_IMAGE_LIST_VIEW = f"{BULK_IMAGE_LIST}-list" +BULK_IMAGE_CLEAR_VIEW = f"{BULK_IMAGE_LIST}-clear" diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py new file mode 100644 index 00000000..ea6f42d7 --- /dev/null +++ b/tests/api/test_auth.py @@ -0,0 +1,59 @@ +from http import HTTPMethod +from typing import Any +from uuid import uuid4 + +from conftest import get_auth_headers +from const import ( + BULK_IMAGE_CLEAR_VIEW, + BULK_IMAGE_LIST_VIEW, + DEDUPLICATION_SET_DETAIL_VIEW, + DEDUPLICATION_SET_LIST_VIEW, + IMAGE_DETAIL_VIEW, + IMAGE_LIST_VIEW, + JSON, +) +from pytest import mark +from rest_framework import status +from rest_framework.reverse import reverse +from rest_framework.test import APIClient +from testutils.factories.api import TokenFactory + +from hope_dedup_engine.apps.security.models import User + +PK = uuid4() + + +REQUESTS = ( + (DEDUPLICATION_SET_LIST_VIEW, HTTPMethod.GET, ()), + (DEDUPLICATION_SET_LIST_VIEW, HTTPMethod.POST, ()), + (DEDUPLICATION_SET_DETAIL_VIEW, HTTPMethod.DELETE, (PK,)), + (IMAGE_LIST_VIEW, HTTPMethod.GET, (PK,)), + (IMAGE_LIST_VIEW, HTTPMethod.POST, (PK,)), + (BULK_IMAGE_LIST_VIEW, HTTPMethod.POST, (PK,)), + (IMAGE_DETAIL_VIEW, HTTPMethod.DELETE, (PK, PK)), + (BULK_IMAGE_CLEAR_VIEW, HTTPMethod.DELETE, (PK,)), +) + + +@mark.parametrize(("view_name", "method", "args"), REQUESTS) +def test_anonymous_cannot_access( + anonymous_api_client: APIClient, view_name: str, method: HTTPMethod, args: tuple[Any, ...] +) -> None: + response = getattr(anonymous_api_client, method.lower())(reverse(view_name, args)) + assert response.status_code == status.HTTP_401_UNAUTHORIZED + + +@mark.parametrize(("view_name", "method", "args"), REQUESTS) +def test_authenticated_can_access( + api_client: APIClient, view_name: str, method: HTTPMethod, args: tuple[Any, ...] +) -> None: + response = getattr(api_client, method.lower())(reverse(view_name, args), format=JSON) + assert response.status_code != status.HTTP_401_UNAUTHORIZED + + +def test_multiple_tokens_can_be_used(api_client: APIClient, user: User) -> None: + tokens = [TokenFactory(user=user) for _ in range(5)] + for token in tokens: + api_client.credentials(**get_auth_headers(token)) + response = api_client.get(reverse(DEDUPLICATION_SET_LIST_VIEW)) + assert response.status_code == status.HTTP_200_OK diff --git a/tests/api/test_business_logic.py b/tests/api/test_business_logic.py new file mode 100644 index 00000000..1833c1f3 --- /dev/null +++ b/tests/api/test_business_logic.py @@ -0,0 +1,77 @@ +from unittest.mock import MagicMock + +from const import ( + DEDUPLICATION_SET_DETAIL_VIEW, + DEDUPLICATION_SET_LIST_VIEW, + DEDUPLICATION_SET_PROCESS_VIEW, + IMAGE_DETAIL_VIEW, + IMAGE_LIST_VIEW, + JSON, +) +from pytest import mark +from rest_framework import status +from rest_framework.reverse import reverse +from rest_framework.test import APIClient +from testutils.factories.api import DeduplicationSetFactory, ImageFactory + +from hope_dedup_engine.apps.api.models import DeduplicationSet +from hope_dedup_engine.apps.api.models.deduplication import Image +from hope_dedup_engine.apps.api.serializers import DeduplicationSetSerializer, ImageSerializer + + +def test_new_deduplication_set_status_is_clean(api_client: APIClient) -> None: + data = DeduplicationSetSerializer(DeduplicationSetFactory.build()).data + + response = api_client.post(reverse(DEDUPLICATION_SET_LIST_VIEW), data=data, format=JSON) + assert response.status_code == status.HTTP_201_CREATED + deduplication_set = response.json() + assert deduplication_set["state"] == DeduplicationSet.State.CLEAN.label + + +@mark.parametrize( + "deduplication_set__state", + (DeduplicationSet.State.CLEAN, DeduplicationSet.State.DIRTY, DeduplicationSet.State.ERROR), +) +def test_deduplication_set_processing_trigger( + api_client: APIClient, start_processing: MagicMock, deduplication_set: DeduplicationSet +) -> None: + response = api_client.post(reverse(DEDUPLICATION_SET_PROCESS_VIEW, (deduplication_set.pk,))) + assert response.status_code == status.HTTP_200_OK + start_processing.assert_called_once_with(deduplication_set) + + +def test_new_image_makes_deduplication_set_state_dirty( + api_client: APIClient, deduplication_set: DeduplicationSet +) -> None: + assert deduplication_set.state == DeduplicationSet.State.CLEAN + response = api_client.post( + reverse(IMAGE_LIST_VIEW, (deduplication_set.pk,)), data=ImageSerializer(ImageFactory.build()).data, format=JSON + ) + assert response.status_code == status.HTTP_201_CREATED + deduplication_set.refresh_from_db() + assert deduplication_set.state == DeduplicationSet.State.DIRTY + + +def test_image_deletion_makes_deduplication_state_dirty( + api_client: APIClient, deduplication_set: DeduplicationSet, image: Image +) -> None: + response = api_client.delete(reverse(IMAGE_DETAIL_VIEW, (deduplication_set.pk, image.pk))) + assert response.status_code == status.HTTP_204_NO_CONTENT + deduplication_set.refresh_from_db() + assert deduplication_set.state == DeduplicationSet.State.DIRTY + + +def test_deletion_triggers_model_data_deletion( + api_client: APIClient, deduplication_set: DeduplicationSet, delete_model_data: MagicMock +) -> None: + response = api_client.delete(reverse(DEDUPLICATION_SET_DETAIL_VIEW, (deduplication_set.pk,))) + assert response.status_code == status.HTTP_204_NO_CONTENT + delete_model_data.assert_called_once_with(deduplication_set) + + +def test_unauthorized_deletion_does_not_trigger_model_data_deletion( + another_system_api_client: APIClient, deduplication_set: DeduplicationSet, delete_model_data: MagicMock +) -> None: + response = another_system_api_client.delete(reverse(DEDUPLICATION_SET_DETAIL_VIEW, (deduplication_set.pk,))) + assert response.status_code == status.HTTP_403_FORBIDDEN + delete_model_data.assert_not_called() diff --git a/tests/api/test_deduplication_set_create.py b/tests/api/test_deduplication_set_create.py new file mode 100644 index 00000000..10430bf6 --- /dev/null +++ b/tests/api/test_deduplication_set_create.py @@ -0,0 +1,62 @@ +from typing import Any + +from const import DEDUPLICATION_SET_LIST_VIEW, JSON +from pytest import mark +from rest_framework import status +from rest_framework.reverse import reverse +from rest_framework.test import APIClient +from testutils.factories.api import DeduplicationSetFactory + +from hope_dedup_engine.apps.api.models import DeduplicationSet +from hope_dedup_engine.apps.api.serializers import DeduplicationSetSerializer + + +def test_can_create_deduplication_set(api_client: APIClient) -> None: + previous_amount = DeduplicationSet.objects.count() + data = DeduplicationSetSerializer(DeduplicationSetFactory.build()).data + + response = api_client.post(reverse(DEDUPLICATION_SET_LIST_VIEW), data=data, format=JSON) + assert response.status_code == status.HTTP_201_CREATED + assert DeduplicationSet.objects.count() == previous_amount + 1 + + +@mark.parametrize( + "omit", + ( + "name", + "reference_pk", + ("name", "reference_pk"), + ), +) +def test_missing_fields_handling(api_client: APIClient, omit: str | tuple[str, ...]) -> None: + data = DeduplicationSetSerializer(DeduplicationSetFactory.build()).data + missing_fields = (omit,) if isinstance(omit, str) else omit + for field in missing_fields: + del data[field] + + response = api_client.post(reverse(DEDUPLICATION_SET_LIST_VIEW), data=data, format=JSON) + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json() + assert len(errors) == len(missing_fields) + for field in missing_fields: + assert field in errors + + +@mark.parametrize( + ("field", "value"), + ( + ("name", ""), + ("name", None), + ("reference_pk", "foo"), + ("reference_pk", None), + ("reference_pk", 3.14), + ), +) +def test_invalid_values_handling(api_client: APIClient, field: str, value: Any) -> None: + data = DeduplicationSetSerializer(DeduplicationSetFactory.build()).data + data[field] = value + response = api_client.post(reverse(DEDUPLICATION_SET_LIST_VIEW), data=data, format=JSON) + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json() + assert len(errors) == 1 + assert field in errors diff --git a/tests/api/test_deduplication_set_delete.py b/tests/api/test_deduplication_set_delete.py new file mode 100644 index 00000000..0a0e5bf3 --- /dev/null +++ b/tests/api/test_deduplication_set_delete.py @@ -0,0 +1,33 @@ +from unittest.mock import MagicMock + +from const import DEDUPLICATION_SET_DETAIL_VIEW +from rest_framework import status +from rest_framework.reverse import reverse +from rest_framework.test import APIClient + +from hope_dedup_engine.apps.api.models import DeduplicationSet +from hope_dedup_engine.apps.security.models import User + + +def test_can_delete_deduplication_set(api_client: APIClient, user: User, deduplication_set: DeduplicationSet) -> None: + assert not deduplication_set.deleted + assert deduplication_set.updated_by is None + previous_amount = DeduplicationSet.objects.count() + + response = api_client.delete(reverse(DEDUPLICATION_SET_DETAIL_VIEW, (deduplication_set.pk,))) + assert response.status_code == status.HTTP_204_NO_CONTENT + + # object is only marked as deleted + assert DeduplicationSet.objects.count() == previous_amount + deduplication_set.refresh_from_db() + assert deduplication_set.deleted + assert deduplication_set.updated_by == user + + +def test_cannot_delete_deduplication_set_between_systems( + another_system_api_client: APIClient, deduplication_set: DeduplicationSet, delete_model_data: MagicMock +) -> None: + set_count = DeduplicationSet.objects.filter(deleted=False).count() + response = another_system_api_client.delete(reverse(DEDUPLICATION_SET_DETAIL_VIEW, (deduplication_set.pk,))) + assert response.status_code == status.HTTP_403_FORBIDDEN + assert DeduplicationSet.objects.filter(deleted=False).count() == set_count diff --git a/tests/api/test_deduplication_set_list.py b/tests/api/test_deduplication_set_list.py new file mode 100644 index 00000000..2260e07a --- /dev/null +++ b/tests/api/test_deduplication_set_list.py @@ -0,0 +1,23 @@ +from const import DEDUPLICATION_SET_LIST_VIEW +from rest_framework import status +from rest_framework.reverse import reverse +from rest_framework.test import APIClient + +from hope_dedup_engine.apps.api.models import DeduplicationSet + + +def test_can_list_deduplication_sets(api_client: APIClient, deduplication_set: DeduplicationSet) -> None: + response = api_client.get(reverse(DEDUPLICATION_SET_LIST_VIEW)) + assert response.status_code == status.HTTP_200_OK + data = response.json() + assert len(data) == 1 + + +def test_cannot_list_deduplication_sets_between_systems( + another_system_api_client: APIClient, deduplication_set: DeduplicationSet +) -> None: + assert DeduplicationSet.objects.count() + response = another_system_api_client.get(reverse(DEDUPLICATION_SET_LIST_VIEW)) + assert response.status_code == status.HTTP_200_OK + data = response.json() + assert len(data) == 0 diff --git a/tests/api/test_image_bulk_create.py b/tests/api/test_image_bulk_create.py new file mode 100644 index 00000000..46b4ca5e --- /dev/null +++ b/tests/api/test_image_bulk_create.py @@ -0,0 +1,36 @@ +from const import BULK_IMAGE_LIST_VIEW, JSON +from rest_framework import status +from rest_framework.reverse import reverse +from rest_framework.test import APIClient +from testutils.factories.api import ImageFactory + +from hope_dedup_engine.apps.api.models import DeduplicationSet +from hope_dedup_engine.apps.api.serializers import ImageSerializer +from hope_dedup_engine.apps.security.models import User + + +def test_can_bulk_create_images(api_client: APIClient, deduplication_set: DeduplicationSet) -> None: + data = ImageSerializer(ImageFactory.build_batch(10), many=True).data + response = api_client.post(reverse(BULK_IMAGE_LIST_VIEW, (deduplication_set.pk,)), data=data, format=JSON) + assert response.status_code == status.HTTP_201_CREATED + + +def test_cannot_bulk_create_images_between_systems( + another_system_api_client: APIClient, deduplication_set: DeduplicationSet +) -> None: + data = ImageSerializer(ImageFactory.build_batch(10), many=True).data + response = another_system_api_client.post( + reverse(BULK_IMAGE_LIST_VIEW, (deduplication_set.pk,)), data=data, format=JSON + ) + assert response.status_code == status.HTTP_403_FORBIDDEN + + +def test_deduplication_set_is_updated(api_client: APIClient, user: User, deduplication_set: DeduplicationSet) -> None: + assert deduplication_set.updated_by is None + + data = ImageSerializer(ImageFactory.build_batch(10), many=True).data + response = api_client.post(reverse(BULK_IMAGE_LIST_VIEW, (deduplication_set.pk,)), data=data, format=JSON) + + assert response.status_code == status.HTTP_201_CREATED + deduplication_set.refresh_from_db() + assert deduplication_set.updated_by == user diff --git a/tests/api/test_image_bulk_delete.py b/tests/api/test_image_bulk_delete.py new file mode 100644 index 00000000..dcce98af --- /dev/null +++ b/tests/api/test_image_bulk_delete.py @@ -0,0 +1,34 @@ +from const import BULK_IMAGE_CLEAR_VIEW +from rest_framework import status +from rest_framework.reverse import reverse +from rest_framework.test import APIClient + +from hope_dedup_engine.apps.api.models import DeduplicationSet +from hope_dedup_engine.apps.api.models.deduplication import Image +from hope_dedup_engine.apps.security.models import User + + +def test_can_delete_all_images(api_client: APIClient, deduplication_set: DeduplicationSet, image: Image) -> None: + image_count = Image.objects.filter(deduplication_set=deduplication_set).count() + response = api_client.delete(reverse(BULK_IMAGE_CLEAR_VIEW, (deduplication_set.pk,))) + assert response.status_code == status.HTTP_204_NO_CONTENT + assert Image.objects.filter(deduplication_set=deduplication_set).count() == image_count - 1 + + +def test_cannot_delete_images_between_systems( + another_system_api_client: APIClient, deduplication_set: DeduplicationSet, image: Image +) -> None: + image_count = Image.objects.filter(deduplication_set=deduplication_set).count() + response = another_system_api_client.delete(reverse(BULK_IMAGE_CLEAR_VIEW, (deduplication_set.pk,))) + assert response.status_code == status.HTTP_403_FORBIDDEN + assert Image.objects.filter(deduplication_set=deduplication_set).count() == image_count + + +def test_deduplication_set_is_updated( + api_client: APIClient, user: User, deduplication_set: DeduplicationSet, image: Image +) -> None: + assert deduplication_set.updated_by is None + response = api_client.delete(reverse(BULK_IMAGE_CLEAR_VIEW, (deduplication_set.pk,))) + assert response.status_code == status.HTTP_204_NO_CONTENT + deduplication_set.refresh_from_db() + assert deduplication_set.updated_by == user diff --git a/tests/api/test_image_create.py b/tests/api/test_image_create.py new file mode 100644 index 00000000..89c01ad4 --- /dev/null +++ b/tests/api/test_image_create.py @@ -0,0 +1,71 @@ +from const import IMAGE_LIST_VIEW, JSON +from pytest import mark +from rest_framework import status +from rest_framework.reverse import reverse +from rest_framework.test import APIClient +from testutils.factories.api import ImageFactory + +from hope_dedup_engine.apps.api.models import DeduplicationSet +from hope_dedup_engine.apps.api.models.deduplication import Image +from hope_dedup_engine.apps.api.serializers import ImageSerializer +from hope_dedup_engine.apps.security.models import User + + +def test_can_create_image(api_client: APIClient, deduplication_set: DeduplicationSet) -> None: + previous_amount = Image.objects.filter(deduplication_set=deduplication_set).count() + data = ImageSerializer(ImageFactory.build()).data + + response = api_client.post(reverse(IMAGE_LIST_VIEW, (deduplication_set.pk,)), data=data, format=JSON) + assert response.status_code == status.HTTP_201_CREATED + assert Image.objects.filter(deduplication_set=deduplication_set).count() == previous_amount + 1 + + +def test_cannot_create_image_between_systems( + another_system_api_client: APIClient, deduplication_set: DeduplicationSet +) -> None: + previous_amount = Image.objects.filter(deduplication_set=deduplication_set).count() + data = ImageSerializer(ImageFactory.build()).data + + response = another_system_api_client.post(reverse(IMAGE_LIST_VIEW, (deduplication_set.pk,)), data=data, format=JSON) + assert response.status_code == status.HTTP_403_FORBIDDEN + assert Image.objects.filter(deduplication_set=deduplication_set).count() == previous_amount + + +@mark.parametrize( + "filename", + ( + "", + None, + ), +) +def test_invalid_values_handling( + api_client: APIClient, deduplication_set: DeduplicationSet, filename: str | None +) -> None: + data = ImageSerializer(ImageFactory.build()).data + data["filename"] = filename + response = api_client.post(reverse(IMAGE_LIST_VIEW, (deduplication_set.pk,)), data=data, format=JSON) + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json() + assert len(errors) == 1 + assert "filename" in errors + + +def test_missing_filename_handling(api_client: APIClient, deduplication_set: DeduplicationSet) -> None: + data = ImageSerializer(ImageFactory.build()).data + del data["filename"] + + response = api_client.post(reverse(IMAGE_LIST_VIEW, (deduplication_set.pk,)), data=data, format=JSON) + assert response.status_code == status.HTTP_400_BAD_REQUEST + errors = response.json() + assert "filename" in errors + + +def test_deduplication_set_is_updated(api_client: APIClient, user: User, deduplication_set: DeduplicationSet) -> None: + assert deduplication_set.updated_by is None + + data = ImageSerializer(ImageFactory.build()).data + response = api_client.post(reverse(IMAGE_LIST_VIEW, (deduplication_set.pk,)), data=data, format=JSON) + + assert response.status_code == status.HTTP_201_CREATED + deduplication_set.refresh_from_db() + assert deduplication_set.updated_by == user diff --git a/tests/api/test_image_delete.py b/tests/api/test_image_delete.py new file mode 100644 index 00000000..99815759 --- /dev/null +++ b/tests/api/test_image_delete.py @@ -0,0 +1,33 @@ +from const import IMAGE_DETAIL_VIEW +from rest_framework import status +from rest_framework.reverse import reverse +from rest_framework.test import APIClient + +from hope_dedup_engine.apps.api.models.deduplication import DeduplicationSet, Image +from hope_dedup_engine.apps.security.models import User + + +def test_can_delete_image(api_client: APIClient, deduplication_set: DeduplicationSet, image: Image) -> None: + image_count = Image.objects.filter(deduplication_set=deduplication_set).count() + response = api_client.delete(reverse(IMAGE_DETAIL_VIEW, (deduplication_set.pk, image.pk))) + assert response.status_code == status.HTTP_204_NO_CONTENT + assert Image.objects.filter(deduplication_set=deduplication_set).count() == image_count - 1 + + +def test_cannot_delete_image_between_systems( + another_system_api_client: APIClient, deduplication_set: DeduplicationSet, image: Image +) -> None: + image_count = Image.objects.filter(deduplication_set=deduplication_set).count() + response = another_system_api_client.delete(reverse(IMAGE_DETAIL_VIEW, (deduplication_set.pk, image.pk))) + assert response.status_code == status.HTTP_403_FORBIDDEN + assert Image.objects.filter(deduplication_set=deduplication_set).count() == image_count + + +def test_deduplication_set_is_updated( + api_client: APIClient, user: User, deduplication_set: DeduplicationSet, image: Image +) -> None: + assert deduplication_set.updated_by is None + response = api_client.delete(reverse(IMAGE_DETAIL_VIEW, (deduplication_set.pk, image.pk))) + assert response.status_code == status.HTTP_204_NO_CONTENT + deduplication_set.refresh_from_db() + assert deduplication_set.updated_by == user diff --git a/tests/api/test_image_list.py b/tests/api/test_image_list.py new file mode 100644 index 00000000..3e780f30 --- /dev/null +++ b/tests/api/test_image_list.py @@ -0,0 +1,22 @@ +from api.const import IMAGE_LIST_VIEW +from rest_framework import status +from rest_framework.reverse import reverse +from rest_framework.test import APIClient + +from hope_dedup_engine.apps.api.models import DeduplicationSet +from hope_dedup_engine.apps.api.models.deduplication import Image + + +def test_can_list_images(api_client: APIClient, deduplication_set: DeduplicationSet, image: Image) -> None: + response = api_client.get(reverse(IMAGE_LIST_VIEW, (deduplication_set.pk,))) + assert response.status_code == status.HTTP_200_OK + images = response.json() + assert len(images) + assert len(images) == Image.objects.filter(deduplication_set=deduplication_set).count() + + +def test_cannot_list_images_between_systems( + another_system_api_client: APIClient, deduplication_set: DeduplicationSet +) -> None: + response = another_system_api_client.get(reverse(IMAGE_LIST_VIEW, (deduplication_set.pk,))) + assert response.status_code == status.HTTP_403_FORBIDDEN diff --git a/tests/extras/testutils/factories/__init__.py b/tests/extras/testutils/factories/__init__.py index 1595d1de..630527ee 100644 --- a/tests/extras/testutils/factories/__init__.py +++ b/tests/extras/testutils/factories/__init__.py @@ -3,7 +3,7 @@ from .base import AutoRegisterModelFactory, TAutoRegisterModelFactory, factories_registry from .django_celery_beat import PeriodicTaskFactory # noqa from .social import SocialAuthUserFactory # noqa -from .user import GroupFactory, SuperUserFactory, User, UserFactory # noqa +from .user import ExternalSystemFactory, GroupFactory, SuperUserFactory, User, UserFactory # noqa from .userrole import UserRole, UserRoleFactory # noqa diff --git a/tests/extras/testutils/factories/api.py b/tests/extras/testutils/factories/api.py new file mode 100644 index 00000000..b38b1301 --- /dev/null +++ b/tests/extras/testutils/factories/api.py @@ -0,0 +1,29 @@ +from factory import SubFactory, fuzzy +from factory.django import DjangoModelFactory +from testutils.factories import ExternalSystemFactory + +from hope_dedup_engine.apps.api.models import DeduplicationSet, HDEToken +from hope_dedup_engine.apps.api.models.deduplication import Image + + +class TokenFactory(DjangoModelFactory): + class Meta: + model = HDEToken + + +class DeduplicationSetFactory(DjangoModelFactory): + name = fuzzy.FuzzyText() + reference_pk = fuzzy.FuzzyInteger(low=1) + external_system = SubFactory(ExternalSystemFactory) + state = DeduplicationSet.State.CLEAN + + class Meta: + model = DeduplicationSet + + +class ImageFactory(DjangoModelFactory): + filename = fuzzy.FuzzyText() + deduplication_set = SubFactory(DeduplicationSetFactory) + + class Meta: + model = Image diff --git a/tests/extras/testutils/factories/user.py b/tests/extras/testutils/factories/user.py index 371aabf8..b2af0c3a 100644 --- a/tests/extras/testutils/factories/user.py +++ b/tests/extras/testutils/factories/user.py @@ -1,17 +1,25 @@ from django.contrib.auth.models import Group -import factory +import factory.fuzzy -from hope_dedup_engine.apps.security.models import User +from hope_dedup_engine.apps.security.models import User, ExternalSystem from .base import AutoRegisterModelFactory +class ExternalSystemFactory(AutoRegisterModelFactory): + name = factory.fuzzy.FuzzyText() + + class Meta: + model = ExternalSystem + + class UserFactory(AutoRegisterModelFactory): _password = "password" username = factory.Sequence(lambda n: "m%03d@example.com" % n) password = factory.django.Password(_password) email = factory.Sequence(lambda n: "m%03d@example.com" % n) + external_system = factory.SubFactory(ExternalSystemFactory) class Meta: model = User diff --git a/tests/social/conftest.py b/tests/social/conftest.py index fc092094..72400198 100644 --- a/tests/social/conftest.py +++ b/tests/social/conftest.py @@ -1,5 +1,6 @@ from pytest_factoryboy import register -from testutils.factories import GroupFactory, UserFactory +from testutils.factories import ExternalSystemFactory, GroupFactory, UserFactory +register(ExternalSystemFactory) register(UserFactory) register(GroupFactory) diff --git a/tests/test_commands.py b/tests/test_commands.py index 551ef0a8..25d879fe 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -115,6 +115,7 @@ def test_env_raise(mocked_responses): with pytest.raises(CommandError): call_command("env", ignore_errors=False, check=True) + def test_upgrade_exception(mocked_responses, environment): with mock.patch("hope_dedup_engine.apps.core.management.commands.upgrade.call_command") as m: m.side_effect = Exception @@ -122,6 +123,6 @@ def test_upgrade_exception(mocked_responses, environment): call_command("upgrade") out = StringIO() - with mock.patch.dict(os.environ, {"ADMIN_EMAIL": "2222", "ADMIN_USER":"admin", **environment}, clear=True): + with mock.patch.dict(os.environ, {"ADMIN_EMAIL": "2222", "ADMIN_USER": "admin", **environment}, clear=True): with pytest.raises(SystemExit): call_command("upgrade", stdout=out, check=True, admin_email="") diff --git a/tests/test_state.py b/tests/test_state.py index 25679923..90502366 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -1,5 +1,3 @@ -from http.cookies import SimpleCookie - from django.http import HttpResponse import pytest diff --git a/tests/test_storage.py b/tests/test_storage.py index 21f73303..4a313d1e 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -1,26 +1,27 @@ +from django.core.files.base import ContentFile + import pytest from hope_dedup_engine.apps.core.storage import DataSetStorage, HOPEAzureStorage -from django.core.files.base import ContentFile def test_fs(tmp_path): s = DataSetStorage(tmp_path) - s.save('test', ContentFile("aa", "test.txt")) - s.save('test', ContentFile("bb", "test.txt")) - assert s.listdir(".") == ([], ['test']) - with s.open('test') as fd: + s.save("test", ContentFile("aa", "test.txt")) + s.save("test", ContentFile("bb", "test.txt")) + assert s.listdir(".") == ([], ["test"]) + with s.open("test") as fd: assert fd.read() == b"bb" def test_azure(tmp_path): s = HOPEAzureStorage() with pytest.raises(RuntimeError): - s.open('test', "rw") + s.open("test", "rw") with pytest.raises(RuntimeError): - s.save('test', ContentFile("aa", "test.txt")) + s.save("test", ContentFile("aa", "test.txt")) with pytest.raises(RuntimeError): - s.delete('test') + s.delete("test") assert s.listdir(".") == [] - assert s.open('test', "r") + assert s.open("test", "r") diff --git a/tests/utils/test_utils_http.py b/tests/utils/test_utils_http.py index 28927b65..2a696f2c 100644 --- a/tests/utils/test_utils_http.py +++ b/tests/utils/test_utils_http.py @@ -20,11 +20,13 @@ def r(monkeypatch, rf: "RequestFactory"): m.stop() -def test_absolute_reverse(): +def test_absolute_reverse(settings): + settings.SOCIAL_AUTH_REDIRECT_IS_HTTPS = False assert absolute_reverse("home") == "http://127.0.0.1/" def test_absolute_uri(settings): + settings.SOCIAL_AUTH_REDIRECT_IS_HTTPS = False assert absolute_uri("aa") == "http://127.0.0.1/aa" assert absolute_uri("") == "http://127.0.0.1/" settings.SOCIAL_AUTH_REDIRECT_IS_HTTPS = True @@ -40,6 +42,7 @@ def test_get_server_host(): def test_get_server_url(settings): + settings.SOCIAL_AUTH_REDIRECT_IS_HTTPS = False assert get_server_url() == "http://127.0.0.1" with state.configure(request=None): assert get_server_url() == ""