lint

.github/workflows/test.yml

@@ -162,6 +162,13 @@ jobs:
           sarif_file: 'trivy-results.sarif'
 
   release:
+    if:
+      contains('
+      refs/heads/develop
+      refs/heads/staging
+      refs/heads/master
+      ', github.ref) || contains(github.event.head_commit.message, 'ci:release')
+
     name: "Release Docker"
     needs: [ test ]
     runs-on: ubuntu-latest

src/hope_dedup_engine/__init__.py

@@ -1,6 +1,5 @@
 from hope_dedup_engine.config.celery import app as celery_app
 
-
 VERSION = __version__ = "0.1.0"
 
 __all__ = ("celery_app",)

src/hope_dedup_engine/apps/api/admin.py

@@ -1,6 +1,11 @@
 from django.contrib import admin
 
-from hope_dedup_engine.apps.api.models import DeduplicationSet, Duplicate, HDEToken, Image
+from hope_dedup_engine.apps.api.models import (
+    DeduplicationSet,
+    Duplicate,
+    HDEToken,
+    Image,
+)
 
 admin.site.register(DeduplicationSet)
 admin.site.register(Duplicate)

src/hope_dedup_engine/apps/api/auth.py

@@ -14,7 +14,9 @@ def has_permission(self, request: Request, view: View) -> bool:
 
 class UserAndDeduplicationSetAreOfTheSameSystem(BasePermission):
     def has_permission(self, request: Request, view: View) -> bool:
-        if deduplication_set_pk := view.kwargs.get("deduplication_set_pk") or view.kwargs.get("pk"):
+        if deduplication_set_pk := view.kwargs.get(
+            "deduplication_set_pk"
+        ) or view.kwargs.get("pk"):
             return DeduplicationSet.objects.filter(
                 external_system=request.user.external_system, pk=deduplication_set_pk
             ).exists()

src/hope_dedup_engine/apps/api/models/__init__.py

@@ -1,2 +1,6 @@
 from hope_dedup_engine.apps.api.models.auth import HDEToken  # noqa: F401
-from hope_dedup_engine.apps.api.models.deduplication import DeduplicationSet, Duplicate, Image  # noqa: F401
+from hope_dedup_engine.apps.api.models.deduplication import (  # noqa: F401
+    DeduplicationSet,
+    Duplicate,
+    Image,
+)

src/hope_dedup_engine/apps/api/models/auth.py

@@ -5,4 +5,6 @@
 
 
 class HDEToken(Token):
-    user = models.ForeignKey(settings.AUTH_USER_MODEL, related_name="auth_tokens", on_delete=models.CASCADE)
+    user = models.ForeignKey(
+        settings.AUTH_USER_MODEL, related_name="auth_tokens", on_delete=models.CASCADE
+    )

src/hope_dedup_engine/apps/api/models/deduplication.py

@@ -12,7 +12,10 @@
 class DeduplicationSet(models.Model):
     class State(models.IntegerChoices):
         CLEAN = 0, "Clean"  # Deduplication set is created or already processed
-        DIRTY = 1, "Dirty"  # Images are added to deduplication set, but not yet processed
+        DIRTY = (
+            1,
+            "Dirty",
+        )  # Images are added to deduplication set, but not yet processed
         PROCESSING = 2, "Processing"  # Images are being processed
         ERROR = 3, "Error"  # Error occurred
 
@@ -27,11 +30,19 @@ class State(models.IntegerChoices):
     external_system = models.ForeignKey(ExternalSystem, on_delete=models.CASCADE)
     error = models.CharField(max_length=255, null=True, blank=True)
     created_by = models.ForeignKey(
-        settings.AUTH_USER_MODEL, on_delete=models.CASCADE, null=True, blank=True, related_name="+"
+        settings.AUTH_USER_MODEL,
+        on_delete=models.CASCADE,
+        null=True,
+        blank=True,
+        related_name="+",
     )
     created_at = models.DateTimeField(auto_now_add=True)
     updated_by = models.ForeignKey(
-        settings.AUTH_USER_MODEL, on_delete=models.CASCADE, null=True, blank=True, related_name="+"
+        settings.AUTH_USER_MODEL,
+        on_delete=models.CASCADE,
+        null=True,
+        blank=True,
+        related_name="+",
     )
     updated_at = models.DateTimeField(auto_now=True)
     notification_url = models.CharField(max_length=255, null=True, blank=True)
@@ -43,7 +54,11 @@ class Image(models.Model):
     reference_pk = models.CharField(max_length=REFERENCE_PK_LENGTH)
     filename = models.CharField(max_length=255)
     created_by = models.ForeignKey(
-        settings.AUTH_USER_MODEL, on_delete=models.CASCADE, null=True, blank=True, related_name="+"
+        settings.AUTH_USER_MODEL,
+        on_delete=models.CASCADE,
+        null=True,
+        blank=True,
+        related_name="+",
     )
     created_at = models.DateTimeField(auto_now_add=True)
 
@@ -63,9 +78,15 @@ class IgnoredKeyPair(models.Model):
     second_reference_pk = models.CharField(max_length=REFERENCE_PK_LENGTH)
 
     class Meta:
-        unique_together = "deduplication_set", "first_reference_pk", "second_reference_pk"
+        unique_together = (
+            "deduplication_set",
+            "first_reference_pk",
+            "second_reference_pk",
+        )
 
     @override
     def save(self, **kwargs: Any) -> None:
-        self.first_reference_pk, self.second_reference_pk = sorted((self.first_reference_pk, self.second_reference_pk))
+        self.first_reference_pk, self.second_reference_pk = sorted(
+            (self.first_reference_pk, self.second_reference_pk)
+        )
         super().save(**kwargs)

src/hope_dedup_engine/apps/api/urls.py

@@ -20,12 +20,25 @@
 )
 
 router = routers.SimpleRouter()
-router.register(DEDUPLICATION_SET_LIST, DeduplicationSetViewSet, basename=DEDUPLICATION_SET_LIST)
+router.register(
+    DEDUPLICATION_SET_LIST, DeduplicationSetViewSet, basename=DEDUPLICATION_SET_LIST
+)
 
-deduplication_sets_router = nested_routers.NestedSimpleRouter(router, DEDUPLICATION_SET_LIST, lookup=DEDUPLICATION_SET)
+deduplication_sets_router = nested_routers.NestedSimpleRouter(
+    router, DEDUPLICATION_SET_LIST, lookup=DEDUPLICATION_SET
+)
 deduplication_sets_router.register(IMAGE_LIST, ImageViewSet, basename=IMAGE_LIST)
-deduplication_sets_router.register(BULK_IMAGE_LIST, BulkImageViewSet, basename=BULK_IMAGE_LIST)
-deduplication_sets_router.register(DUPLICATE_LIST, DuplicateViewSet, basename=DUPLICATE_LIST)
-deduplication_sets_router.register(IGNORED_KEYS_LIST, IgnoredKeyPairViewSet, basename=IGNORED_KEYS_LIST)
+deduplication_sets_router.register(
+    BULK_IMAGE_LIST, BulkImageViewSet, basename=BULK_IMAGE_LIST
+)
+deduplication_sets_router.register(
+    DUPLICATE_LIST, DuplicateViewSet, basename=DUPLICATE_LIST
+)
+deduplication_sets_router.register(
+    IGNORED_KEYS_LIST, IgnoredKeyPairViewSet, basename=IGNORED_KEYS_LIST
+)
 
-urlpatterns = [path("", include(router.urls)), path("", include(deduplication_sets_router.urls))]
+urlpatterns = [
+    path("", include(router.urls)),
+    path("", include(deduplication_sets_router.urls)),
+]

src/hope_dedup_engine/apps/core/management/commands/createsystem.py

@@ -10,7 +10,9 @@ def add_arguments(self, parser):
         parser.add_argument("name")
 
     def handle(self, *args, **options):
-        system, created = ExternalSystem.objects.get_or_create(name=(name := options["name"]))
+        system, created = ExternalSystem.objects.get_or_create(
+            name=(name := options["name"])
+        )
         if created:
             self.stdout.write(self.style.SUCCESS(f'"{name}" system created.'))
         else:

src/hope_dedup_engine/apps/core/management/commands/env.py

@@ -33,14 +33,26 @@ def add_arguments(self, parser: "CommandParser") -> None:
             default="export {key}={value}",
             help="Check env for variable availability (default: 'export {key}=\"{value}\"')",
         )
-        parser.add_argument("--develop", action="store_true", help="Display development values")
-        parser.add_argument("--config", action="store_true", help="Only list changed values")
+        parser.add_argument(
+            "--develop", action="store_true", help="Display development values"
+        )
+        parser.add_argument(
+            "--config", action="store_true", help="Only list changed values"
+        )
         parser.add_argument("--diff", action="store_true", help="Mark changed values")
         parser.add_argument(
-            "--check", action="store_true", dest="check", default=False, help="Check env for variable availability"
+            "--check",
+            action="store_true",
+            dest="check",
+            default=False,
+            help="Check env for variable availability",
         )
         parser.add_argument(
-            "--ignore-errors", action="store_true", dest="ignore_errors", default=False, help="Do not fail"
+            "--ignore-errors",
+            action="store_true",
+            dest="ignore_errors",
+            default=False,
+            help="Do not fail",
         )
 
     def handle(self, *args: "Any", **options: "Any") -> None:
@@ -62,7 +74,9 @@ def handle(self, *args: "Any", **options: "Any") -> None:
                 else:
                     value: Any = env.get_value(k)
 
-                line: str = pattern.format(key=k, value=clean(value), help=help, default=default)
+                line: str = pattern.format(
+                    key=k, value=clean(value), help=help, default=default
+                )
                 if options["diff"]:
                     if value != default:
                         line = self.style.SUCCESS(line)

src/hope_dedup_engine/apps/core/management/commands/upgrade.py

@@ -89,7 +89,9 @@ def get_options(self, options: dict[str, Any]) -> None:
         self.debug = options["debug"]
 
         self.admin_email = str(options["admin_email"] or env("ADMIN_EMAIL", ""))
-        self.admin_password = str(options["admin_password"] or env("ADMIN_PASSWORD", ""))
+        self.admin_password = str(
+            options["admin_password"] or env("ADMIN_PASSWORD", "")
+        )
 
     def halt(self, e: Exception) -> None:
         self.stdout.write(str(e), style_func=self.style.ERROR)
@@ -123,7 +125,9 @@ def handle(self, *args: Any, **options: Any) -> None:  # noqa: C901
                 call_command("check", deploy=True, verbosity=self.verbosity - 1)
             if self.static:
                 static_root = Path(env("STATIC_ROOT"))
-                echo(f"Run collectstatic to: '{static_root}' - '{static_root.absolute()}")
+                echo(
+                    f"Run collectstatic to: '{static_root}' - '{static_root.absolute()}"
+                )
                 if not static_root.exists():
                     static_root.mkdir(parents=True)
                 call_command("collectstatic", **extra)
@@ -144,7 +148,10 @@ def handle(self, *args: Any, **options: Any) -> None:  # noqa: C901
                         style_func=self.style.WARNING,
                     )
                 else:
-                    echo(f"Creating superuser: {self.admin_email}", style_func=self.style.WARNING)
+                    echo(
+                        f"Creating superuser: {self.admin_email}",
+                        style_func=self.style.WARNING,
+                    )
                     validate_email(self.admin_email)
                     os.environ["DJANGO_SUPERUSER_USERNAME"] = self.admin_email
                     os.environ["DJANGO_SUPERUSER_EMAIL"] = self.admin_email

src/hope_dedup_engine/apps/security/models.py

@@ -13,7 +13,9 @@ class ExternalSystem(models.Model):
 
 
 class User(SecurityMixin, AbstractUser):
-    external_system = models.ForeignKey(ExternalSystem, on_delete=models.SET_NULL, null=True, blank=True)
+    external_system = models.ForeignKey(
+        ExternalSystem, on_delete=models.SET_NULL, null=True, blank=True
+    )
 
     class Meta:
         abstract = False

src/hope_dedup_engine/apps/social/pipeline.py

@@ -6,7 +6,9 @@
 from social_core.backends.base import BaseAuth
 
 
-def save_to_group(backend: BaseAuth, user: Optional[User] = None, **kwargs: Any) -> dict[str, Any]:
+def save_to_group(
+    backend: BaseAuth, user: Optional[User] = None, **kwargs: Any
+) -> dict[str, Any]:
     if user:
         grp = Group.objects.get(name=config.NEW_USER_DEFAULT_GROUP)
         user.groups.add(grp)

src/hope_dedup_engine/config/__init__.py

@@ -5,7 +5,9 @@
 from environ import Env
 
 if TYPE_CHECKING:
-    ConfigItem: TypeAlias = Union[Tuple[type, Any, str, Any], Tuple[type, Any, str], Tuple[type, Any]]
+    ConfigItem: TypeAlias = Union[
+        Tuple[type, Any, str, Any], Tuple[type, Any, str], Tuple[type, Any]
+    ]
 
 
 DJANGO_HELP_BASE = "https://docs.djangoproject.com/en/5.0/ref/settings"
@@ -20,7 +22,14 @@ class Group(Enum):
 
 
 NOT_SET = "<- not set ->"
-EXPLICIT_SET = ["DATABASE_URL", "SECRET_KEY", "CACHE_URL", "CELERY_BROKER_URL", "MEDIA_ROOT", "STATIC_ROOT"]
+EXPLICIT_SET = [
+    "DATABASE_URL",
+    "SECRET_KEY",
+    "CACHE_URL",
+    "CELERY_BROKER_URL",
+    "MEDIA_ROOT",
+    "STATIC_ROOT",
+]
 
 CONFIG: "Dict[str, ConfigItem]" = {
     "ADMIN_EMAIL": (str, "", "Initial user created at first deploy"),
@@ -29,7 +38,11 @@ class Group(Enum):
     "AUTHENTICATION_BACKENDS": (list, [], setting("authentication-backends")),
     "CACHE_URL": (str, "redis://localhost:6379/0"),
     "CATCH_ALL_EMAIL": (str, "If set all the emails will be sent to this address"),
-    "CELERY_BROKER_URL": (str, NOT_SET, "https://docs.celeryq.dev/en/stable/django/first-steps-with-django.html"),
+    "CELERY_BROKER_URL": (
+        str,
+        NOT_SET,
+        "https://docs.celeryq.dev/en/stable/django/first-steps-with-django.html",
+    ),
     "CELERY_TASK_ALWAYS_EAGER": (
         bool,
         False,
@@ -54,21 +67,47 @@ class Group(Enum):
         "postgres://127.0.0.1:5432/dedupe",
     ),
     "DEBUG": (bool, False, setting("debug"), True),
-    "EMAIL_BACKEND": (str, "django.core.mail.backends.smtp.EmailBackend", setting("email-backend"), True),
+    "EMAIL_BACKEND": (
+        str,
+        "django.core.mail.backends.smtp.EmailBackend",
+        setting("email-backend"),
+        True,
+    ),
     "EMAIL_HOST": (str, "localhost", setting("email-host"), True),
     "EMAIL_HOST_USER": (str, "", setting("email-host-user"), True),
     "EMAIL_HOST_PASSWORD": (str, "", setting("email-host-password"), True),
     "EMAIL_PORT": (int, "25", setting("email-port"), True),
-    "EMAIL_SUBJECT_PREFIX": (str, "[Hope-dedupe]", setting("email-subject-prefix"), True),
+    "EMAIL_SUBJECT_PREFIX": (
+        str,
+        "[Hope-dedupe]",
+        setting("email-subject-prefix"),
+        True,
+    ),
     "EMAIL_USE_LOCALTIME": (bool, False, setting("email-use-localtime"), True),
     "EMAIL_USE_TLS": (bool, False, setting("email-use-tls"), True),
     "EMAIL_USE_SSL": (bool, False, setting("email-use-ssl"), True),
     "EMAIL_TIMEOUT": (str, None, setting("email-timeout"), True),
     "LOGGING_LEVEL": (str, "CRITICAL", setting("logging-level")),
-    "FILE_STORAGE_DEFAULT": (str, "django.core.files.storage.FileSystemStorage", setting("storages")),
-    "FILE_STORAGE_MEDIA": (str, "django.core.files.storage.FileSystemStorage", setting("storages")),
-    "FILE_STORAGE_STATIC": (str, "django.contrib.staticfiles.storage.StaticFilesStorage", setting("storages")),
-    "FILE_STORAGE_HOPE": (str, "django.core.files.storage.FileSystemStorage", setting("storages")),
+    "FILE_STORAGE_DEFAULT": (
+        str,
+        "django.core.files.storage.FileSystemStorage",
+        setting("storages"),
+    ),
+    "FILE_STORAGE_MEDIA": (
+        str,
+        "django.core.files.storage.FileSystemStorage",
+        setting("storages"),
+    ),
+    "FILE_STORAGE_STATIC": (
+        str,
+        "django.contrib.staticfiles.storage.StaticFilesStorage",
+        setting("storages"),
+    ),
+    "FILE_STORAGE_HOPE": (
+        str,
+        "django.core.files.storage.FileSystemStorage",
+        setting("storages"),
+    ),
     "MEDIA_ROOT": (str, None, setting("media-root")),
     "MEDIA_URL": (str, "/media/", setting("media-url")),
     "ROOT_TOKEN": (str, "", ""),
@@ -79,16 +118,29 @@ class Group(Enum):
     "SENTRY_DSN": (str, "", "Sentry DSN"),
     "SENTRY_ENVIRONMENT": (str, "production", "Sentry Environment"),
     "SENTRY_URL": (str, "", "Sentry server url"),
-    "SESSION_COOKIE_DOMAIN": (str, "", setting("std-setting-SESSION_COOKIE_DOMAIN"), "localhost"),
+    "SESSION_COOKIE_DOMAIN": (
+        str,
+        "",
+        setting("std-setting-SESSION_COOKIE_DOMAIN"),
+        "localhost",
+    ),
     "SESSION_COOKIE_HTTPONLY": (bool, True, setting("session-cookie-httponly"), False),
     "SESSION_COOKIE_NAME": (str, "dedupe_session", setting("session-cookie-name")),
     "SESSION_COOKIE_PATH": (str, "/", setting("session-cookie-path")),
     "SESSION_COOKIE_SECURE": (bool, True, setting("session-cookie-secure"), False),
-    "SIGNING_BACKEND": (str, "django.core.signing.TimestampSigner", setting("signing-backend")),
+    "SIGNING_BACKEND": (
+        str,
+        "django.core.signing.TimestampSigner",
+        setting("signing-backend"),
+    ),
     "SOCIAL_AUTH_LOGIN_URL": (str, "/login/", "", ""),
     "SOCIAL_AUTH_RAISE_EXCEPTIONS": (bool, False, "", True),
     "SOCIAL_AUTH_REDIRECT_IS_HTTPS": (bool, True, "", False),
-    "STATIC_FILE_STORAGE": (str, "django.core.files.storage.FileSystemStorage", setting("storages")),
+    "STATIC_FILE_STORAGE": (
+        str,
+        "django.core.files.storage.FileSystemStorage",
+        setting("storages"),
+    ),
     "STATIC_ROOT": (str, None, setting("static-root")),
     "STATIC_URL": (str, "/static/", setting("static-url")),
     "TIME_ZONE": (str, "UTC", setting("std-setting-TIME_ZONE")),