chg ! use svelte jsoneditor

pyproject.toml

@@ -44,6 +44,7 @@ dependencies = [
     "setuptools>=74.1.2",
     "django-smart-env>=0.1.0",
     "jsonschema>=4.23.0",
+    "django-svelte-jsoneditor>=0.4.2",
 ]
 
 [build-system]

src/hope_dedup_engine/apps/api/admin/__init__.py

@@ -1,11 +1,5 @@
-from django.contrib import admin
-
 from .config import ConfigAdmin  # noqa
 from .deduplicationset import DeduplicationSetAdmin  # noqa
 from .duplicate import DuplicateAdmin  # noqa
 from .hdetoken import HDETokenAdmin  # noqa
 from .image import ImageAdmin  # noqa
-
-admin.site.site_header = "HOPE Dedup Engine"
-admin.site.site_title = "HOPE Deduplication Admin"
-admin.site.index_title = "Welcome to the HOPE Deduplication Engine Admin"

src/hope_dedup_engine/apps/api/admin/config.py

@@ -2,17 +2,44 @@
 from typing import Any
 
 from django.contrib import messages
-from django.contrib.admin import ModelAdmin, register
+from django.contrib.admin import ModelAdmin, register, site
+from django.core.exceptions import ValidationError
+from django.db import models
 from django.http import HttpRequest, HttpResponse
 from django.shortcuts import redirect, render
 from django.urls import path, reverse
 
+from admin_extra_buttons.api import button
+from admin_extra_buttons.mixins import ExtraButtonsMixin
+from django_svelte_jsoneditor.widgets import SvelteJSONEditorWidget
+
+from hope_dedup_engine.apps.api.forms import EditSchemaForm
 from hope_dedup_engine.apps.api.models import Config
+from hope_dedup_engine.apps.api.validators import DefaultValidatingValidator
+from hope_dedup_engine.utils.security import is_root
+from src.hope_dedup_engine.apps.api.utils.shema_manager import SchemaManager
 
 
 @register(Config)
-class ConfigAdmin(ModelAdmin):
+class ConfigAdmin(ExtraButtonsMixin, ModelAdmin):
     list_display = ("name", "settings")
+    change_list_template = "admin/api/config/change_list.html"
+
+    formfield_overrides = {
+        models.JSONField: {
+            "widget": SvelteJSONEditorWidget,
+        }
+    }
+
+    def get_changeform_initial_data(self, request: HttpRequest) -> dict[str, str]:
+        initial_data = super().get_changeform_initial_data(request)
+        initial_data["settings"] = {}
+        try:
+            schema = SchemaManager.get_or_create()
+            DefaultValidatingValidator(schema).validate(initial_data["settings"])
+        except ValidationError as e:
+            self.message_user(request, e.message, level=messages.ERROR)
+        return initial_data
 
     def get_urls(self):
         urls = super().get_urls()
@@ -22,6 +49,11 @@ def get_urls(self):
                 self.admin_site.admin_view(self.confirm_save),
                 name="confirm_save_config",
             ),
+            path(
+                "change-settings-schema/",
+                self.admin_site.admin_view(self.change_settings_schema),
+                name="change_settings_schema",
+            ),
         ]
         return custom_urls + urls
 
@@ -38,7 +70,7 @@ def response_change(self, request: HttpRequest, obj: Any) -> HttpResponse:
             return redirect(confirm_url)
         return super().response_change(request, obj)
 
-    def confirm_save(self, request, object_id):  # pragma: no cover
+    def confirm_save(self, request, object_id) -> HttpResponse:  # pragma: no cover
         obj = self.get_object(request, object_id)
         if request.method == "POST":
             form_data = request.session.get("unsaved_data", None)
@@ -59,3 +91,41 @@ def confirm_save(self, request, object_id):  # pragma: no cover
                 "form_data": request.session.get("unsaved_data"),
             },
         )
+
+    @button(permission=is_root)
+    def change_settings_schema(
+        self, request: HttpRequest
+    ) -> HttpResponse:  # pragma: no cover
+        context = {
+            "opts": self.model._meta,
+            "site_header": site.site_header,
+            "title": "Change settings shema",
+            "trail_label": "Settings schema",
+            "has_view_permission": self.has_view_permission(request),
+        }
+
+        if request.method == "POST":
+            form = EditSchemaForm(request.POST)
+            if form.is_valid():
+                try:
+                    SchemaManager.save(form.cleaned_data["schema"])
+                except ValidationError as e:
+                    self.message_user(request, e.message, level=messages.ERROR)
+                else:
+                    self.message_user(request, "Schema has been updated.")
+                    return redirect(reverse("admin:api_config_changelist"))
+        else:
+            try:
+                form = EditSchemaForm(initial={"schema": SchemaManager.get_or_create()})
+            except ValidationError as e:
+                self.message_user(request, e.message, level=messages.ERROR)
+                return redirect(reverse("admin:api_config_changelist"))
+
+        return render(
+            request,
+            "admin/api/config/change_settings_schema.html",
+            {
+                "form": form,
+                **context,
+            },
+        )

src/hope_dedup_engine/apps/api/admin/deduplicationset.py

@@ -50,7 +50,9 @@ def has_add_permission(self, request):
         return False
 
     @button(permission=can_reprocess)
-    def process(self, request: HttpRequest, pk: UUID) -> HttpResponseRedirect:
+    def process(
+        self, request: HttpRequest, pk: UUID
+    ) -> HttpResponseRedirect:  # pragma: no cover
         obj = self.get_object(request, pk)
         start_processing(obj)
         self.message_user(

src/hope_dedup_engine/apps/api/config_settings_schema.json

@@ -0,0 +1,72 @@
+{
+    "type": "object",
+    "properties": {
+        "detection": {
+            "type": "object",
+            "properties": {
+                "confidence": {
+                    "type": "number",
+                    "exclusiveMinimum": 0,
+                    "maximum": 1,
+                    "default": "constance.config.FACE_DETECTION_CONFIDENCE"
+                }
+            },
+            "default": {},
+            "required": [
+                "confidence"
+            ]
+        },
+        "recognition": {
+            "type": "object",
+            "properties": {
+                "num_jitters": {
+                    "type": "integer",
+                    "minimum": 1,
+                    "default": "constance.config.FACE_ENCODINGS_NUM_JITTERS"
+                },
+                "model": {
+                    "type": "string",
+                    "enum": [
+                        "small",
+                        "large"
+                    ],
+                    "default": "constance.config.FACE_ENCODINGS_MODEL"
+                },
+                "preprocessors": {
+                    "type": "array",
+                    "items": {
+                        "type": "string",
+                        "enum": []
+                    },
+                    "uniquItems": true,
+                    "default": []
+                }
+            },
+            "default": {},
+            "required": [
+                "num_jitters",
+                "model"
+            ]
+        },
+        "duplicates": {
+            "type": "object",
+            "properties": {
+                "tolerance": {
+                    "type": "number",
+                    "exclusiveMinimum": 0,
+                    "maximum": 1,
+                    "default": "constance.config.FACE_DISTANCE_THRESHOLD"
+                }
+            },
+            "default": {},
+            "required": [
+                "tolerance"
+            ]
+        }
+    },
+    "required": [
+        "detection",
+        "recognition",
+        "duplicates"
+    ]
+}

src/hope_dedup_engine/apps/api/deduplication/adapters.py

@@ -1,6 +1,6 @@
 from collections.abc import Generator
-from typing import Any
 
+from hope_dedup_engine.apps.api.deduplication.config import get_config
 from hope_dedup_engine.apps.api.deduplication.registry import DuplicateKeyPair
 from hope_dedup_engine.apps.api.models import DeduplicationSet
 from hope_dedup_engine.apps.faces.services.duplication_detector import (
@@ -21,12 +21,10 @@ def run(self) -> Generator[DuplicateKeyPair, None, None]:
                 "reference_pk", "filename"
             )
         }
-        ds_config: dict[str, Any] = (
-            self.deduplication_set.config and self.deduplication_set.config.settings
-        ) or {}
         # ignored key pairs are not handled correctly in DuplicationDetector
         detector = DuplicationDetector(
-            tuple[str](filename_to_reference_pk.keys()), ds_config
+            tuple[str](filename_to_reference_pk.keys()),
+            get_config(self.deduplication_set.config),
         )
         for first_filename, second_filename, distance in detector.find_duplicates():
             yield filename_to_reference_pk[first_filename], filename_to_reference_pk[

src/hope_dedup_engine/apps/api/deduplication/config.py

@@ -0,0 +1,62 @@
+from typing import Any, Literal, TypedDict
+
+from constance import config as constance_cfg
+
+from hope_dedup_engine.apps.api.models import Config
+
+
+class DetectionConfig(TypedDict):
+    dnn_files_source: str
+    dnn_backend: int
+    dnn_target: int
+    blob_from_image_scale_factor: float
+    blob_from_image_mean_values: str
+    confidence: float
+    nms_threshold: float
+
+
+class RecognitionConfig(TypedDict):
+    num_jitters: int
+    model: Literal["small", "large"]
+    preprocessors: list[str]
+
+
+class DuplicatesConfig(TypedDict):
+    tolerance: float
+
+
+class ConfigDefaults(TypedDict):
+    detection: DetectionConfig
+    recognition: RecognitionConfig
+    duplicates: DuplicatesConfig
+
+
+def get_config(config: Config | None = None) -> dict[str, Any]:
+    DEFAULT: ConfigDefaults = {
+        "detection": {
+            "dnn_files_source": constance_cfg.DNN_FILES_SOURCE,
+            "dnn_backend": constance_cfg.DNN_BACKEND,
+            "dnn_target": constance_cfg.DNN_TARGET,
+            "blob_from_image_scale_factor": constance_cfg.BLOB_FROM_IMAGE_SCALE_FACTOR,
+            "blob_from_image_mean_values": constance_cfg.BLOB_FROM_IMAGE_MEAN_VALUES,
+            "confidence": constance_cfg.FACE_DETECTION_CONFIDENCE,
+            "nms_threshold": constance_cfg.NMS_THRESHOLD,
+        },
+        "recognition": {
+            "num_jitters": constance_cfg.FACE_ENCODINGS_NUM_JITTERS,
+            "model": constance_cfg.FACE_ENCODINGS_MODEL,
+            "preprocessors": [],
+        },
+        "duplicates": {
+            "tolerance": constance_cfg.FACE_DISTANCE_THRESHOLD,
+        },
+    }
+    config_settings = {}
+
+    for section, defaults in DEFAULT.items():
+        section_settings = defaults.copy()
+        if config and config.settings:
+            section_settings.update(config.settings.get(section, {}))
+        config_settings[section] = section_settings
+
+    return config_settings

src/hope_dedup_engine/apps/api/forms.py

@@ -0,0 +1,7 @@
+from django import forms
+
+from django_svelte_jsoneditor.widgets import SvelteJSONEditorWidget
+
+
+class EditSchemaForm(forms.Form):
+    schema = forms.JSONField(widget=SvelteJSONEditorWidget())

src/hope_dedup_engine/apps/api/models/config.py

@@ -1,12 +1,8 @@
 from django.core.exceptions import ValidationError
 from django.db import models
 
-from jsonschema import ValidationError as JSONSchemaValidationError
-
-from hope_dedup_engine.apps.api.utils.config_schema import (
-    DefaultValidatingValidator,
-    settings_schema,
-)
+from hope_dedup_engine.apps.api.validators import DefaultValidatingValidator
+from src.hope_dedup_engine.apps.api.utils.shema_manager import SchemaManager
 
 
 class Config(models.Model):
@@ -20,6 +16,7 @@ def __str__(self) -> str:
 
     def clean(self) -> None:
         try:
-            DefaultValidatingValidator(settings_schema).validate(self.settings)
-        except JSONSchemaValidationError as e:
+            schema = SchemaManager.get_or_create()
+            DefaultValidatingValidator(schema).validate(self.settings)
+        except Exception as e:
             raise ValidationError({"settings": e.message})

src/hope_dedup_engine/apps/api/serializers.py

@@ -11,7 +11,7 @@
     IgnoredReferencePkPair,
     Image,
 )
-from hope_dedup_engine.apps.api.utils.config_schema import settings_schema
+from src.hope_dedup_engine.apps.api.utils.shema_manager import SchemaManager
 
 
 class ConfigSerializer(serializers.ModelSerializer):
@@ -20,7 +20,7 @@ class Meta:
         exclude = ("id",)
 
     def validate_settings(self, value):
-        validator = Draft202012Validator(settings_schema)
+        validator = Draft202012Validator(SchemaManager.get_or_create())
         try:
             validator.validate(value)
         except JSONSchemaValidationError as e: