diff --git a/pdm.lock b/pdm.lock
index f9da9675..5328b528 100644
--- a/pdm.lock
+++ b/pdm.lock
@@ -5,7 +5,7 @@
groups = ["default", "dev"]
strategy = ["inherit_metadata"]
lock_version = "4.5.0"
-content_hash = "sha256:9ca78853e306e245544b961ba5f90ebef6c59c8021963a029c4713b55197ad80"
+content_hash = "sha256:2156364a0618f169212de0e69eb68ac20ce8fd7854dbe287ad76e5c74e1cd8ab"
[[metadata.targets]]
requires_python = ">=3.12"
@@ -811,6 +811,21 @@ files = [
{file = "django_storages-1.14.4-py3-none-any.whl", hash = "sha256:d61930acb4a25e3aebebc6addaf946a3b1df31c803a6bf1af2f31c9047febaa3"},
]
+[[package]]
+name = "django-svelte-jsoneditor"
+version = "0.4.2"
+requires_python = ">=3.9,<4.0"
+summary = "A widget for django's JSONField using the latest-and-greatest Json Editor"
+groups = ["default"]
+dependencies = [
+ "django<5.0,>=4.0; python_version >= \"3.9\" and python_version < \"3.10\"",
+ "django>=3.1; python_version ~= \"3.10\"",
+]
+files = [
+ {file = "django_svelte_jsoneditor-0.4.2-py3-none-any.whl", hash = "sha256:bfa715a796e5c8a0c3d0edf5b064b21417c65e101fa131ee4a7beb670bbef53c"},
+ {file = "django_svelte_jsoneditor-0.4.2.tar.gz", hash = "sha256:1cc73511297bc6e11d5bbd151543da22a36e67dcc6f652d92943174b88153024"},
+]
+
[[package]]
name = "django-timezone-field"
version = "7.0"
diff --git a/pyproject.toml b/pyproject.toml
index 9695f876..ee66de3b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -44,6 +44,7 @@ dependencies = [
"setuptools>=74.1.2",
"django-smart-env>=0.1.0",
"jsonschema>=4.23.0",
+ "django-svelte-jsoneditor>=0.4.2",
]
[build-system]
diff --git a/src/hope_dedup_engine/apps/api/admin/__init__.py b/src/hope_dedup_engine/apps/api/admin/__init__.py
index 2cd9c3d5..9c068505 100644
--- a/src/hope_dedup_engine/apps/api/admin/__init__.py
+++ b/src/hope_dedup_engine/apps/api/admin/__init__.py
@@ -1,11 +1,5 @@
-from django.contrib import admin
-
from .config import ConfigAdmin # noqa
from .deduplicationset import DeduplicationSetAdmin # noqa
from .duplicate import DuplicateAdmin # noqa
from .hdetoken import HDETokenAdmin # noqa
from .image import ImageAdmin # noqa
-
-admin.site.site_header = "HOPE Dedup Engine"
-admin.site.site_title = "HOPE Deduplication Admin"
-admin.site.index_title = "Welcome to the HOPE Deduplication Engine Admin"
diff --git a/src/hope_dedup_engine/apps/api/admin/config.py b/src/hope_dedup_engine/apps/api/admin/config.py
index a9cd0f6e..2bd75852 100644
--- a/src/hope_dedup_engine/apps/api/admin/config.py
+++ b/src/hope_dedup_engine/apps/api/admin/config.py
@@ -2,17 +2,44 @@
from typing import Any
from django.contrib import messages
-from django.contrib.admin import ModelAdmin, register
+from django.contrib.admin import ModelAdmin, register, site
+from django.core.exceptions import ValidationError
+from django.db import models
from django.http import HttpRequest, HttpResponse
from django.shortcuts import redirect, render
from django.urls import path, reverse
+from admin_extra_buttons.api import button
+from admin_extra_buttons.mixins import ExtraButtonsMixin
+from django_svelte_jsoneditor.widgets import SvelteJSONEditorWidget
+
+from hope_dedup_engine.apps.api.forms import EditSchemaForm
from hope_dedup_engine.apps.api.models import Config
+from hope_dedup_engine.apps.api.validators import DefaultValidatingValidator
+from hope_dedup_engine.utils.security import is_root
+from src.hope_dedup_engine.apps.api.utils.shema_manager import SchemaManager
@register(Config)
-class ConfigAdmin(ModelAdmin):
+class ConfigAdmin(ExtraButtonsMixin, ModelAdmin):
list_display = ("name", "settings")
+ change_list_template = "admin/api/config/change_list.html"
+
+ formfield_overrides = {
+ models.JSONField: {
+ "widget": SvelteJSONEditorWidget,
+ }
+ }
+
+ def get_changeform_initial_data(self, request: HttpRequest) -> dict[str, str]:
+ initial_data = super().get_changeform_initial_data(request)
+ initial_data["settings"] = {}
+ try:
+ schema = SchemaManager.get_or_create()
+ DefaultValidatingValidator(schema).validate(initial_data["settings"])
+ except ValidationError as e:
+ self.message_user(request, e.message, level=messages.ERROR)
+ return initial_data
def get_urls(self):
urls = super().get_urls()
@@ -22,6 +49,11 @@ def get_urls(self):
self.admin_site.admin_view(self.confirm_save),
name="confirm_save_config",
),
+ path(
+ "change-settings-schema/",
+ self.admin_site.admin_view(self.change_settings_schema),
+ name="change_settings_schema",
+ ),
]
return custom_urls + urls
@@ -38,7 +70,7 @@ def response_change(self, request: HttpRequest, obj: Any) -> HttpResponse:
return redirect(confirm_url)
return super().response_change(request, obj)
- def confirm_save(self, request, object_id): # pragma: no cover
+ def confirm_save(self, request, object_id) -> HttpResponse: # pragma: no cover
obj = self.get_object(request, object_id)
if request.method == "POST":
form_data = request.session.get("unsaved_data", None)
@@ -59,3 +91,41 @@ def confirm_save(self, request, object_id): # pragma: no cover
"form_data": request.session.get("unsaved_data"),
},
)
+
+ @button(permission=is_root)
+ def change_settings_schema(
+ self, request: HttpRequest
+ ) -> HttpResponse: # pragma: no cover
+ context = {
+ "opts": self.model._meta,
+ "site_header": site.site_header,
+ "title": "Change settings shema",
+ "trail_label": "Settings schema",
+ "has_view_permission": self.has_view_permission(request),
+ }
+
+ if request.method == "POST":
+ form = EditSchemaForm(request.POST)
+ if form.is_valid():
+ try:
+ SchemaManager.save(form.cleaned_data["schema"])
+ except ValidationError as e:
+ self.message_user(request, e.message, level=messages.ERROR)
+ else:
+ self.message_user(request, "Schema has been updated.")
+ return redirect(reverse("admin:api_config_changelist"))
+ else:
+ try:
+ form = EditSchemaForm(initial={"schema": SchemaManager.get_or_create()})
+ except ValidationError as e:
+ self.message_user(request, e.message, level=messages.ERROR)
+ return redirect(reverse("admin:api_config_changelist"))
+
+ return render(
+ request,
+ "admin/api/config/change_settings_schema.html",
+ {
+ "form": form,
+ **context,
+ },
+ )
diff --git a/src/hope_dedup_engine/apps/api/admin/deduplicationset.py b/src/hope_dedup_engine/apps/api/admin/deduplicationset.py
index 654b8ee5..98aeb49e 100644
--- a/src/hope_dedup_engine/apps/api/admin/deduplicationset.py
+++ b/src/hope_dedup_engine/apps/api/admin/deduplicationset.py
@@ -50,7 +50,9 @@ def has_add_permission(self, request):
return False
@button(permission=can_reprocess)
- def process(self, request: HttpRequest, pk: UUID) -> HttpResponseRedirect:
+ def process(
+ self, request: HttpRequest, pk: UUID
+ ) -> HttpResponseRedirect: # pragma: no cover
obj = self.get_object(request, pk)
start_processing(obj)
self.message_user(
diff --git a/src/hope_dedup_engine/apps/api/config_settings_schema.json b/src/hope_dedup_engine/apps/api/config_settings_schema.json
new file mode 100644
index 00000000..ac56a0fe
--- /dev/null
+++ b/src/hope_dedup_engine/apps/api/config_settings_schema.json
@@ -0,0 +1,72 @@
+{
+ "type": "object",
+ "properties": {
+ "detection": {
+ "type": "object",
+ "properties": {
+ "confidence": {
+ "type": "number",
+ "exclusiveMinimum": 0,
+ "maximum": 1,
+ "default": "constance.config.FACE_DETECTION_CONFIDENCE"
+ }
+ },
+ "default": {},
+ "required": [
+ "confidence"
+ ]
+ },
+ "recognition": {
+ "type": "object",
+ "properties": {
+ "num_jitters": {
+ "type": "integer",
+ "minimum": 1,
+ "default": "constance.config.FACE_ENCODINGS_NUM_JITTERS"
+ },
+ "model": {
+ "type": "string",
+ "enum": [
+ "small",
+ "large"
+ ],
+ "default": "constance.config.FACE_ENCODINGS_MODEL"
+ },
+ "preprocessors": {
+ "type": "array",
+ "items": {
+ "type": "string",
+ "enum": []
+ },
+ "uniquItems": true,
+ "default": []
+ }
+ },
+ "default": {},
+ "required": [
+ "num_jitters",
+ "model"
+ ]
+ },
+ "duplicates": {
+ "type": "object",
+ "properties": {
+ "tolerance": {
+ "type": "number",
+ "exclusiveMinimum": 0,
+ "maximum": 1,
+ "default": "constance.config.FACE_DISTANCE_THRESHOLD"
+ }
+ },
+ "default": {},
+ "required": [
+ "tolerance"
+ ]
+ }
+ },
+ "required": [
+ "detection",
+ "recognition",
+ "duplicates"
+ ]
+}
\ No newline at end of file
diff --git a/src/hope_dedup_engine/apps/api/forms.py b/src/hope_dedup_engine/apps/api/forms.py
new file mode 100644
index 00000000..75ad68de
--- /dev/null
+++ b/src/hope_dedup_engine/apps/api/forms.py
@@ -0,0 +1,7 @@
+from django import forms
+
+from django_svelte_jsoneditor.widgets import SvelteJSONEditorWidget
+
+
+class EditSchemaForm(forms.Form):
+ schema = forms.JSONField(widget=SvelteJSONEditorWidget())
diff --git a/src/hope_dedup_engine/apps/api/models/config.py b/src/hope_dedup_engine/apps/api/models/config.py
index 8d5e8120..b0618819 100644
--- a/src/hope_dedup_engine/apps/api/models/config.py
+++ b/src/hope_dedup_engine/apps/api/models/config.py
@@ -1,12 +1,8 @@
from django.core.exceptions import ValidationError
from django.db import models
-from jsonschema import ValidationError as JSONSchemaValidationError
-
-from hope_dedup_engine.apps.api.utils.config_schema import (
- DefaultValidatingValidator,
- settings_schema,
-)
+from hope_dedup_engine.apps.api.validators import DefaultValidatingValidator
+from src.hope_dedup_engine.apps.api.utils.shema_manager import SchemaManager
class Config(models.Model):
@@ -20,6 +16,7 @@ def __str__(self) -> str:
def clean(self) -> None:
try:
- DefaultValidatingValidator(settings_schema).validate(self.settings)
- except JSONSchemaValidationError as e:
+ schema = SchemaManager.get_or_create()
+ DefaultValidatingValidator(schema).validate(self.settings)
+ except Exception as e:
raise ValidationError({"settings": e.message})
diff --git a/src/hope_dedup_engine/apps/api/serializers.py b/src/hope_dedup_engine/apps/api/serializers.py
index 495145a0..c91207bd 100644
--- a/src/hope_dedup_engine/apps/api/serializers.py
+++ b/src/hope_dedup_engine/apps/api/serializers.py
@@ -11,7 +11,7 @@
IgnoredReferencePkPair,
Image,
)
-from hope_dedup_engine.apps.api.utils.config_schema import settings_schema
+from src.hope_dedup_engine.apps.api.utils.shema_manager import SchemaManager
class ConfigSerializer(serializers.ModelSerializer):
@@ -20,7 +20,7 @@ class Meta:
exclude = ("id",)
def validate_settings(self, value):
- validator = Draft202012Validator(settings_schema)
+ validator = Draft202012Validator(SchemaManager.get_or_create())
try:
validator.validate(value)
except JSONSchemaValidationError as e:
diff --git a/src/hope_dedup_engine/apps/api/utils/config_schema.py b/src/hope_dedup_engine/apps/api/utils/config_schema.py
deleted file mode 100644
index 49fb50c1..00000000
--- a/src/hope_dedup_engine/apps/api/utils/config_schema.py
+++ /dev/null
@@ -1,97 +0,0 @@
-from django.conf import settings
-
-from constance import config
-from jsonschema import Draft202012Validator, validators
-
-settings_schema: dict = {
- "type": "object",
- "properties": {
- "detection": {
- "type": "object",
- "properties": {
- "confidence": {
- "type": "number",
- "exclusiveMinimum": 0,
- "maximum": 1.0,
- "default": "constance.config.FACE_DETECTION_CONFIDENCE",
- },
- },
- "default": {},
- },
- "recognition": {
- "type": "object",
- "properties": {
- "num_jitters": {
- "type": "integer",
- "minimum": 1,
- "default": "constance.config.FACE_ENCODINGS_NUM_JITTERS",
- },
- "model": {
- "type": "string",
- "enum": tuple(
- ch[0]
- for ch in settings.CONSTANCE_ADDITIONAL_FIELDS.get(
- "face_encodings_model"
- )[1].get("choices")
- ),
- "default": "constance.config.FACE_ENCODINGS_MODEL",
- },
- "preprocessors": {
- type: "array",
- "items": {
- "type": "string",
- "enum": ["contrast"],
- },
- "uniqueItems": True,
- "default": [],
- },
- },
- "default": {},
- },
- "duplicates": {
- "type": "object",
- "properties": {
- "tolerance": {
- "type": "number",
- "exclusiveMinimum": 0,
- "maximum": 1.0,
- "default": "constance.config.FACE_DISTANCE_THRESHOLD",
- },
- },
- "default": {},
- },
- },
-}
-
-
-def extend_with_default(validator_class):
- validate_properties = validator_class.VALIDATORS["properties"]
-
- def set_defaults(validator, properties, instance, schema):
- for property, subschema in properties.items():
-
- if "default" in subschema:
- default_value = subschema["default"]
- if isinstance(default_value, str) and default_value.startswith(
- "constance.config."
- ):
- config_name = default_value.split(".")[-1]
- default_value = getattr(config, config_name, None)
-
- instance.setdefault(property, default_value)
-
- for error in validate_properties(
- validator,
- properties,
- instance,
- schema,
- ):
- yield error
-
- return validators.extend(
- validator_class,
- {"properties": set_defaults},
- )
-
-
-DefaultValidatingValidator = extend_with_default(Draft202012Validator)
diff --git a/src/hope_dedup_engine/apps/api/utils/shema_manager.py b/src/hope_dedup_engine/apps/api/utils/shema_manager.py
new file mode 100644
index 00000000..975fbfec
--- /dev/null
+++ b/src/hope_dedup_engine/apps/api/utils/shema_manager.py
@@ -0,0 +1,39 @@
+import json
+import logging
+from pathlib import Path
+
+from django.conf import settings
+from django.core.exceptions import ValidationError
+
+from jsonschema import Draft202012Validator, exceptions
+
+logger = logging.getLogger(__name__) # pragma: no cover
+
+
+class SchemaManager: # pragma: no cover
+ schema_path = Path(settings.CONFIG_SETTINGS_SCHEMA_FILE)
+
+ @classmethod
+ def get_or_create(cls) -> dict:
+ try:
+ schema = json.loads(cls.schema_path.read_text())
+ Draft202012Validator.check_schema(schema)
+ return schema
+ except FileNotFoundError:
+ logger.warning("Schema file not found.")
+ return {}
+ except (json.JSONDecodeError, exceptions.SchemaError) as e:
+ logger.error(f"Failed to load schema: {e}")
+ raise ValidationError("Failed to load the schema file.") from e
+
+ @classmethod
+ def save(cls, schema: dict) -> None:
+ try:
+ Draft202012Validator.check_schema(schema)
+ cls.schema_path.write_text(json.dumps(schema, indent=4))
+ logger.info(f"Schema saved to {cls.schema_path}")
+ except exceptions.SchemaError as e:
+ raise ValidationError("Invalid schema format.") from e
+ except IOError as e:
+ logger.error(f"Failed to save schema: {e}")
+ raise ValidationError("Failed to save the schema file.") from e
diff --git a/src/hope_dedup_engine/apps/api/validators.py b/src/hope_dedup_engine/apps/api/validators.py
new file mode 100644
index 00000000..c7373633
--- /dev/null
+++ b/src/hope_dedup_engine/apps/api/validators.py
@@ -0,0 +1,35 @@
+from constance import config
+from jsonschema import Draft202012Validator, validators
+
+
+def extend_with_default(validator_class):
+ validate_properties = validator_class.VALIDATORS["properties"]
+
+ def set_defaults(validator, properties, instance, schema):
+ for property, subschema in properties.items():
+
+ if "default" in subschema:
+ default_value = subschema["default"]
+ if isinstance(default_value, str) and default_value.startswith(
+ "constance.config."
+ ):
+ config_name = default_value.split(".")[-1]
+ default_value = getattr(config, config_name, None)
+
+ instance.setdefault(property, default_value)
+
+ for error in validate_properties(
+ validator,
+ properties,
+ instance,
+ schema,
+ ):
+ yield error
+
+ return validators.extend(
+ validator_class,
+ {"properties": set_defaults},
+ )
+
+
+DefaultValidatingValidator = extend_with_default(Draft202012Validator)
diff --git a/src/hope_dedup_engine/config/fragments/storages.py b/src/hope_dedup_engine/config/fragments/storages.py
index 76349948..42d76772 100644
--- a/src/hope_dedup_engine/config/fragments/storages.py
+++ b/src/hope_dedup_engine/config/fragments/storages.py
@@ -1,3 +1,4 @@
+from pathlib import Path
from typing import Final
DNN_FILES: Final[dict[str, dict[str, str]]] = {
@@ -16,3 +17,7 @@
},
},
}
+
+CONFIG_SETTINGS_SCHEMA_FILE: Final[str] = (
+ Path(__file__).resolve().parents[2] / "apps/api" / "config_settings_schema.json"
+)
diff --git a/src/hope_dedup_engine/config/settings.py b/src/hope_dedup_engine/config/settings.py
index ce1adf73..44ee73e4 100644
--- a/src/hope_dedup_engine/config/settings.py
+++ b/src/hope_dedup_engine/config/settings.py
@@ -44,6 +44,7 @@
"hope_dedup_engine.apps.faces",
"storages",
"smart_env",
+ "django_svelte_jsoneditor",
)
MIDDLEWARE = (
diff --git a/src/hope_dedup_engine/config/urls.py b/src/hope_dedup_engine/config/urls.py
index f3d5de4b..d930d232 100644
--- a/src/hope_dedup_engine/config/urls.py
+++ b/src/hope_dedup_engine/config/urls.py
@@ -20,3 +20,7 @@
if settings.DEBUG:
urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)
+
+admin.site.site_header = "HOPE Dedup Engine"
+admin.site.site_title = "HOPE Deduplication Admin"
+admin.site.index_title = "Welcome to the HOPE Deduplication Engine Admin"
diff --git a/src/hope_dedup_engine/web/templates/admin/api/config/change_list.html b/src/hope_dedup_engine/web/templates/admin/api/config/change_list.html
new file mode 100644
index 00000000..dc2e6403
--- /dev/null
+++ b/src/hope_dedup_engine/web/templates/admin/api/config/change_list.html
@@ -0,0 +1,8 @@
+{% extends "admin/change_list.html" %}
+
+{% block object-tools-items %}
+ {{ block.super }}
+ {% include "admin_extra_buttons/includes/change_list_buttons.html" %}
+{% endblock %}
+
+{% block pagination %}{% endblock %}
diff --git a/src/hope_dedup_engine/web/templates/admin/api/config/change_settings_schema.html b/src/hope_dedup_engine/web/templates/admin/api/config/change_settings_schema.html
new file mode 100644
index 00000000..408364b5
--- /dev/null
+++ b/src/hope_dedup_engine/web/templates/admin/api/config/change_settings_schema.html
@@ -0,0 +1,21 @@
+{% extends "admin/base_site.html" %}
+{% load i18n static %}
+
+{% if not is_popup %}
+{% block breadcrumbs %}
+
+{% endblock %}
+{% endif %}
+
+{% block content %}
+
+{% endblock %}
diff --git a/tests/admin/test_admin_smoke.py b/tests/admin/test_admin_smoke.py
index 1c6fc2e8..b65a58db 100644
--- a/tests/admin/test_admin_smoke.py
+++ b/tests/admin/test_admin_smoke.py
@@ -183,7 +183,6 @@ def test_admin_delete(app, modeladmin, record, monkeypatch):
pytest.skip("No 'delete' permission")
-@pytest.mark.skip_buttons("api.DeduplicationSetAdmin:process")
def test_admin_buttons(app, modeladmin, button_handler, record, monkeypatch):
from admin_extra_buttons.handlers import LinkHandler
@@ -198,5 +197,8 @@ def test_admin_buttons(app, modeladmin, button_handler, record, monkeypatch):
else:
url = reverse(f"admin:{button_handler.url_name}", args=[record.pk])
- res = app.get(url)
- assert res.status_code in [200, 302]
+ res = app.get(url, expect_errors=True)
+ if button_handler.permission:
+ assert res.status_code == 403
+ else:
+ assert res.status_code in [200, 302]
diff --git a/tests/extras/demoapp/dnn_files/config_settings_schema.json b/tests/extras/demoapp/dnn_files/config_settings_schema.json
new file mode 100644
index 00000000..3822e8bb
--- /dev/null
+++ b/tests/extras/demoapp/dnn_files/config_settings_schema.json
@@ -0,0 +1,54 @@
+{
+ "type": "object",
+ "properties": {
+ "detection": {
+ "type": "object",
+ "properties": {
+ "confidence": {
+ "type": "number",
+ "exclusiveMinimum": 0,
+ "maximum": 1.0,
+ "default": "constance.config.FACE_DETECTION_CONFIDENCE"
+ }
+ },
+ "default": {}
+ },
+ "recognition": {
+ "type": "object",
+ "properties": {
+ "num_jitters": {
+ "type": "integer",
+ "minimum": 1,
+ "default": "constance.config.FACE_ENCODINGS_NUM_JITTERS"
+ },
+ "model": {
+ "type": "string",
+ "enum": ["small", "large"],
+ "default": "constance.config.FACE_ENCODINGS_MODEL"
+ },
+ "preprocessors": {
+ "type": "array",
+ "items": {
+ "type": "string",
+ "enum": ["contrast"]
+ },
+ "uniqueItems": true,
+ "default": []
+ }
+ },
+ "default": {}
+ },
+ "duplicates": {
+ "type": "object",
+ "properties": {
+ "tolerance": {
+ "type": "number",
+ "exclusiveMinimum": 0,
+ "maximum": 1.0,
+ "default": "constance.config.FACE_DISTANCE_THRESHOLD"
+ }
+ },
+ "default": {}
+ }
+ }
+}