[RFC] Arches model QuerySets #11595

arches/app/datatypes/base.py

@@ -1,15 +1,20 @@
-import json, urllib
+import json
+import logging
+import urllib
+
 from django.urls import reverse
+from django.utils.translation import gettext as _
+
 from arches.app.models import models
-from arches.app.models.system_settings import settings
 from arches.app.search.elasticsearch_dsl_builder import Dsl, Bool, Terms, Exists, Nested
-from django.utils.translation import gettext as _
-import logging
 
 logger = logging.getLogger(__name__)
 
 
 class BaseDataType(object):
+    rest_framework_model_field = None
+    """Django model field if the datatype were to be a real table column."""
+
     def __init__(self, model=None):
         self.datatype_model = model
         self.datatype_name = model.datatype if model else None
@@ -336,7 +341,7 @@ def get_default_language_value_from_localized_node(self, tile, nodeid):
         """
         return tile.data[str(nodeid)]
 
-    def post_tile_save(self, tile, nodeid, request):
+    def post_tile_save(self, tile, nodeid, request=None):
         """
         Called after the tile is saved to the database
 
@@ -532,3 +537,13 @@ def validate_node(self, node):
         a GraphValidationError
         """
         pass
+
+    def get_base_orm_lookup(self, node):
+        """This expression gets the tile data for a specific node. It can be
+        overridden to extract something more specific, especially where the
+        node value is JSON and only certain k/v pairs are useful to query.
+        """
+        return f"data__{node.pk}"
+
+    def to_python(self, tile_val):
+        return tile_val

arches/app/datatypes/concept_types.py

@@ -2,11 +2,14 @@
 import uuid
 import csv
 import logging
+
+from django.contrib.postgres.fields import ArrayField
 from django.core.exceptions import ObjectDoesNotExist
+from django.db.models import fields
 from django.utils.translation import gettext as _
-from arches.app.models import models
-from arches.app.models import concept
 from django.core.cache import cache
+
+from arches.app.models import models
 from arches.app.models.system_settings import settings
 from arches.app.datatypes.base import BaseDataType
 from arches.app.datatypes.datatypes import DataTypeFactory, get_value_from_jsonld
@@ -32,7 +35,6 @@
 from rdflib.namespace import RDF, RDFS, XSD, DC, DCTERMS, SKOS
 from arches.app.models.concept import ConceptValue
 from arches.app.models.concept import Concept
-from io import StringIO
 
 archesproject = Namespace(settings.ARCHES_NAMESPACE_FOR_DATA_EXPORT)
 cidoc_nm = Namespace("http://www.cidoc-crm.org/cidoc-crm/")
@@ -41,6 +43,8 @@
 
 
 class BaseConceptDataType(BaseDataType):
+    rest_framework_model_field = fields.UUIDField(null=True)
+
     def __init__(self, model=None):
         super(BaseConceptDataType, self).__init__(model=model)
         self.value_lookup = {}
@@ -253,6 +257,8 @@ def validate(
         return errors
 
     def transform_value_for_tile(self, value, **kwargs):
+        if isinstance(value, uuid.UUID):
+            return str(value)
         try:
             stripped = value.strip()
             uuid.UUID(stripped)
@@ -409,6 +415,8 @@ def ignore_keys(self):
 
 
 class ConceptListDataType(BaseConceptDataType):
+    rest_framework_model_field = ArrayField(base_field=fields.UUIDField(), null=True)
+
     def validate(
         self,
         value,
@@ -425,13 +433,20 @@ def validate(
         if value is not None:
             validate_concept = DataTypeFactory().get_instance("concept")
             for v in value:
-                val = v.strip()
+                if isinstance(v, uuid.UUID):
+                    val = str(v)
+                else:
+                    val = v.strip()
                 errors += validate_concept.validate(val, row_number)
         return errors
 
     def transform_value_for_tile(self, value, **kwargs):
         ret = []
-        for val in csv.reader([value], delimiter=",", quotechar='"'):
+        if not isinstance(value, list):
+            value = [value]
+        if all(isinstance(inner, uuid.UUID) for inner in value):
+            return [str(inner) for inner in value]
+        for val in csv.reader(value, delimiter=",", quotechar='"'):
             lines = [line for line in val]
             for v in lines:
                 try:

arches/app/datatypes/core/non_localized_string.py

@@ -1,10 +1,11 @@
+from django.conf import settings
+from django.db.models import fields
 from django.utils.translation import gettext as _
 from rdflib import URIRef, Literal, ConjunctiveGraph as Graph
 from rdflib.namespace import RDF
 
 from arches.app.datatypes.base import BaseDataType
 from arches.app.datatypes.core.util import get_value_from_jsonld
-from django.conf import settings
 from arches.app.search.elasticsearch_dsl_builder import (
     Bool,
     Exists,
@@ -18,6 +19,8 @@
 
 
 class NonLocalizedStringDataType(BaseDataType):
+    rest_framework_model_field = fields.CharField(null=True)
+
     def validate(
         self,
         value,

arches/app/datatypes/datatypes.py

@@ -1,4 +1,5 @@
 import copy
+import itertools
 import uuid
 import json
 import decimal
@@ -15,9 +16,6 @@
 from datetime import datetime
 from mimetypes import MimeTypes
 
-from django.core.files.images import get_image_dimensions
-from django.db.models import fields
-
 from arches.app.const import ExtensionType
 from arches.app.datatypes.base import BaseDataType
 from arches.app.models import models
@@ -47,18 +45,18 @@
 from arches.app.search.search_engine_factory import SearchEngineInstance as se
 from arches.app.search.search_term import SearchTerm
 from arches.app.search.mappings import RESOURCES_INDEX
+
+from django.contrib.postgres.fields import ArrayField
 from django.core.cache import cache
 from django.core.files import File
-from django.core.files.base import ContentFile
-from django.core.files.storage import FileSystemStorage, default_storage
+from django.core.files.images import get_image_dimensions
+from django.core.files.storage import default_storage
 from django.core.exceptions import ObjectDoesNotExist
-from django.core.exceptions import ValidationError
-from django.db import connection, transaction
+from django.db import connection
+from django.db.models import fields
+from django.db.models.fields.json import JSONField
 from django.utils.translation import get_language, gettext as _
 
-from elasticsearch import Elasticsearch
-from elasticsearch.exceptions import NotFoundError
-
 # One benefit of shifting to python3.x would be to use
 # importlib.util.LazyLoader to load rdflib (and other lesser
 # used but memory soaking libs)
@@ -118,6 +116,8 @@ def get_instance(self, datatype):
 
 
 class StringDataType(BaseDataType):
+    rest_framework_model_field = JSONField(null=True)
+
     def validate(
         self,
         value,
@@ -458,6 +458,8 @@ def pre_structure_tile_data(self, tile, nodeid, **kwargs):
 
 
 class NumberDataType(BaseDataType):
+    rest_framework_model_field = fields.FloatField(null=True)
+
     def validate(
         self,
         value,
@@ -574,6 +576,8 @@ def default_es_mapping(self):
 
 
 class BooleanDataType(BaseDataType):
+    rest_framework_model_field = fields.BooleanField(null=True)
+
     def validate(
         self,
         value,
@@ -675,6 +679,8 @@ def default_es_mapping(self):
 
 
 class DateDataType(BaseDataType):
+    rest_framework_model_field = fields.DateField(null=True)
+
     def validate(
         self,
         value,
@@ -886,13 +892,17 @@ def get_display_value(self, tile, node, **kwargs):
 
 
 class EDTFDataType(BaseDataType):
+    rest_framework_model_field = fields.CharField(null=True)
+
     def transform_value_for_tile(self, value, **kwargs):
         transformed_value = ExtendedDateFormat(value)
         if transformed_value.edtf is None:
             return value
         return str(transformed_value.edtf)
 
     def pre_tile_save(self, tile, nodeid):
+        # TODO: This is likely to be duplicative once we clean this up:
+        # https://github.com/archesproject/arches/issues/10851#issuecomment-2427305853
         tile.data[nodeid] = self.transform_value_for_tile(tile.data[nodeid])
 
     def validate(
@@ -1057,6 +1067,8 @@ def default_es_mapping(self):
 
 
 class FileListDataType(BaseDataType):
+    rest_framework_model_field = ArrayField(base_field=fields.CharField(), null=True)
+
     def __init__(self, model=None):
         super(FileListDataType, self).__init__(model=model)
         self.node_lookup = {}
@@ -1258,7 +1270,7 @@ def to_json(self, tile, node):
         if data:
             return self.compile_json(tile, node, file_details=data[str(node.nodeid)])
 
-    def post_tile_save(self, tile, nodeid, request):
+    def post_tile_save(self, tile, nodeid, request=None):
         if request is not None:
             # this does not get called when saving data from the mobile app
             previously_saved_tile = models.TileModel.objects.filter(pk=tile.tileid)
@@ -2013,6 +2025,8 @@ class ResourceInstanceDataType(BaseDataType):
 
     """
 
+    rest_framework_model_field = fields.UUIDField(null=True)
+
     def validate(
         self,
         value,
@@ -2060,14 +2074,14 @@ def validate(
                                     raise ObjectDoesNotExist()
                         except ObjectDoesNotExist:
                             message = _(
-                                "The related resource with id '{0}' is not in the system.".format(
+                                "The related resource with id '{0}' is not in the system".format(
                                     resourceid
                                 )
                             )
                             errors.append({"type": "ERROR", "message": message})
                 except (ValueError, TypeError):
                     message = _(
-                        "The related resource with id '{0}' is not a valid uuid.".format(
+                        "The related resource with id '{0}' is not a valid uuid".format(
                             str(value)
                         )
                     )
@@ -2090,7 +2104,7 @@ def pre_tile_save(self, tile, nodeid):
             for relationship in relationships:
                 relationship["resourceXresourceId"] = str(uuid.uuid4())
 
-    def post_tile_save(self, tile, nodeid, request):
+    def post_tile_save(self, tile, nodeid, request=None):
         ret = False
         sql = """
             SELECT * FROM __arches_create_resource_x_resource_relationships('%s') as t;
@@ -2217,6 +2231,14 @@ def get_search_terms(self, nodevalue, nodeid=None):
         return terms
 
     def transform_value_for_tile(self, value, **kwargs):
+        def from_id_string(uuid_string):
+            nonlocal kwargs
+            return {
+                "resourceId": uuid_string,
+                "inverseOntology": kwargs.get("inverseOntology", ""),
+                "inverseOntologyProperty": kwargs.get("inverseOntologyProperty", ""),
+            }
+
         try:
             return json.loads(value)
         except ValueError:
@@ -2228,7 +2250,16 @@ def transform_value_for_tile(self, value, **kwargs):
         except TypeError:
             # data should come in as json but python list is accepted as well
             if isinstance(value, list):
-                return value
+                if all(isinstance(inner, models.ResourceInstance) for inner in value):
+                    return [from_id_string(str(instance.pk)) for instance in value]
+                elif all(isinstance(inner, uuid.UUID) for inner in value):
+                    return [from_id_string(str(uid)) for uid in value]
+                elif all(isinstance(inner, str) for inner in value):
+                    return [from_id_string(uid) for uid in value]
+                else:
+                    return value
+            if isinstance(value, models.ResourceInstance):
+                return [from_id_string(str(value.pk))]
 
     def transform_export_values(self, value, *args, **kwargs):
         return json.dumps(value)
@@ -2354,8 +2385,23 @@ def default_es_mapping(self):
         }
         return mapping
 
+    def _get_base_orm_lookup(self, node):
+        """Filter down to the resourceId."""
+        return f"data__{node.pk}__0__resourceId"
+
+    def values_match(self, value1, value2):
+        if not isinstance(value1, list) or not isinstance(value2, list):
+            return value1 == value2
+        copy1 = [{**inner_val} for inner_val in value1]
+        copy2 = [{**inner_val} for inner_val in value2]
+        for inner_val in itertools.chain(copy1, copy2):
+            inner_val.pop("resourceXresourceId", None)
+        return copy1 == copy2
+
 
 class ResourceInstanceListDataType(ResourceInstanceDataType):
+    rest_framework_model_field = ArrayField(base_field=fields.UUIDField(), null=True)
+
     def to_json(self, tile, node):
         from arches.app.models.resource import (
             Resource,
@@ -2383,6 +2429,18 @@ def to_json(self, tile, node):
     def collects_multiple_values(self):
         return True
 
+    def _get_base_orm_lookup(self, node):
+        """Undo the override in ResourceInstanceDataType. TODO: write a better lookup.
+        Currently the unpacking into UUID[] is done in to_python(), but this isn't
+        useful for querying."""
+        return f"data__{node.pk}"
+
+    def to_python(self, tile_val):
+        if tile_val is None:
+            return tile_val
+        resource_ids = [inner["resourceId"] if inner else None for inner in tile_val]
+        return resource_ids
+
 
 class NodeValueDataType(BaseDataType):
     def validate(

arches/app/datatypes/url.py

@@ -27,6 +27,7 @@
 from rdflib import ConjunctiveGraph as Graph
 from rdflib import URIRef, Literal, Namespace
 from rdflib.namespace import RDF, RDFS, XSD, DC, DCTERMS
+from django.db.models import fields
 from django.utils.translation import gettext as _
 
 archesproject = Namespace(settings.ARCHES_NAMESPACE_FOR_DATA_EXPORT)
@@ -70,6 +71,8 @@ class URLDataType(BaseDataType):
     URL Datatype to store an optionally labelled hyperlink to a (typically) external resource
     """
 
+    rest_framework_model_field = fields.URLField(null=True)
+
     URL_REGEX = re.compile(
         r"https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)"
     )

arches/app/models/functions.py

@@ -0,0 +1,7 @@
+from django.db import models
+
+
+class UUID4(models.Func):
+    function = "uuid_generate_v4"
+    arity = 0
+    output_field = models.UUIDField()