Feature/ogc 508 replace elastic search by postgres v2

src/onegov/activity/models/attendee.py

@@ -65,7 +65,10 @@ class Attendee(Base, TimestampMixin, ORMSearchable):
         'name': {'type': 'text'},
         'notes': {'type': 'localized'}
     }
-    es_public = False
+
+    @hybrid_property
+    def es_public(self) -> bool:
+        return False
 
     @property
     def es_suggestion(self) -> str:

src/onegov/agency/models/agency.py

@@ -1,3 +1,8 @@
+from sqlalchemy import and_
+from sqlalchemy.orm import object_session
+from sqlalchemy.orm import relationship
+from sqlalchemy.ext.hybrid import hybrid_property
+
 from onegov.agency.models.membership import ExtendedAgencyMembership
 from onegov.agency.utils import get_html_paragraph_with_line_breaks
 from onegov.core.crypto import random_token
@@ -11,9 +16,6 @@
 from onegov.org.models.extensions import PublicationExtension
 from onegov.people import Agency
 from onegov.user import RoleMapping
-from sqlalchemy.orm import object_session
-from sqlalchemy.orm import relationship
-
 
 from typing import Any
 from typing import IO
@@ -24,6 +26,7 @@
     from markupsafe import Markup
     from onegov.agency.request import AgencyRequest
     from onegov.core.types import AppenderQuery
+    from sqlalchemy.sql import ClauseElement
     from uuid import UUID
 
 
@@ -40,10 +43,17 @@ class ExtendedAgency(Agency, AccessExtension, PublicationExtension):
 
     es_type_name = 'extended_agency'
 
-    @property
-    def es_public(self) -> bool:  # type:ignore[override]
+    @hybrid_property
+    def es_public(self) -> bool:
         return self.access == 'public' and self.published
 
+    @es_public.expression  # type:ignore[no-redef]
+    def es_public(cls) -> 'ClauseElement':
+        return and_(
+            cls.access == 'public',
+            cls.published == True
+        )
+
     #: Defines which fields of a membership and person should be exported to
     #: the PDF. The fields are expected to contain two parts seperated by a
     #: point. The first part is either `membership` or `person`, the second

src/onegov/agency/models/membership.py

@@ -1,15 +1,18 @@
+from sqlalchemy import case, select
+from sqlalchemy.ext.hybrid import hybrid_property
+
 from onegov.core.orm.mixins import dict_property
 from onegov.core.orm.mixins import meta_property
 from onegov.org.models.extensions import AccessExtension
 from onegov.org.models.extensions import PublicationExtension
 from onegov.people import AgencyMembership
 
-
 from typing import TYPE_CHECKING
 if TYPE_CHECKING:
     from onegov.agency.models import ExtendedAgency
     from onegov.agency.models import ExtendedPerson
     from sqlalchemy.orm import relationship
+    from sqlalchemy.sql import ClauseElement
 
 
 class ExtendedAgencyMembership(AgencyMembership, AccessExtension,
@@ -20,8 +23,8 @@ class ExtendedAgencyMembership(AgencyMembership, AccessExtension,
 
     es_type_name = 'extended_membership'
 
-    @property
-    def es_public(self) -> bool:  # type:ignore[override]
+    @hybrid_property
+    def es_public(self) -> bool:
         if self.agency:
             if self.agency.meta.get('access', 'public') != 'public':
                 return False
@@ -36,6 +39,36 @@ def es_public(self) -> bool:  # type:ignore[override]
 
         return self.access == 'public'
 
+    @es_public.expression  # type:ignore[no-redef]
+    def es_public(cls) -> 'ClauseElement':
+        from onegov.agency.models import ExtendedAgency, ExtendedPerson
+
+        person_meta = select([ExtendedPerson.meta]).where(
+            ExtendedPerson.id == cls.person_id
+        ).as_scalar()
+
+        person_published = select([ExtendedPerson.published]).where(
+            ExtendedPerson.id == cls.person_id
+        ).as_scalar()
+
+        agency_meta = select([ExtendedAgency.meta]).where(
+            ExtendedAgency.id == cls.agency_id
+        ).as_scalar()
+
+        agency_published = select([ExtendedAgency.published]).where(
+            ExtendedAgency.id == cls.agency_id
+        ).as_scalar()
+
+        return case(
+            [
+                (person_meta['access'] != 'public', False),
+                (person_published != True, False),
+                (agency_meta['access'] != 'public', False),
+                (agency_published != True, False),
+            ],
+            else_=cls.meta['access'] == 'public'
+        )
+
     # Todo: It is very unclear how this should be used. In the PDF rendering,
     # it is placed a middle column with 0.5 cm after the title.
     # On the agency, it is placed after the membership title, so not a prefix

src/onegov/agency/models/person.py

@@ -1,9 +1,12 @@
+from sqlalchemy import func, select, and_
+from sqlalchemy.orm import object_session
+from sqlalchemy.ext.hybrid import hybrid_property
+
 from onegov.agency.utils import get_html_paragraph_with_line_breaks
 from onegov.org.models import Organisation
 from onegov.org.models.extensions import AccessExtension
 from onegov.org.models.extensions import PublicationExtension
 from onegov.people import Person
-from sqlalchemy.orm import object_session
 
 
 from typing import TYPE_CHECKING
@@ -13,6 +16,7 @@
     from onegov.agency.request import AgencyRequest
     from onegov.core.types import AppenderQuery
     from sqlalchemy.orm import relationship
+    from sqlalchemy.sql import ClauseElement
 
 
 class ExtendedPerson(Person, AccessExtension, PublicationExtension):
@@ -22,10 +26,17 @@
 
     es_type_name = 'extended_person'
 
-    @property
-    def es_public(self) -> bool:  # type:ignore[override]
+    @hybrid_property
+    def es_public(self) -> bool:
         return self.access == 'public' and self.published
 
+    @es_public.expression  # type:ignore[no-redef]
+    def es_public(cls) -> 'ClauseElement':
+        return and_(
+            cls.access == 'public',
+            cls.published == True
+        )
+
     es_properties = {
         'title': {'type': 'text'},
         'function': {'type': 'localized'},
@@ -50,14 +61,28 @@
             AppenderQuery[ExtendedAgencyMembership]
         ]
 
-    @property
+    @hybrid_property
     def phone_internal(self) -> str:
         org = object_session(self).query(Organisation).one()
         number = getattr(self, org.agency_phone_internal_field)
         digits = org.agency_phone_internal_digits
         return number.replace(' ', '')[-digits:] if number and digits else ''
 
-    @property
+    @phone_internal.expression  # type:ignore[no-redef]
+    def phone_internal(cls) -> 'ClauseElement':
+        org_subquery = (
+            select([Organisation.agency_phone_internal_field,
+                    Organisation.agency_phone_internal_digits])
+            .limit(1)
+            .scalar_subquery()
+        )
+        return func.substr(
+            func.replace(getattr(
+                cls, org_subquery.c.agency_phone_internal_field), ' ', ''),
+            -org_subquery.c.agency_phone_internal_field_digits
+        ).label('phone_internal')
+
+    @hybrid_property
     def phone_es(self) -> list[str]:
         result = [self.phone_internal]
         for number in (self.phone, self.phone_direct):

src/onegov/agency/views/search.py

@@ -1,9 +1,8 @@
 from onegov.agency import AgencyApp
 from onegov.agency.layout import AgencySearchLayout
 from onegov.core.security import Public
-from onegov.org.models import Search
-from onegov.org.views.search import search as search_view
-
+from onegov.org.models import Search, SearchPostgres
+from onegov.org.views.search import search as search_view, search_postgres
 
 from typing import TYPE_CHECKING
 if TYPE_CHECKING:
@@ -23,3 +22,15 @@ def search(
     if isinstance(data, dict):
         data['layout'] = AgencySearchLayout(self, request)
     return data
+
+
+@AgencyApp.html(model=SearchPostgres, template='search_postgres.pt',
+                permission=Public)
+def agency_search_postgres(
+    self: SearchPostgres['Base'],
+    request: 'AgencyRequest'
+) -> 'RenderData | Response':
+    data = search_postgres(self, request)
+    if isinstance(data, dict):
+        data['layout'] = AgencySearchLayout(self, request)
+    return data

src/onegov/core/elements.py

@@ -13,6 +13,7 @@
 This module should eventually replace the elements.py module.
 
 """
+from sqlalchemy.ext.hybrid import hybrid_property
 
 from onegov.core.templates import render_macro
 
@@ -137,7 +138,7 @@ class AccessMixin:
 
     __slots__ = ()
 
-    @property
+    @hybrid_property
     def access(self) -> str:
         """ Wraps model.access, ensuring it is always available, even if the
         model does not use it.

src/onegov/directory/models/directory.py

@@ -2,6 +2,9 @@
 
 from email_validator import validate_email
 from enum import Enum
+
+from sqlalchemy.ext.hybrid import hybrid_property
+
 from onegov.core.cache import instance_lru_cache
 from onegov.core.cache import lru_cache
 from onegov.core.crypto import random_token
@@ -90,7 +93,7 @@ def directory_entry(self) -> 'DirectoryEntry | None':
         entries = self.linked_directory_entries
         return entries[0] if entries else None
 
-    @property
+    @hybrid_property
     def access(self) -> str:
         # we don't want these files to show up in search engines
         return 'secret' if self.published else 'private'
@@ -110,7 +113,7 @@ class Directory(Base, ContentMixin, TimestampMixin,
         'lead': {'type': 'localized'}
     }
 
-    @property
+    @hybrid_property
     def es_public(self) -> bool:
         return False  # to be overridden downstream
 

src/onegov/directory/models/directory_entry.py

@@ -1,3 +1,5 @@
+from sqlalchemy.ext.hybrid import hybrid_property
+
 from onegov.core.orm import Base
 from onegov.core.orm.mixins import ContentMixin
 from onegov.core.orm.mixins import TimestampMixin
@@ -6,7 +8,7 @@
 from onegov.file import AssociatedFiles
 from onegov.gis import CoordinatesMixin
 from onegov.search import SearchableContent
-from sqlalchemy import Column
+from sqlalchemy import Column, func, cast, ARRAY, String
 from sqlalchemy import ForeignKey
 from sqlalchemy import Index
 from sqlalchemy import Text
@@ -30,18 +32,18 @@ class DirectoryEntry(Base, ContentMixin, CoordinatesMixin, TimestampMixin,
     __tablename__ = 'directory_entries'
 
     es_properties = {
-        'keywords': {'type': 'keyword'},
         'title': {'type': 'localized'},
         'lead': {'type': 'localized'},
-        'directory_id': {'type': 'keyword'},
+        'keywords': {'type': 'keyword'},
+        # 'directory_id': {'type': 'keyword'},
 
         # since the searchable text might include html, we remove it
         # even if there's no html -> possibly decreasing the search
         # quality a bit
         'text': {'type': 'localized_html'}
     }
 
-    @property
+    @hybrid_property
     def es_public(self) -> bool:
         return False  # to be overridden downstream
 
@@ -113,17 +115,26 @@ def external_link_visible(self) -> bool | None:
     def directory_name(self) -> str:
         return self.directory.name
 
-    @property
+    @hybrid_property
     def keywords(self) -> set[str]:
         return set(self._keywords.keys()) if self._keywords else set()
 
     # FIXME: asymmetric properties are not supported by mypy, switch to
     #        a custom descriptor, if desired.
-    @keywords.setter
+    @keywords.setter  # type:ignore[no-redef]
     def keywords(self, value: 'Collection[str] | None') -> None:
         self._keywords = dict.fromkeys(value, '') if value else None
 
-    @property
+    @keywords.expression  # type:ignore[no-redef]
+    def keywords(cls):
+        return func.array_to_string(
+            func.array_agg(
+                cast(func.jsonb_each_text(cls._keywords).keys(), ARRAY(String))
+            ),
+            ' '
+        )
+
+    @hybrid_property
     def text(self) -> str:
         return self.directory.configuration.extract_searchable(self.values)
 

src/onegov/event/models/event.py

@@ -1,12 +1,12 @@
 import warnings
 
 from datetime import datetime
-
 from dateutil import rrule
 from dateutil.rrule import rrulestr
 from icalendar import Calendar as vCalendar
 from icalendar import Event as vEvent
 from icalendar import vRecur
+from sqlalchemy.ext.hybrid import hybrid_property
 
 from onegov.core.orm import Base
 from onegov.core.orm.abstract import associated
@@ -194,7 +194,7 @@ def set_blob(
         'filter_keywords': {'type': 'keyword'}
     }
 
-    @property
+    @hybrid_property
     def es_public(self) -> bool:
         return self.state == 'published'