Add MS SQL Server as a DBMS option.

Some institutions may prefer. (Not been thoroughly road tested.)
- Mostly accounts for SQL differences and sqlalchemy:
	- exists not allowed in columns clause of select
	- boolean as BIT (is True, casting etc.)
	- tuple comparison
	- group by must include all returned columns
	- don't assume case sensitivity (depends on collation setup)
	- can't use Text without a length if setting unique, conversely
	  Text() in postgresql doesn't accept a length, use String
	- CheckConstraint passes string verbatim and therefore has
	  "limited database independent behaviour".  Switch to
	  enforcing at app level
- some threading issues during tests not seen in other DBMSs
- Test MultiprocCounter against provided database unless using an
  in-memory sqlite db (in which case switch to a temp file)

+ GenericExporterTests - fix occasional fail on datetime type tests
  i.e. when test runs over the second (seems more common using my
  current docker MSSQL setup)
+ some bits and pieces

NOTE:
- end user may still need to install a driver:
  osx - HOMEBREW_ACCEPT_EULA=Y brew install msodbcsql17 mssql-tools
  windows - download and install VC_redist.x64.exe
- pyodbc mingw windows awaiting mkleehammer/pyodbc#1169

.appveyor.yml

@@ -17,6 +17,7 @@ for:
 
   install:
     - brew install pygobject3 gtk+3 adwaita-icon-theme openjdk
+    - brew install unixodbc
 
   build_script:
     - . ~/venv3.10/bin/activate
@@ -25,6 +26,7 @@ for:
     - scripts/get_fop.sh
     - pip install --upgrade pip
     - pip install psycopg2-binary
+    - pip install pyodbc
     - pip install .
     - pip install pyinstaller
     - pyinstaller --clean --noconfirm scripts/ghini_fop.spec
@@ -55,6 +57,7 @@ for:
     - bash -lc "python --version"
     - bash -lc "python -m pip install --upgrade pip"
     - bash -lc "SETUPTOOLS_USE_DISTUTILS=stdlib pip install pyproj==3.3.1"
+    # - bash -lc "SETUPTOOLS_USE_DISTUTILS=stdlib pip install pyodbc"
 
   build_script:
     - bash -lc "pip install ."

bauble/btypes.py

@@ -230,11 +230,24 @@ def coerce_compared_value(self, op, value):
 
 
 class Boolean(types.TypeDecorator):
-    """A Boolean type that allows True/False as strings."""
+    """A Boolean type that allows True/False as strings.
+
+    For compatibility with MSSQL converts is_() to = and is_not() to !="""
     impl = types.Boolean
 
     cache_ok = True
 
+    class comparator_factory(types.Boolean.Comparator):
+        # pylint: disable=invalid-name
+
+        def is_(self, other):
+            """override is_"""
+            return self.op("=")(other)
+
+        def is_not(self, other):
+            """override is_not"""
+            return self.op("!=")(other)
+
     def process_bind_param(self, value, dialect):
         if not isinstance(value, str):
             return value

bauble/connmgr.py

@@ -52,7 +52,9 @@ def is_package_name(name):
         return False
 
 
-DBS = [('sqlite3', 'SQLite'), ('psycopg2', 'PostgreSQL')]
+DBS = [('sqlite3', 'SQLite'),
+       ('psycopg2', 'PostgreSQL'),
+       ('pyodbc', 'MSSQL')]
 #     ('mysql', 'MySQL'),
 #     ('pyodbc', 'MS SQL Server'),
 #     ('cx_Oracle', 'Oracle'),

bauble/plugins/garden/accession.py

@@ -43,7 +43,8 @@
                         Integer,
                         UnicodeText,
                         func,
-                        exists)
+                        exists,
+                        literal)
 from sqlalchemy.orm import relationship, validates, backref
 from sqlalchemy.orm.session import object_session
 from sqlalchemy.exc import DBAPIError
@@ -872,9 +873,9 @@ def top_level_count(self):
     def has_children(self):
         cls = self.__class__.plants.prop.mapper.class_
         session = object_session(self)
-        return session.query(
-            exists().where(cls.accession_id == self.id)
-        ).scalar()
+        return bool(session.query(literal(True))
+                    .filter(exists().where(cls.accession_id == self.id))
+                    .scalar())
 
     def count_children(self):
         cls = self.__class__.plants.prop.mapper.class_

bauble/plugins/garden/location.py

@@ -28,7 +28,7 @@
 
 from gi.repository import Gtk
 
-from sqlalchemy import Column, Unicode, UnicodeText
+from sqlalchemy import Column, Unicode, UnicodeText, literal
 from sqlalchemy.orm import relationship, backref, validates, deferred
 from sqlalchemy.orm.session import object_session
 from sqlalchemy.exc import DBAPIError
@@ -228,9 +228,9 @@ def has_children(self):
         cls = self.__class__.plants.prop.mapper.class_
         from sqlalchemy import exists
         session = object_session(self)
-        return session.query(
-            exists().where(cls.location_id == self.id)
-        ).scalar()
+        return bool(session.query(literal(True))
+                    .filter(exists().where(cls.location_id == self.id))
+                    .scalar())
 
     def count_children(self):
         cls = self.__class__.plants.prop.mapper.class_

bauble/plugins/garden/plant.py

@@ -304,10 +304,24 @@ def search(self, text, session): \
                 acc_code, plant_code = value.rsplit(delimiter, 1)
                 vals.append((acc_code, plant_code))
             logger.debug('"in" PlantSearch vals: %s', vals)
-            query = (session.query(Plant)
-                     .join(Accession)
-                     .filter(tuple_(Accession.code, Plant.code)
-                     .in_(vals)))
+            if db.engine.name == 'mssql':
+                from sqlalchemy import String
+                from sqlalchemy.sql import exists, values, column
+                sql_vals = values(
+                    column('acc_code', String),
+                    column('plt_code', String)
+                ).data(vals).alias('val')
+                query = (session.query(Plant)
+                         .join(Accession)
+                         .filter(exists()
+                                 .where(Accession.code == sql_vals.c.acc_code,
+                                        Plant.code == sql_vals.c.plt_code)))
+            else:
+                # sqlite, postgresql
+                query = (session.query(Plant)
+                         .join(Accession)
+                         .filter(tuple_(Accession.code, Plant.code)
+                         .in_(vals)))
 
         if prefs.prefs.get(prefs.exclude_inactive_pref):
             query = query.filter(Plant.active.is_(True))
@@ -732,8 +746,8 @@ def active(self):
     @active.expression
     def active(cls):
         # pylint: disable=no-self-argument
-        from sqlalchemy.sql.expression import case, cast
-        return cast(cls.quantity > 0, types.Boolean)
+        from sqlalchemy.sql.expression import cast, case
+        return cast(case([(cls.quantity > 0, 1)], else_=0), types.Boolean)
 
     def __str__(self):
         return f'{self.accession}{self.delimiter}{self.code}'

bauble/plugins/garden/source.py

@@ -39,7 +39,8 @@
                         Integer,
                         ForeignKey,
                         Float,
-                        UnicodeText)
+                        UnicodeText,
+                        literal)
 from sqlalchemy.orm import relationship, backref
 from sqlalchemy.orm.session import object_session
 
@@ -976,9 +977,9 @@ def search_view_markup_pair(self):
     def has_children(self):
         from sqlalchemy import exists
         session = object_session(self)
-        return session.query(
-            exists().where(Source.source_detail_id == self.id)
-        ).scalar()
+        return bool(session.query(literal(True))
+                    .filter(exists().where(Source.source_detail_id == self.id))
+                    .scalar())
 
     def count_children(self):
         session = object_session(self)

bauble/plugins/imex/test_imex.py

@@ -26,6 +26,7 @@
 import tempfile
 import json
 from datetime import datetime
+from dateutil.parser import parse as date_parse
 from pathlib import Path
 from tempfile import TemporaryDirectory
 
@@ -287,7 +288,7 @@ def test_sequences(self):
         if db.engine.name == 'postgresql':
             stmt = "SELECT nextval('family_id_seq')"
             nextval = conn.execute(stmt).fetchone()[0]
-        elif db.engine.name == 'sqlite':
+        elif db.engine.name in ('sqlite', 'mssql'):
             # max(id) isn't really safe in production use but is ok for a test
             stmt = "SELECT max(id) from family;"
             nextval = conn.execute(stmt).fetchone()[0] + 1
@@ -307,9 +308,8 @@ def test_import_no_inherit(self):
         """
         Test importing a row with None doesn't inherit from previous row.
         """
-        query = self.session.query(Genus)
-        self.assertTrue(query[1].author != query[0].author,
-                        (query[1].author, query[0].author))
+        query = self.session.query(Genus).all()
+        self.assertNotEqual(query[1].author, query[0].author)
 
     def test_export_none_is_empty(self):
         """
@@ -354,7 +354,7 @@ def test_sequences(self):
         if db.engine.name == 'postgresql':
             stmt = "SELECT nextval('family_id_seq')"
             nextval = conn.execute(stmt).fetchone()[0]
-        elif db.engine.name == 'sqlite':
+        elif db.engine.name in ('sqlite', 'mssql'):
             # max(id) isn't really safe in production use but is ok for a test
             stmt = "SELECT max(id) from family;"
             nextval = conn.execute(stmt).fetchone()[0] + 1
@@ -1764,36 +1764,40 @@ def setUp(self):
         garden_test.setUp_data()
 
     def test_get_item_value_gets_datetime_datetime_type(self):
-        datetime_fmat = prefs.prefs.get(prefs.datetime_format_pref)
         item = Plant(code='3', accession_id=1, location_id=1, quantity=10)
         self.session.add(item)
         self.session.commit()
-        now = datetime.now().strftime(datetime_fmat)
+        now = datetime.now().timestamp()
         val = GenericExporter.get_item_value('planted.date', item)
-        # accuracy is seconds, chance of a mismatch should be uncommon
-        self.assertEqual(val, now)
+        # accuracy is seconds
+        val = date_parse(val).timestamp()
+        self.assertAlmostEqual(val, now, delta=1)
 
     def test_get_item_value_gets_date_type(self):
-        date_fmat = prefs.prefs.get(prefs.date_format_pref)
         item = Accession(code='2020.4',
                          species_id=1,
                          date_accd=datetime.now())
         self.session.add(item)
         self.session.commit()
-        now = datetime.now().strftime(date_fmat)
+        now = (datetime.now()
+               .replace(hour=0, minute=0, second=0, microsecond=0)
+               .timestamp())
         val = GenericExporter.get_item_value('date_accd', item)
-        # accuracy is seconds, chance of a mismatch should be very uncommon
-        self.assertEqual(val, now)
+        # accuracy is a day - i.e. very rarely this could spill over from one
+        # day to the next
+        val = date_parse(val).timestamp()
+        secs_in_day = 86400
+        self.assertAlmostEqual(val, now, delta=secs_in_day)
 
     def test_get_item_value_gets_datetime_type(self):
-        datetime_fmat = prefs.prefs.get(prefs.datetime_format_pref)
         item = Plant(code='3', accession_id=1, location_id=1, quantity=10)
         self.session.add(item)
         self.session.commit()
-        now = datetime.now().strftime(datetime_fmat)
+        now = datetime.now().timestamp()
         val = GenericExporter.get_item_value('_created', item)
-        # accuracy is seconds, chance of a mismatch should be uncommon
-        self.assertEqual(val, now)
+        # accuracy is seconds
+        val = date_parse(val).timestamp()
+        self.assertAlmostEqual(val, now, delta=1)
 
     def test_get_item_value_gets_path(self):
         item = self.session.query(Plant).get(1)

bauble/plugins/plants/family.py

@@ -30,8 +30,13 @@
 
 from gi.repository import Gtk
 
-from sqlalchemy import (Column, Integer, ForeignKey, and_, UniqueConstraint,
-                        String)
+from sqlalchemy import (Column,
+                        Integer,
+                        ForeignKey,
+                        and_,
+                        UniqueConstraint,
+                        String,
+                        literal)
 from sqlalchemy.orm import relationship, validates
 from sqlalchemy.orm import synonym as sa_synonym
 from sqlalchemy.orm.session import object_session
@@ -289,9 +294,9 @@ def has_children(self):
         cls = self.__class__.genera.prop.mapper.class_
         from sqlalchemy import exists
         session = object_session(self)
-        return session.query(
-            exists().where(cls.family_id == self.id)
-        ).scalar()
+        return bool(session.query(literal(True))
+                    .filter(exists().where(cls.family_id == self.id))
+                    .scalar())
 
     def count_children(self):
         cls = self.__class__.genera.prop.mapper.class_

bauble/plugins/plants/genus.py

@@ -30,8 +30,14 @@
 
 from gi.repository import Gtk  # noqa
 
-from sqlalchemy import (Column, Unicode, Integer, ForeignKey, String,
-                        UniqueConstraint, and_)
+from sqlalchemy import (Column,
+                        Unicode,
+                        Integer,
+                        ForeignKey,
+                        String,
+                        UniqueConstraint,
+                        and_,
+                        literal)
 from sqlalchemy.orm import relationship, backref
 from sqlalchemy.orm import synonym as sa_synonym
 from sqlalchemy.orm.session import object_session
@@ -405,7 +411,9 @@ def has_children(self):
         cls = self.__class__.species.prop.mapper.class_
         from sqlalchemy import exists
         session = object_session(self)
-        return session.query(exists().where(cls.genus_id == self.id)).scalar()
+        return bool(session.query(literal(True))
+                    .filter(exists().where(cls.genus_id == self.id))
+                    .scalar())
 
     def count_children(self):
         cls = self.__class__.species.prop.mapper.class_

bauble/plugins/plants/geography.py

@@ -37,7 +37,8 @@
                         String,
                         Integer,
                         ForeignKey,
-                        and_)
+                        and_,
+                        literal)
 from sqlalchemy.orm import object_session, relationship, backref, deferred
 
 from bauble import db, utils
@@ -241,9 +242,10 @@ def has_children(self):
             parent_ids = [i[0] for i in child_id]
             ids.update(parent_ids)
 
-        return session.query(
-            exists().where(SpeciesDistribution.geography_id.in_(ids))
-        ).scalar()
+        return bool(session.query(literal(True))
+                    .filter(exists()
+                            .where(SpeciesDistribution.geography_id.in_(ids)))
+                    .scalar())
 
     def count_children(self):
         # Much more expensive than other models

bauble/plugins/plants/species_model.py

@@ -34,7 +34,8 @@
                         ForeignKey,
                         UnicodeText,
                         UniqueConstraint,
-                        func)
+                        func,
+                        literal)
 from sqlalchemy.orm import relationship, backref, object_session
 from sqlalchemy.orm import synonym as sa_synonym
 from sqlalchemy.ext.hybrid import hybrid_property
@@ -901,9 +902,9 @@ def has_children(self):
         cls = self.__class__.accessions.prop.mapper.class_
         from sqlalchemy import exists
         session = object_session(self)
-        return session.query(
-            exists().where(cls.species_id == self.id)
-        ).scalar()
+        return bool(session.query(literal(True))
+                    .filter(exists().where(cls.species_id == self.id))
+                    .scalar())
 
     def count_children(self):
         cls = self.__class__.accessions.prop.mapper.class_

bauble/plugins/plants/test_plants.py

@@ -36,7 +36,8 @@
 from bauble.test import (BaubleTestCase,
                          check_dupids,
                          mockfunc,
-                         update_gui)
+                         update_gui,
+                         wait_on_threads)
 from . import SplashInfoBox
 from .species import (Species,
                       VernacularName,
@@ -3072,7 +3073,7 @@ class SplashInfoBoxTests(BaubleTestCase):
     def test_update_sensitise_exclude_inactive(self, _mock_gui):
         splash = SplashInfoBox()
         splash.update()
-        # wait_on_threads()
+        wait_on_threads()
         for widget in [splash.splash_nplttot,
                        splash.splash_npltnot,
                        splash.splash_nacctot,
@@ -3083,7 +3084,7 @@ def test_update_sensitise_exclude_inactive(self, _mock_gui):
 
         prefs.prefs[prefs.exclude_inactive_pref] = True
         splash.update()
-        # wait_on_threads()
+        wait_on_threads()
         for widget in [splash.splash_nplttot,
                        splash.splash_npltnot,
                        splash.splash_nacctot,