Merge pull request #128 from feltech/work/51-libraryFromJSONString

Add `"library_json"` setting

RELEASE_NOTES.md

@@ -1,6 +1,17 @@
 Release Notes
 =============
 
+v1.0.0-beta.x.y
+---------------
+
+### New features
+
+- Added new setting `"library_json"` to provide easier control of BAL's
+  internal library database when used in host test suites. It is a string
+  value presenting BAL's in-memory library as serialised JSON.
+  [#51](https://github.com/OpenAssetIO/OpenAssetIO-Manager-BAL/issues/51)
+
+
 v1.0.0-beta.1.0
 ---------------
 

plugin/openassetio_manager_bal/BasicAssetLibraryInterface.py

@@ -19,6 +19,7 @@
 """
 A single-class module, providing the BasicAssetLibraryInterface class.
 """
+import json
 import os
 import re
 import time
@@ -57,6 +58,7 @@
 DEFAULT_IDENTIFIER = "org.openassetio.examples.manager.bal"
 ENV_VAR_IDENTIFIER_OVERRIDE = "OPENASSETIO_BAL_IDENTIFIER"
 SETTINGS_KEY_LIBRARY_PATH = "library_path"
+SETTINGS_KEY_LIBRARY_JSON = "library_json"
 SETTINGS_KEY_SIMULATED_QUERY_LATENCY = "simulated_query_latency_ms"
 SETTINGS_KEY_ENTITY_REFERENCE_URL_SCHEME = "entity_reference_url_scheme"
 
@@ -118,7 +120,9 @@ def info(self):
         return {constants.kInfoKey_EntityReferencesMatchPrefix: self.__entity_refrence_prefix()}
 
     def settings(self, hostSession):
-        return self.__settings.copy()
+        augmented_settings = self.__settings.copy()
+        augmented_settings[SETTINGS_KEY_LIBRARY_JSON] = json.dumps(self.__library)
+        return augmented_settings
 
     def hasCapability(self, capability):
         """
@@ -168,37 +172,44 @@ def simulated_latency(self):
 
     def initialize(self, managerSettings, hostSession):
         self.__validate_settings(managerSettings)
-
+        logger = hostSession.logger()
         # Settings updates can be partial, so make sure we keep any
         # existing path.
         existing_library_path = self.__settings.get("library_path")
         library_path = managerSettings.get("library_path", existing_library_path)
 
         if not library_path:
-            hostSession.logger().log(
-                hostSession.logger().Severity.kDebug,
+            logger.log(
+                logger.Severity.kDebug,
                 "'library_path' not in settings or is empty, checking "
                 f"{self.__lib_path_envvar_name}",
             )
-            library_path = os.environ.get(self.__lib_path_envvar_name)
+            library_path = os.environ.get(self.__lib_path_envvar_name, "")
 
-        if not library_path:
+        # Pop from dictionary so it doesn't get merged into persistent
+        # settings, since the library will be serialised on-demand in
+        # `settings()`.
+        library_json = managerSettings.pop("library_json", None)
+
+        if not library_path and not library_json:
             raise ConfigurationException(
-                f"'library_path'/{self.__lib_path_envvar_name} not set or is empty"
+                f"'library_json'/'library_path'/{self.__lib_path_envvar_name} not set or is empty"
             )
 
         self.__settings.update(managerSettings)
         self.__settings["library_path"] = library_path
 
-        self.__library = {}
-        hostSession.logger().log(
-            hostSession.logger().Severity.kDebug,
-            f"Loading library from '{library_path}'",
-        )
-        self.__library = bal.load_library(library_path)
+        if library_json is not None:
+            if logger.isSeverityLogged(logger.Severity.kDebug):
+                logger.log(logger.Severity.kDebug, f"Parsing library from '{library_json}'")
+            self.__library = bal.parse_library(library_json)
 
-        hostSession.logger().log(
-            hostSession.logger().Severity.kDebug,
+        else:
+            logger.log(logger.Severity.kDebug, f"Loading library from '{library_path}'")
+            self.__library = bal.load_library(library_path)
+
+        logger.log(
+            logger.Severity.kDebug,
             f"Running with simulated query latency of "
             f"{self.__settings[SETTINGS_KEY_SIMULATED_QUERY_LATENCY]}ms",
         )
@@ -799,6 +810,7 @@ def __handle_exception(exc, idx, error_callback):
     def __make_default_settings() -> dict:
         """
         Generates a default settings dict for BAL.
+
         Note: as a library is required, the default settings are not enough
         to initialize the manager.
         """
@@ -814,10 +826,21 @@ def __validate_settings(settings: dict):
         Parses the supplied settings dict, raising if there are any
         unrecognized keys present.
         """
+        # pylint: disable=too-many-branches
         if SETTINGS_KEY_LIBRARY_PATH in settings:
             if not isinstance(settings[SETTINGS_KEY_LIBRARY_PATH], str):
                 raise ValueError(f"{SETTINGS_KEY_LIBRARY_PATH} must be a str")
 
+        if SETTINGS_KEY_LIBRARY_JSON in settings:
+            if not isinstance(settings[SETTINGS_KEY_LIBRARY_JSON], str):
+                raise ValueError(f"{SETTINGS_KEY_LIBRARY_JSON} must be a str")
+            try:
+                json.loads(settings[SETTINGS_KEY_LIBRARY_JSON])
+            except json.decoder.JSONDecodeError as err:
+                raise ValueError(
+                    f"{SETTINGS_KEY_LIBRARY_JSON} must be a valid JSON string"
+                ) from err
+
         if SETTINGS_KEY_SIMULATED_QUERY_LATENCY in settings:
             query_latency = settings[SETTINGS_KEY_SIMULATED_QUERY_LATENCY]
             # This bool check is because bools are also ints as far as

plugin/openassetio_manager_bal/bal.py

@@ -98,6 +98,13 @@ def load_library(path: str) -> dict:
     return library
 
 
+def parse_library(library_json: str):
+    """
+    Parse the library from a JSON string.
+    """
+    return json.loads(library_json)
+
+
 def exists(entity_info: EntityInfo, library: dict) -> bool:
     """
     Determines if the supplied entity exists in the library
@@ -358,7 +365,7 @@ def _copy_and_expand_trait_properties(entity_version_dict: dict, library: dict)
                 # append the other vars as kwarg. Fortunately this has
                 # exactly the precedence behaviour we want.
                 trait_data[prop] = string.Template(value).safe_substitute(
-                    os.environ, **library["variables"]
+                    os.environ, **library.get("variables", {})
                 )
 
                 subbed_val = trait_data[prop]

tests/bal_business_logic_suite.py

@@ -21,6 +21,7 @@
 # pylint: disable=invalid-name, missing-function-docstring, missing-class-docstring,
 # pylint: disable=too-few-public-methods,too-many-lines
 
+import json
 import operator
 import os
 import pathlib
@@ -85,6 +86,10 @@ def setUp(self):
             "resources",
             self._library,
         )
+        # library_json takes precedence, so remove library_json to
+        # ensure library_path is used.
+        del new_settings["library_json"]
+
         self.addCleanup(self.cleanUp)
         self._manager.initialize(new_settings)
 
@@ -225,6 +230,186 @@ def initialize_and_assert_scheme(self, scheme=None):
         self.assertTrue(str(published_refs[0]).startswith(prefix))
 
 
+class Test_initialize_library_as_json_string(LibraryOverrideTestCase):
+    # Override library just to ensure the cleanup step gets added,
+    # restoring the library back to its original state. See base class.
+    _library = "library_apiComplianceSuite.json"
+
+    def test_when_library_loaded_from_file_then_library_setting_contains_file_contents(self):
+        settings = self._manager.settings()
+        library_path = pathlib.Path(settings["library_path"])
+        expected_library = json.loads(library_path.read_text(encoding="utf-8"))
+        actual_library = json.loads(settings["library_json"])
+
+        # For simplicity, strip dynamically calculated values.
+        del actual_library["variables"]
+        self.assertDictEqual(expected_library, actual_library)
+
+    def test_when_library_json_updated_then_settings_updated(self):
+        expected_library = {"managementPolicy": {"read": {"default": {"some.policy": {}}}}}
+
+        self._manager.initialize({"library_json": json.dumps(expected_library)})
+
+        actual_library = json.loads(self._manager.settings()["library_json"])
+
+        self.assertDictEqual(actual_library, expected_library)
+
+    def test_when_library_json_is_invalid_primitive_value_then_raises(self):
+        with self.assertRaises(ValueError) as err:
+            self._manager.initialize({"library_json": ""})
+
+        self.assertEqual("library_json must be a valid JSON string", str(err.exception))
+
+    def test_when_library_json_is_invalid_object_then_raises(self):
+        with self.assertRaises(TypeError) as err:
+            self._manager.initialize({"library_json": {"variables": {"a": "b"}}})
+
+        # Error comes from pybind11 trying to coerce dict.
+        self.assertIn("incompatible function arguments", str(err.exception))
+
+    def test_when_library_json_provided_and_library_path_blank_then_settings_updated(self):
+        # Test to ensure we don't error on a blank library_path if
+        # library_json is given
+
+        expected_library = {"managementPolicy": {"read": {"default": {"some.policy": {}}}}}
+
+        self._manager.initialize(
+            {"library_json": json.dumps(expected_library), "library_path": ""}
+        )
+
+        actual_library = json.loads(self._manager.settings()["library_json"])
+
+        self.assertDictEqual(actual_library, expected_library)
+
+    def test_when_no_library_json_and_library_path_blank_then_raises(self):
+        expected_error = "'library_json'/'library_path'/BAL_LIBRARY_PATH not set or is empty"
+
+        with self.assertRaises(ConfigurationException) as exc:
+            self._manager.initialize({"library_path": ""})
+
+        self.assertEqual(str(exc.exception), expected_error)
+
+    def test_when_library_provided_as_json_and_as_file_then_json_takes_precedence(self):
+        library_path = self._manager.settings()["library_path"]
+        self.assertGreater(len(library_path), 0)  # Confidence check.
+        expected_library = {"variables": {"a": "b"}}
+
+        self._manager.initialize(
+            {"library_json": json.dumps(expected_library), "library_path": library_path}
+        )
+        actual_library = json.loads(self._manager.settings()["library_json"])
+
+        self.assertDictEqual(expected_library, actual_library)
+
+    def test_when_initialised_with_no_library_json_then_resets_to_library_file(self):
+        # Read in initial library file.
+        library_path = pathlib.Path(self._manager.settings()["library_path"])
+        expected_library = json.loads(library_path.read_text(encoding="utf-8"))
+        self.assertGreater(len(expected_library), 0)  # Confidence check.
+
+        # Mutate library (to empty dict).
+        self._manager.initialize({"library_json": "{}"})
+        self.assertEqual("{}", self._manager.settings()["library_json"])  # Confidence check.
+
+        # Re-`initialize` with an empty settings dict, triggering a
+        # reset of the library to use the previous `library_path` file.
+        self._manager.initialize({})
+
+        actual_library = json.loads(self._manager.settings()["library_json"])
+
+        # For simplicity, strip dynamically calculated values.
+        del actual_library["variables"]
+        self.assertDictEqual(expected_library, actual_library)
+
+    def test_when_in_memory_library_is_updated_then_library_json_is_updated(self):
+        # Publish a new entity that is not in the initial JSON library.
+        # This will mutate BAL's in-memory library.
+        self._manager.register(
+            self._manager.createEntityReference("bal:///new_entity"),
+            TraitsData(),
+            PublishingAccess.kWrite,
+            self.createTestContext(),
+        )
+
+        library = json.loads(self._manager.settings()["library_json"])
+
+        self.assertIn("new_entity", library["entities"])
+
+    def test_when_library_uses_undefined_substitution_variables_then_variables_not_substituted(
+        self,
+    ):
+        # Test illustrating that implicit variables for interpolation
+        # are not available when library is given as a JSON string,
+        # unlike for library files.
+
+        # setup
+
+        expected_library_json = json.dumps(
+            {
+                "entities": {
+                    "some_entity": {
+                        "versions": [
+                            {"traits": {"some.trait": {"some_key": "${bal_library_path}"}}}
+                        ]
+                    }
+                }
+            }
+        )
+
+        # action
+
+        self._manager.initialize({"library_json": expected_library_json})
+
+        # confirm
+
+        traits_data = self._manager.resolve(
+            self._manager.createEntityReference("bal:///some_entity"),
+            {"some.trait"},
+            ResolveAccess.kRead,
+            self.createTestContext(),
+        )
+        self.assertEqual(
+            traits_data.getTraitProperty("some.trait", "some_key"), "${bal_library_path}"
+        )
+
+    def test_when_library_uses_defined_substitution_variables_then_variables_are_substituted(
+        self,
+    ):
+        # Test illustrating that variables for interpolation must be
+        # explicitly provided when library is given as a JSON string.
+        # I.e. there are no implicit variables, unlike when the library
+        # is given as a JSON file.
+
+        # setup
+
+        expected_library_json = json.dumps(
+            {
+                "variables": {"bal_library_path": "/some/path"},
+                "entities": {
+                    "some_entity": {
+                        "versions": [
+                            {"traits": {"some.trait": {"some_key": "${bal_library_path}"}}}
+                        ]
+                    }
+                },
+            }
+        )
+
+        # action
+
+        self._manager.initialize({"library_json": expected_library_json})
+
+        # confirm
+
+        traits_data = self._manager.resolve(
+            self._manager.createEntityReference("bal:///some_entity"),
+            {"some.trait"},
+            ResolveAccess.kRead,
+            self.createTestContext(),
+        )
+        self.assertEqual(traits_data.getTraitProperty("some.trait", "some_key"), "/some/path")
+
+
 class Test_managementPolicy_missing_completely(LibraryOverrideTestCase):
     """
     Tests error case when  BAL library managementPolicy is missing.

tests/fixtures.py

@@ -101,6 +101,7 @@
         "test_when_settings_have_all_keys_then_all_settings_updated": {
             "some_settings_with_all_keys": {
                 "library_path": blank_library_path,
+                "library_json": json.dumps({"variables": {"a": "b"}}),
                 "simulated_query_latency_ms": 0,
                 "entity_reference_url_scheme": "thingy",
             }