XML and Text formatting support (#184)

* Add support for XML.

Add bindings to XMLFragment, XMLElement, XMLText, and XMLEvent. Allows
reading and writing of XML fragments from the document, which rich text
editors use heavily.

Closes #151

* Add support for formatting and embeds in Text

Rich text editors often store their markup in attributes and embeds.

* Allow apply_update in a transaction

Currently trying to use apply_update in a transaction causes a deadlock,
as apply_update tries to make its own new transaction but is locked out.
This reuses the existing transaction, which also allows applying
multiple updates in a single transaction.

* Add __eq__ and __hash__ to XML types

Allows comparing whether objects refer to the same element in a doc.
Important since every access generates a new, unique Python wrapper
instance.

__eq__ is based directly from the underlying rust objects' `Eq`
implementation. __hash__ is based off of the `Hash` implementation of
the `BranchID` of the rust objects.

* Add XmlText.diff

Like Text, XmlText can contain embedded formatting, so make that
available.

* Mypy and ruff fixes and formatting

* Document XML types

* More lint fixes

* Add XML related tests and fixes for found bugs

.gitignore

@@ -3,3 +3,6 @@ __pycache__/
 *.so
 Cargo.lock
 .coverage
+/site
+/dist
+_pycrdt.*.pyd

docs/api_reference.md

@@ -24,6 +24,11 @@
       - Transaction
       - TransactionEvent
       - UndoManager
+      - XmlElement
+      - XmlFragment
+      - XmlText
+      - XmlChildrenView
+      - XmlAttributesView
       - YMessageType
       - YSyncMessageType
       - create_awareness_message

python/pycrdt/__init__.py

@@ -28,3 +28,7 @@
 from ._update import get_state as get_state
 from ._update import get_update as get_update
 from ._update import merge_updates as merge_updates
+from ._xml import XmlElement as XmlElement
+from ._xml import XmlEvent as XmlEvent
+from ._xml import XmlFragment as XmlFragment
+from ._xml import XmlText as XmlText

python/pycrdt/_base.py

@@ -21,6 +21,11 @@
 event_types: dict[Any, type[BaseEvent]] = {}
 
 
+def forbid_read_transaction(txn: Transaction):
+    if isinstance(txn, ReadTransaction):
+        raise RuntimeError("Read-only transaction cannot be used to modify document structure")
+
+
 class BaseDoc:
     _doc: _Doc
     _twin_doc: BaseDoc | None
@@ -91,8 +96,7 @@ def _get_or_insert(self, name: str, doc: Doc) -> Any: ...
     def _init(self, value: Any | None) -> None: ...
 
     def _forbid_read_transaction(self, txn: Transaction):
-        if isinstance(txn, ReadTransaction):
-            raise RuntimeError("Read-only transaction cannot be used to modify document structure")
+        forbid_read_transaction(txn)
 
     def _integrate(self, doc: Doc, integrated: Any) -> Any:
         prelim = self._prelim

python/pycrdt/_doc.py

@@ -2,7 +2,7 @@
 
 from typing import Any, Callable, Iterable, Type, TypeVar, cast
 
-from ._base import BaseDoc, BaseType, base_types
+from ._base import BaseDoc, BaseType, base_types, forbid_read_transaction
 from ._pycrdt import Doc as _Doc
 from ._pycrdt import SubdocsEvent, Subscription, TransactionEvent
 from ._pycrdt import Transaction as _Transaction
@@ -160,7 +160,10 @@ def apply_update(self, update: bytes) -> None:
             except Exception as e:
                 self._twin_doc = Doc(dict(self))
                 raise e
-        self._doc.apply_update(update)
+        with self.transaction() as txn:
+            forbid_read_transaction(txn)
+            assert txn._txn is not None
+            self._doc.apply_update(txn._txn, update)
 
     def __setitem__(self, key: str, value: BaseType) -> None:
         """

python/pycrdt/_pycrdt.pyi

@@ -1,4 +1,4 @@
-from typing import Any, Callable
+from typing import Any, Callable, Iterator
 
 class Doc:
     """Shared document."""
@@ -28,13 +28,16 @@ class Doc:
     def get_or_insert_map(self, name: str) -> Map:
         """Create a map root type on this document, or get an existing one."""
 
+    def get_or_insert_xml_fragment(self, name: str) -> XmlFragment:
+        """Create an XML fragment root type on this document, or get an existing one."""
+
     def get_state(self) -> bytes:
         """Get the current document state."""
 
     def get_update(self, state: bytes) -> bytes:
         """Get the update from the given state to the current state."""
 
-    def apply_update(self, update: bytes) -> None:
+    def apply_update(self, txn: Transaction, update: bytes) -> None:
         """Apply the update to the document."""
 
     def roots(self, txn: Transaction) -> dict[str, Text | Array | Map]:
@@ -94,22 +97,49 @@ class MapEvent:
     """Event generated by `Map.observe` method. Emitted during transaction commit
     phase."""
 
+class XmlEvent:
+    """Event generated by `Xml*.observe` methods. Emitted during transaction commit
+    phase."""
+
 class Text:
     """Shared text."""
 
     def len(self, txn: Transaction) -> int:
         """Returns the number of characters visible in the current shared text."""
 
-    def insert(self, txn: Transaction, index: int, chunk: str) -> None:
+    def insert(
+        self,
+        txn: Transaction,
+        index: int,
+        chunk: str,
+        attrs: Iterator[tuple[str, Any]] | None = None,
+    ) -> None:
         """Inserts a `chunk` of text at a given `index`."""
 
+    def insert_embed(
+        self,
+        txn: Transaction,
+        index: int,
+        embed: Any,
+        attrs: Iterator[tuple[str, Any]] | None = None,
+    ) -> None:
+        """Inserts an embed at a given `index`."""
+
+    def format(
+        self, txn: Transaction, index: int, len: int, attrs: Iterator[tuple[str, Any]]
+    ) -> None:
+        """Formats a range of elements"""
+
     def remove_range(self, txn: Transaction, index: int, len: int) -> None:
         """Removes up to `len` characters from th current shared text, starting at
         given`index`."""
 
     def get_string(self, txn: Transaction) -> str:
         """Returns a text representation of the current shared text."""
 
+    def diff(self, txn: Transaction) -> list[tuple[Any, dict[str, Any] | None]]:
+        """Returns a sequence of formatted chunks"""
+
     def observe(self, callback: Callable[[TextEvent], None]) -> Subscription:
         """Subscribes a callback to be called with the shared text change event.
         Returns a subscription that can be used to unsubscribe."""
@@ -183,6 +213,126 @@ class Map:
         """Unsubscribes previously subscribed event callback identified by given
         `subscription`."""
 
+class XmlFragment:
+    def parent(self) -> XmlFragment | XmlElement | XmlText | None: ...
+    def get_string(self, txn: Transaction) -> str:
+        """Returns a text representation of the current shared xml."""
+
+    def len(self, txn: Transaction) -> int:
+        """Returns the numer of children of the current shared xml."""
+
+    def get(self, txn: Transaction, index: int) -> XmlFragment | XmlElement | XmlText | None:
+        """Gets a child item by index, or None if the index is out of bounds"""
+
+    def remove_range(self, txn: Transaction, index: int, len: int) -> None:
+        """Removes a range of children"""
+
+    def insert_str(self, txn: Transaction, index: int, text: str) -> XmlText:
+        """Inserts a text node"""
+
+    def insert_element_prelim(self, txn: Transaction, index: int, tag: str) -> XmlElement:
+        """Inserts an empty element node"""
+
+    def observe(self, callback: Callable[[XmlEvent], None]) -> Subscription:
+        """Subscribes a callback to be called with the xml change event.
+        Returns a subscription that can be used to unsubscribe."""
+
+    def observe_deep(self, callback: Callable[[XmlEvent], None]) -> Subscription:
+        """Subscribes a callback to be called with the xml change event
+        and its nested elements.
+        Returns a subscription that can be used to unsubscribe."""
+
+class XmlElement:
+    def parent(self) -> XmlFragment | XmlElement | XmlText | None: ...
+    def get_string(self, txn: Transaction) -> str:
+        """Returns a text representation of the current shared xml."""
+
+    def len(self, txn: Transaction) -> int:
+        """Returns the numer of children of the current shared xml."""
+
+    def get(self, txn: Transaction, index: int) -> XmlFragment | XmlElement | XmlText | None:
+        """Gets a child item by index, or None if the index is out of bounds"""
+
+    def remove_range(self, txn: Transaction, index: int, len: int) -> None:
+        """Removes a range of children"""
+
+    def insert_str(self, txn: Transaction, index: int, text: str) -> XmlText:
+        """Inserts a text node"""
+
+    def insert_element_prelim(self, txn: Transaction, index: int, tag: str) -> XmlElement:
+        """Inserts an empty element node"""
+
+    def attributes(self, txn: Transaction) -> list[tuple[str, str]]:
+        """Gets all attributes, as a list of `(key, value)` tuples"""
+
+    def attribute(self, txn: Transaction, name: str) -> str | None:
+        """Gets an attribute, or None if the attribute does not exist"""
+
+    def insert_attribute(self, txn: Transaction, name: str, value: str) -> None:
+        """Inserts or overwrites an attribute."""
+
+    def remove_attribute(self, txn: Transaction, name: str) -> None:
+        """Removes an attribute"""
+
+    def siblings(self, txn) -> list[XmlFragment | XmlElement | XmlText]:
+        """Gets the siblings of this node"""
+
+    def observe(self, callback: Callable[[XmlEvent], None]) -> Subscription:
+        """Subscribes a callback to be called with the xml change event.
+        Returns a subscription that can be used to unsubscribe."""
+
+    def observe_deep(self, callback: Callable[[XmlEvent], None]) -> Subscription:
+        """Subscribes a callback to be called with the xml change event
+        and its nested elements.
+        Returns a subscription that can be used to unsubscribe."""
+
+class XmlText:
+    def parent(self) -> XmlFragment | XmlElement | XmlText | None: ...
+    def get_string(self, txn: Transaction) -> str:
+        """Returns a text representation of the current shared xml."""
+
+    def attributes(self, txn: Transaction) -> list[tuple[str, str]]:
+        """Gets all attributes, as a list of `(key, value)` tuples"""
+
+    def attribute(self, txn: Transaction, name: str) -> str | None:
+        """Gets an attribute, or None if the attribute does not exist"""
+
+    def insert_attribute(self, txn: Transaction, name: str, value: str) -> None:
+        """Inserts or overwrites an attribute."""
+
+    def remove_attribute(self, txn: Transaction, name: str) -> None:
+        """Removes an attribute"""
+
+    def siblings(self, txn: Transaction) -> list[XmlFragment | XmlElement | XmlText]:
+        """Gets the siblings of this node"""
+
+    def insert(
+        self,
+        txn: Transaction,
+        index: int,
+        text: str,
+        attrs: Iterator[tuple[str, Any]] | None = None,
+    ):
+        """Inserts text, optionally with attributes"""
+
+    def remove_range(self, txn: Transaction, index: int, len: int):
+        """Removes text"""
+
+    def format(self, txn: Transaction, index: int, len: int, attrs: Iterator[tuple[str, Any]]):
+        """Adds attributes to a section of text"""
+
+    def diff(self, txn: Transaction) -> list[tuple[Any, dict[str, Any] | None]]:
+        """Returns a sequence of formatted chunks"""
+
+    def observe(self, callback: Callable[[XmlEvent], None]) -> Subscription:
+        """Subscribes a callback to be called with the xml change event.
+        Returns a subscription that can be used to unsubscribe."""
+
+    def observe_deep(self, callback: Callable[[XmlEvent], None]) -> Subscription:
+        """Subscribes a callback to be called with the xml change event
+        and its nested elements.
+        Returns a subscription that can be used to unsubscribe."""
+
 class UndoManager:
     """Undo manager."""
 

python/pycrdt/_text.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Callable, cast
+from typing import TYPE_CHECKING, Any, Callable, cast
 
 from ._base import BaseEvent, BaseType, base_types, event_types
 from ._pycrdt import Subscription
@@ -233,18 +233,56 @@ def clear(self) -> None:
         """Remove the entire range of characters."""
         del self[:]
 
-    def insert(self, index: int, value: str) -> None:
+    def insert(self, index: int, value: str, attrs: dict[str, Any] | None = None) -> None:
         """
         Inserts a string at a given index in the text.
+        ```py
         Doc()["text"] = text = Text("Hello World!")
         text.insert(5, ",")
         assert text == "Hello, World!"
+        ```
 
         Args:
             index: The index where to insert the string.
             value: The string to insert in the text.
+            attrs: Optional dictionary of attributes to apply
+        """
+        with self.doc.transaction() as txn:
+            self._forbid_read_transaction(txn)
+            self.integrated.insert(
+                txn._txn, index, value, iter(attrs.items()) if attrs is not None else None
+            )
+
+    def insert_embed(self, index: int, value: Any, attrs: dict[str, Any] | None = None) -> None:
+        """
+        Insert 'value' as an embed at a given index in the text.
         """
-        self[index:index] = value
+        with self.doc.transaction() as txn:
+            self._forbid_read_transaction(txn)
+            self.integrated.insert_embed(
+                txn._txn, index, value, iter(attrs.items()) if attrs is not None else None
+            )
+
+    def format(self, start: int, stop: int, attrs: dict[str, Any]) -> None:
+        """
+        Adds attribute to a section of text
+        """
+        with self.doc.transaction() as txn:
+            self._forbid_read_transaction(txn)
+            start, stop = self._check_slice(slice(start, stop))
+            length = stop - start
+            if length > 0:
+                self.integrated.format(txn._txn, start, length, iter(attrs.items()))
+
+    def diff(self) -> list[tuple[Any, dict[str, Any] | None]]:
+        """
+        Returns list of formatted chunks that the current text corresponds to.
+
+        Each list item is a tuple containing the chunk's contents and formatting attributes. The
+        contents is usually the text as a string, but may be other data for embedded objects.
+        """
+        with self.doc.transaction() as txn:
+            return self.integrated.diff(txn._txn)
 
     def observe(self, callback: Callable[[TextEvent], None]) -> Subscription:
         """