Fix using xcresulttool on Xcode 16.0 beta 3+ (#424)

CHANGELOG.md

@@ -1,3 +1,17 @@
+Version 0.53.4
+-------------
+
+This is a bugfix version containing changes from [PR #424](https://github.com/codemagic-ci-cd/cli-tools/pull/424) to fix test result parsing with Xcode 16.0 beta 3+.
+
+**Bugfixes**
+- The following actions were fixed when used in conjunction with Xcode 16.0 beta 3+:
+    - `xcode-project run-tests`,
+    - `xcode-project test-summary`,
+    - `xcode-project junit-test-results`.
+
+**Development**
+- Add `get_tool_version` method to `codemagic.models.xctests.XcResultTool` which can be used to detect `xcresulttool` version from currently active Xcode developer directory.
+
 Version 0.53.3
 -------------
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "codemagic-cli-tools"
-version = "0.53.3"
+version = "0.53.4"
 description = "CLI tools used in Codemagic builds"
 readme = "README.md"
 authors = [

src/codemagic/__version__.py

@@ -1,5 +1,5 @@
 __title__ = "codemagic-cli-tools"
 __description__ = "CLI tools used in Codemagic builds"
-__version__ = "0.53.3.dev"
+__version__ = "0.53.4.dev"
 __url__ = "https://github.com/codemagic-ci-cd/cli-tools"
 __licence__ = "GNU General Public License v3.0"

src/codemagic/models/xctests/xcresulttool.py

@@ -2,7 +2,9 @@
 
 import json
 import pathlib
+import re
 import subprocess
+from functools import lru_cache
 from tempfile import NamedTemporaryFile
 from typing import TYPE_CHECKING
 from typing import Any
@@ -11,9 +13,12 @@
 from typing import Optional
 from typing import Sequence
 
+from packaging.version import Version
+
 from codemagic.cli import CommandArg
 from codemagic.mixins import RunningCliAppMixin
 from codemagic.mixins import StringConverterMixin
+from codemagic.utilities import log
 
 if TYPE_CHECKING:
     from codemagic.cli import CliApp
@@ -26,6 +31,42 @@ def __init__(self, message: str, stderr: str):
 
 
 class XcResultTool(RunningCliAppMixin, StringConverterMixin):
+    @classmethod
+    @lru_cache(1)
+    def get_tool_version(cls) -> Optional[Version]:
+        # Cache xcresulttool version to avoid repeated checks.
+        # Assumes Xcode (and thus xcresulttool) version remains constant during execution.
+
+        cmd_args = ["xcrun", "xcresulttool", "version"]
+        try:
+            stdout = cls._run_command(cmd_args, "Failed to obtain xcresulttool version")
+        except XcResultToolError as e:
+            log.get_file_logger(cls).exception(str(e))
+            return None
+
+        version_output = cls._str(stdout.strip())
+        # Expected version output of xcresulttool (bundled with Xcode 16.0 beta 3) is as follows:
+        # xcresulttool version 23024, format version 3.53 (current)
+        match = re.match(r"^xcresulttool version (?P<version>\d+(\.\d+)?)", version_output)
+
+        if not match:
+            log.get_file_logger(cls).error("Failed to capture xcresulttool version from %r", version_output)
+            return None
+
+        return Version(match.group("version"))
+
+    @classmethod
+    def _requires_legacy_flag(cls) -> bool:
+        """
+        Starting from Xcode 16 beta 3 'xcresulttool get --format json' has been deprecated and
+        cannot be used without '--legacy' flag.
+        """
+
+        version = cls.get_tool_version()
+        if not version:
+            return False
+        return version >= Version("23021")
+
     @classmethod
     def get_bundle(cls, xcresult: pathlib.Path) -> Dict[str, Any]:
         cmd_args: List[CommandArg] = [
@@ -37,6 +78,10 @@ def get_bundle(cls, xcresult: pathlib.Path) -> Dict[str, Any]:
             "--path",
             xcresult.expanduser(),
         ]
+
+        if cls._requires_legacy_flag():
+            cmd_args.append("--legacy")
+
         stdout = cls._run_command(cmd_args, f"Failed to get result bundle object from {xcresult}")
         return json.loads(stdout)
 
@@ -53,6 +98,10 @@ def get_object(cls, xcresult: pathlib.Path, object_id: str) -> Dict[str, Any]:
             "--id",
             object_id,
         ]
+
+        if cls._requires_legacy_flag():
+            cmd_args.append("--legacy")
+
         stdout = cls._run_command(cmd_args, f"Failed to get result bundle object {object_id} from {xcresult}")
         return json.loads(stdout)
 
@@ -82,8 +131,8 @@ def _run_command_with_cli_app(cls, cli_app: CliApp, command_args: Sequence[Comma
         Replace default stdout stream with direct file handle to bypass stream
         processing in Python which can be very slow. For example
         `xcrun xcresulttool get --format json --path results.xcresult --id 'object-id'`
-        can output 500K+ lines at 30+ MB. Processing it in small chunks in Python is very time
-        consuming whereas using file handles is almost instantaneous.
+        can output 500K+ lines at 30+ MB. Processing it in small chunks in Python is very
+        time-consuming whereas using file handles is almost instantaneous.
         """
         with NamedTemporaryFile(mode="w+b") as stdout_fd:
             process = cli_app.execute(command_args, suppress_output=True, stdout=stdout_fd)

tests/models/xctests/test_xcresulttool.py

@@ -0,0 +1,24 @@
+from typing import Optional
+from unittest import mock
+
+import pytest
+from codemagic.models.xctests import XcResultTool
+from packaging.version import Version
+
+
+@pytest.mark.parametrize(
+    ("xcresulttool_version", "expected_result"),
+    (
+        (None, False),
+        (Version("0"), False),
+        (Version("1"), False),
+        (Version("23020.9"), False),
+        (Version("23021"), True),
+        (Version("23021.1"), True),
+        (Version("23022"), True),
+        (Version("33022"), True),
+    ),
+)
+def test_requires_legacy_flag(xcresulttool_version: Optional[Version], expected_result: bool):
+    with mock.patch.object(XcResultTool, "get_tool_version", new=mock.MagicMock(return_value=xcresulttool_version)):
+        assert expected_result is XcResultTool._requires_legacy_flag()