fix: support python3.12 f-string tokenizing

.github/workflows/ci.yaml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: [ 3.8, 3.9, "3.10", "3.11" ]
+        python-version: [ 3.8, 3.9, "3.10", "3.11", "3.12" ]
         os: [ ubuntu-latest, macos-latest ]
 
     steps:
@@ -40,7 +40,7 @@ jobs:
               # have to do this as use of tmpdir in tests changes directory
               covg_report="$(pwd)/coverage.xml"
               poetry run pytest --cov=snakefmt --cov-report=xml:$covg_report --cov-branch tests/
-          
+
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
         with:

.github/workflows/release-please.yaml

@@ -20,15 +20,13 @@ jobs:
       - uses: actions/checkout@v4
         if: ${{ steps.release.outputs.release_created }}
 
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v5
         if: ${{ steps.release.outputs.release_created }}
         with:
-          python-version: "3.7"
+          python-version: "3.11"
 
       - name: Install and configure Poetry
         uses: snok/install-poetry@v1
-        with:
-          version: 1.3.1
 
       - name: Install dependencies
         if: ${{ steps.release.outputs.release_created }}

pyproject.toml

@@ -14,16 +14,16 @@ keywords = ["python", "snakemake", "code", "formatter", "parser", "workflow", "m
 snakefmt = 'snakefmt.snakefmt:main'
 
 [tool.poetry.dependencies]
-python = "^3.8.0"
+python = "^3.8.1"
 click = "^8.0.0"
 black = "^23.12.1"
 toml = "^0.10.2"
 importlib_metadata = {version = ">=1.7.0,<5.0", python = "<3.8"}
 
 [tool.poetry.dev-dependencies]
-pytest = "^6.2.5"
-pytest-cov = "^2.8.1"
-flake8 = "^3.7.9"
+pytest = "^7.4.4"
+pytest-cov = "^4.1.0"
+flake8 = "^7.0"
 snakemake = "<8.0"
 isort = "^5.1.0"
 pynvim = "~0.4.3"

snakefmt/parser/syntax.py

@@ -1,6 +1,7 @@
 """
 Code in charge of parsing and validating Snakemake syntax
 """
+import sys
 import tokenize
 from abc import ABC, abstractmethod
 from re import match as re_match
@@ -36,6 +37,10 @@
     tokenize.NUMBER: {tokenize.NAME, tokenize.OP},
     tokenize.OP: {tokenize.NAME, tokenize.STRING, tokenize.NUMBER, tokenize.OP},
 }
+# add fstring start to spacing_triggers if python 3.12 or higher
+if hasattr(tokenize, "FSTRING_START"):
+    spacing_triggers[tokenize.NAME].add(tokenize.FSTRING_START)
+    spacing_triggers[tokenize.OP].add(tokenize.FSTRING_START)
 
 
 def operator_skip_spacing(prev_token: Token, token: Token) -> bool:
@@ -50,6 +55,8 @@
         return True
     elif prev_token.type == tokenize.NAME and token.string == "(":
         return True
+    elif prev_token.type == tokenize.STRING and token.string == ",":
+        return True
     else:
         return False
 
@@ -323,9 +330,17 @@
     def parse_params(self, snakefile: TokenIterator):
         cur_param = Parameter(self.token)
         prev_token = None
-
         while True:
             cur_param = self.process_token(cur_param, prev_token)
+            if (
+                self.token is not None
+                and sys.version_info >= (3, 12)
+                and self.token.type == tokenize.FSTRING_MIDDLE
+            ):
+                if self.token.string.endswith("}"):
+                    cur_param.value += "}"
+                elif self.token.string.endswith("{"):
+                    cur_param.value += "{"
             try:
                 prev_token = self.token
                 self.token = next(snakefile)

tests/test_formatter.py

@@ -393,6 +393,14 @@ def test_decorator_is_handled_correctly(self):
         actual = formatter.get_formatted()
         assert actual == snakecode
 
+    def test_f_strings(self):
+        """This is relevant for python3.12"""
+        snakecode = 'a = f"{1 + 2}" if 1 > 0 else f"{1 - 2}"\n'
+        formatter = setup_formatter(snakecode)
+
+        actual = formatter.get_formatted()
+        assert actual == snakecode
+
 
 class TestComplexPythonFormatting:
     """
@@ -837,6 +845,19 @@ def test_tpq_inside_run_block(self):
 
         assert formatter.get_formatted() == snakecode
 
+    def test_f_string_with_double_braces_in_input(self):
+        """https://github.com/snakemake/snakefmt/issues/207"""
+        snakecode = (
+            "rule align:\n"
+            f"{TAB * 1}input:\n"
+            f'{TAB * 2}sequences="results/{{build_name}}/filtered.fasta",\n'
+            f"{TAB * 1}params:\n"
+            f"{TAB * 2}"
+            + 'translation_template=lambda w: f"{w.build_name}/{{cds}}.fasta",\n'
+        )
+        formatter = setup_formatter(snakecode)
+        assert formatter.get_formatted() == snakecode
+
 
 class TestReformatting_SMK_BREAK:
     """

tests/test_snakefmt.py

@@ -282,7 +282,7 @@ def test_diff_doesnt_output_diff_if_error(self, cli_runner):
 
         result = cli_runner.invoke(main, params, input=stdin)
 
-        assert type(result.exception) == SyntaxError
+        assert isinstance(result.exception, SyntaxError)
         assert result.exit_code != 0
         assert result.output == ""