From 5ea098c400ff14d42847def768f7962965ddf3d4 Mon Sep 17 00:00:00 2001 From: Michael Hall Date: Sun, 7 Jan 2024 06:48:38 +1000 Subject: [PATCH 1/8] build: update dev dependencies --- .github/workflows/ci.yaml | 2 +- poetry.lock | 107 ++++++++++++++++++-------------------- pyproject.toml | 8 +-- 3 files changed, 55 insertions(+), 62 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e207c6e..22cf1e0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -40,7 +40,7 @@ jobs: # have to do this as use of tmpdir in tests changes directory covg_report="$(pwd)/coverage.xml" poetry run pytest --cov=snakefmt --cov-report=xml:$covg_report --cov-branch tests/ - + - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: diff --git a/poetry.lock b/poetry.lock index cf7f851..e22a0ae 100644 --- a/poetry.lock +++ b/poetry.lock @@ -11,16 +11,6 @@ files = [ {file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"}, ] -[[package]] -name = "atomicwrites" -version = "1.4.1" -description = "Atomic file writes." -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -files = [ - {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, -] - [[package]] name = "attrs" version = "23.2.0" @@ -307,6 +297,9 @@ files = [ {file = "coverage-7.4.0.tar.gz", hash = "sha256:707c0f58cb1712b8809ece32b68996ee1e609f71bd14615bd8f87a1293cb610e"}, ] +[package.dependencies] +tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} + [package.extras] toml = ["tomli"] @@ -366,6 +359,20 @@ files = [ {file = "dpath-2.1.6.tar.gz", hash = "sha256:f1e07c72e8605c6a9e80b64bc8f42714de08a789c7de417e49c3f87a19692e47"}, ] +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + [[package]] name = "fastjsonschema" version = "2.19.1" @@ -382,19 +389,19 @@ devel = ["colorama", "json-spec", "jsonschema", "pylint", "pytest", "pytest-benc [[package]] name = "flake8" -version = "3.9.2" +version = "7.0.0" description = "the modular source code checker: pep8 pyflakes and co" optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +python-versions = ">=3.8.1" files = [ - {file = "flake8-3.9.2-py2.py3-none-any.whl", hash = "sha256:bf8fd333346d844f616e8d47905ef3a3384edae6b4e9beb0c5101e25e3110907"}, - {file = "flake8-3.9.2.tar.gz", hash = "sha256:07528381786f2a6237b061f6e96610a4167b226cb926e2aa2b6b1d78057c576b"}, + {file = "flake8-7.0.0-py2.py3-none-any.whl", hash = "sha256:a6dfbb75e03252917f2473ea9653f7cd799c3064e54d4c8140044c5c065f53c3"}, + {file = "flake8-7.0.0.tar.gz", hash = "sha256:33f96621059e65eec474169085dc92bf26e7b2d47366b70be2f67ab80dc25132"}, ] [package.dependencies] -mccabe = ">=0.6.0,<0.7.0" -pycodestyle = ">=2.7.0,<2.8.0" -pyflakes = ">=2.3.0,<2.4.0" +mccabe = ">=0.7.0,<0.8.0" +pycodestyle = ">=2.11.0,<2.12.0" +pyflakes = ">=3.2.0,<3.3.0" [[package]] name = "gitdb" @@ -712,13 +719,13 @@ files = [ [[package]] name = "mccabe" -version = "0.6.1" +version = "0.7.0" description = "McCabe checker, plugin for flake8" optional = false -python-versions = "*" +python-versions = ">=3.6" files = [ - {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, - {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, + {file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"}, + {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, ] [[package]] @@ -931,37 +938,26 @@ files = [ {file = "PuLP-2.7.0.tar.gz", hash = "sha256:e73ee6b32d639c9b8cf4b4aded334ba158be5f8313544e056f796ace0a10ae63"}, ] -[[package]] -name = "py" -version = "1.11.0" -description = "library with cross-python path, ini-parsing, io, code, log facilities" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -files = [ - {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, - {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, -] - [[package]] name = "pycodestyle" -version = "2.7.0" +version = "2.11.1" description = "Python style guide checker" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +python-versions = ">=3.8" files = [ - {file = "pycodestyle-2.7.0-py2.py3-none-any.whl", hash = "sha256:514f76d918fcc0b55c6680472f0a37970994e07bbb80725808c17089be302068"}, - {file = "pycodestyle-2.7.0.tar.gz", hash = "sha256:c389c1d06bf7904078ca03399a4816f974a1d590090fecea0c63ec26ebaf1cef"}, + {file = "pycodestyle-2.11.1-py2.py3-none-any.whl", hash = "sha256:44fe31000b2d866f2e41841b18528a505fbd7fef9017b04eff4e2648a0fadc67"}, + {file = "pycodestyle-2.11.1.tar.gz", hash = "sha256:41ba0e7afc9752dfb53ced5489e89f8186be00e599e712660695b7a75ff2663f"}, ] [[package]] name = "pyflakes" -version = "2.3.1" +version = "3.2.0" description = "passive checker of Python programs" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +python-versions = ">=3.8" files = [ - {file = "pyflakes-2.3.1-py2.py3-none-any.whl", hash = "sha256:7893783d01b8a89811dd72d7dfd4d84ff098e5eed95cfa8905b22bbffe52efc3"}, - {file = "pyflakes-2.3.1.tar.gz", hash = "sha256:f5bc8ecabc05bb9d291eb5203d6810b49040f6ff446a756326104746cc00c1db"}, + {file = "pyflakes-3.2.0-py2.py3-none-any.whl", hash = "sha256:84b5be138a2dfbb40689ca07e2152deb896a65c3a3e24c251c5c62489568074a"}, + {file = "pyflakes-3.2.0.tar.gz", hash = "sha256:1c61603ff154621fb2a9172037d84dca3500def8c8b630657d1701f026f8af3f"}, ] [[package]] @@ -995,43 +991,40 @@ files = [ [[package]] name = "pytest" -version = "6.2.5" +version = "7.4.4" description = "pytest: simple powerful testing with Python" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" files = [ - {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, - {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, + {file = "pytest-7.4.4-py3-none-any.whl", hash = "sha256:b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8"}, + {file = "pytest-7.4.4.tar.gz", hash = "sha256:2cf0005922c6ace4a3e2ec8b4080eb0d9753fdc93107415332f50ce9e7994280"}, ] [package.dependencies] -atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} -attrs = ">=19.2.0" colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} iniconfig = "*" packaging = "*" pluggy = ">=0.12,<2.0" -py = ">=1.8.2" -toml = "*" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} [package.extras] -testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] [[package]] name = "pytest-cov" -version = "2.12.1" +version = "4.1.0" description = "Pytest plugin for measuring coverage." optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = ">=3.7" files = [ - {file = "pytest-cov-2.12.1.tar.gz", hash = "sha256:261ceeb8c227b726249b376b8526b600f38667ee314f910353fa318caa01f4d7"}, - {file = "pytest_cov-2.12.1-py2.py3-none-any.whl", hash = "sha256:261bb9e47e65bd099c89c3edf92972865210c36813f80ede5277dceb77a4a62a"}, + {file = "pytest-cov-4.1.0.tar.gz", hash = "sha256:3904b13dfbfec47f003b8e77fd5b589cd11904a21ddf1ab38a64f204d6a10ef6"}, + {file = "pytest_cov-4.1.0-py3-none-any.whl", hash = "sha256:6ba70b9e97e69fcc3fb45bfeab2d0a138fb65c4d0d6a41ef33983ad114be8c3a"}, ] [package.dependencies] -coverage = ">=5.2.1" +coverage = {version = ">=5.2.1", extras = ["toml"]} pytest = ">=4.6" -toml = "*" [package.extras] testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtualenv"] @@ -1572,5 +1565,5 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" -python-versions = "^3.8.0" -content-hash = "1cd7fc178467b378023a46cd076a9d5529f2d4d1d0e1ddf7473e3a875269b561" +python-versions = "^3.8.1" +content-hash = "16e63c29e4b70cb2dd143525a98b52d7e479501c67c51b98fd801a591d9550a4" diff --git a/pyproject.toml b/pyproject.toml index 5fa50da..e67c61b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,16 +14,16 @@ keywords = ["python", "snakemake", "code", "formatter", "parser", "workflow", "m snakefmt = 'snakefmt.snakefmt:main' [tool.poetry.dependencies] -python = "^3.8.0" +python = "^3.8.1" click = "^8.0.0" black = "^23.12.1" toml = "^0.10.2" importlib_metadata = {version = ">=1.7.0,<5.0", python = "<3.8"} [tool.poetry.dev-dependencies] -pytest = "^6.2.5" -pytest-cov = "^2.8.1" -flake8 = "^3.7.9" +pytest = "^7.4.4" +pytest-cov = "^4.1.0" +flake8 = "^7.0" snakemake = "<8.0" isort = "^5.1.0" pynvim = "~0.4.3" From e0abb808f60c9d8ce39b296ab64cbc3814015f15 Mon Sep 17 00:00:00 2001 From: Michael Hall Date: Sun, 7 Jan 2024 06:48:52 +1000 Subject: [PATCH 2/8] chore: lint --- tests/test_snakefmt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_snakefmt.py b/tests/test_snakefmt.py index 41e0e06..ce38205 100644 --- a/tests/test_snakefmt.py +++ b/tests/test_snakefmt.py @@ -282,7 +282,7 @@ def test_diff_doesnt_output_diff_if_error(self, cli_runner): result = cli_runner.invoke(main, params, input=stdin) - assert type(result.exception) == SyntaxError + assert isinstance(result.exception, SyntaxError) assert result.exit_code != 0 assert result.output == "" From c055b654913edbb17527aadf0838a39d30daa21b Mon Sep 17 00:00:00 2001 From: Michael Hall Date: Sun, 7 Jan 2024 13:31:40 +1000 Subject: [PATCH 3/8] ci: update release please workflow actions versions --- .github/workflows/release-please.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release-please.yaml b/.github/workflows/release-please.yaml index 22af797..2d59669 100644 --- a/.github/workflows/release-please.yaml +++ b/.github/workflows/release-please.yaml @@ -20,15 +20,13 @@ jobs: - uses: actions/checkout@v4 if: ${{ steps.release.outputs.release_created }} - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v5 if: ${{ steps.release.outputs.release_created }} with: - python-version: "3.7" + python-version: "3.11" - name: Install and configure Poetry uses: snok/install-poetry@v1 - with: - version: 1.3.1 - name: Install dependencies if: ${{ steps.release.outputs.release_created }} From 3410511d5a8ad3b3135d20f9952b9aabe77973a9 Mon Sep 17 00:00:00 2001 From: Michael Hall Date: Sun, 7 Jan 2024 20:47:07 +1000 Subject: [PATCH 4/8] fix: handle python3.12 f-string tokenization [closes #210] --- snakefmt/parser/syntax.py | 4 ++++ tests/test_formatter.py | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/snakefmt/parser/syntax.py b/snakefmt/parser/syntax.py index 6491856..9c5b6c2 100644 --- a/snakefmt/parser/syntax.py +++ b/snakefmt/parser/syntax.py @@ -36,6 +36,10 @@ tokenize.NUMBER: {tokenize.NAME, tokenize.OP}, tokenize.OP: {tokenize.NAME, tokenize.STRING, tokenize.NUMBER, tokenize.OP}, } +# add fstring start to spacing_triggers if python 3.12 or higher +if hasattr(tokenize, "FSTRING_START"): + spacing_triggers[tokenize.NAME].add(tokenize.FSTRING_START) + spacing_triggers[tokenize.OP].add(tokenize.FSTRING_START) def operator_skip_spacing(prev_token: Token, token: Token) -> bool: diff --git a/tests/test_formatter.py b/tests/test_formatter.py index cff23b4..7d1fae1 100644 --- a/tests/test_formatter.py +++ b/tests/test_formatter.py @@ -393,6 +393,14 @@ def test_decorator_is_handled_correctly(self): actual = formatter.get_formatted() assert actual == snakecode + def test_f_strings(self): + """This is relevant for python3.12""" + snakecode = 'a = f"{1 + 2}" if 1 > 0 else f"{1 - 2}"\n' + formatter = setup_formatter(snakecode) + + actual = formatter.get_formatted() + assert actual == snakecode + class TestComplexPythonFormatting: """ From a3220fc1915de13dff6cb13bd88b50d232572c30 Mon Sep 17 00:00:00 2001 From: Michael Hall Date: Sun, 7 Jan 2024 20:48:16 +1000 Subject: [PATCH 5/8] fix: don't add space between string and comma [python3.12 f-string tokenize] --- snakefmt/parser/syntax.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/snakefmt/parser/syntax.py b/snakefmt/parser/syntax.py index 9c5b6c2..b8203f0 100644 --- a/snakefmt/parser/syntax.py +++ b/snakefmt/parser/syntax.py @@ -54,6 +54,8 @@ def operator_skip_spacing(prev_token: Token, token: Token) -> bool: return True elif prev_token.type == tokenize.NAME and token.string == "(": return True + elif prev_token.type == tokenize.STRING and token.string == ",": + return True else: return False From 18fb5072a29f694710052f2673f428483e63296f Mon Sep 17 00:00:00 2001 From: Michael Hall Date: Sun, 7 Jan 2024 20:49:32 +1000 Subject: [PATCH 6/8] ci: add python3.12 to CI testing --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 22cf1e0..a5a2ad9 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -14,7 +14,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: [ 3.8, 3.9, "3.10", "3.11" ] + python-version: [ 3.8, 3.9, "3.10", "3.11", "3.12" ] os: [ ubuntu-latest, macos-latest ] steps: From a3f31c48f4e8c024858e75ae88758778d2eccc43 Mon Sep 17 00:00:00 2001 From: Michael Hall Date: Sun, 7 Jan 2024 22:02:26 +1000 Subject: [PATCH 7/8] fix: don't remove double braces in f-strings in rule directives [closes #207] --- snakefmt/parser/syntax.py | 11 ++++++++++- tests/test_formatter.py | 17 +++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/snakefmt/parser/syntax.py b/snakefmt/parser/syntax.py index b8203f0..becda7a 100644 --- a/snakefmt/parser/syntax.py +++ b/snakefmt/parser/syntax.py @@ -1,6 +1,7 @@ """ Code in charge of parsing and validating Snakemake syntax """ +import sys import tokenize from abc import ABC, abstractmethod from re import match as re_match @@ -329,9 +330,17 @@ def in_brackets(self): def parse_params(self, snakefile: TokenIterator): cur_param = Parameter(self.token) prev_token = None - while True: cur_param = self.process_token(cur_param, prev_token) + if ( + self.token is not None + and sys.version_info >= (3, 12) + and self.token.type == tokenize.FSTRING_MIDDLE + ): + if self.token.string.endswith("}"): + cur_param.value += "}" + elif self.token.string.endswith("{"): + cur_param.value += "{" try: prev_token = self.token self.token = next(snakefile) diff --git a/tests/test_formatter.py b/tests/test_formatter.py index 7d1fae1..02a8d72 100644 --- a/tests/test_formatter.py +++ b/tests/test_formatter.py @@ -845,6 +845,23 @@ def test_tpq_inside_run_block(self): assert formatter.get_formatted() == snakecode + def test_f_string_with_double_braces_in_input(self): + """rule align: + input: + sequences="results/{build_name}/filtered.fasta", + params: + translation_template=lambda w: f"{w.build_name}/{{cds}}.fasta",""" + snakecode = ( + "rule align:\n" + f"{TAB * 1}input:\n" + f'{TAB * 2}sequences="results/{{build_name}}/filtered.fasta",\n' + f"{TAB * 1}params:\n" + f"{TAB * 2}" + + 'translation_template=lambda w: f"{w.build_name}/{{cds}}.fasta",\n' + ) + formatter = setup_formatter(snakecode) + assert formatter.get_formatted() == snakecode + class TestReformatting_SMK_BREAK: """ From 62bca5909610afca6ebc88eb046b3e3dd390eafe Mon Sep 17 00:00:00 2001 From: Michael Hall Date: Sun, 7 Jan 2024 22:25:41 +1000 Subject: [PATCH 8/8] chore: correct reference to test source --- tests/test_formatter.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/test_formatter.py b/tests/test_formatter.py index 02a8d72..67a51f6 100644 --- a/tests/test_formatter.py +++ b/tests/test_formatter.py @@ -846,11 +846,7 @@ def test_tpq_inside_run_block(self): assert formatter.get_formatted() == snakecode def test_f_string_with_double_braces_in_input(self): - """rule align: - input: - sequences="results/{build_name}/filtered.fasta", - params: - translation_template=lambda w: f"{w.build_name}/{{cds}}.fasta",""" + """https://github.com/snakemake/snakefmt/issues/207""" snakecode = ( "rule align:\n" f"{TAB * 1}input:\n"