From ad6436cc554824c46eb7ec9ffb4dac1fdde73b5b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 4 Nov 2024 18:32:20 +0500 Subject: [PATCH 1/3] Update Python versions. --- .github/workflows/publish.yml | 2 +- .github/workflows/test.yml | 5 ++--- pyproject.toml | 4 ++-- tox.ini | 2 +- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index e91b782..139bae7 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -17,7 +17,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version: '3.13' - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 50366ee..835d098 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,12 +16,11 @@ jobs: fail-fast: false matrix: include: - - python-version: "3.8" - python-version: "3.9" - python-version: "3.10" - python-version: "3.11" - python-version: "3.12" - - python-version: "3.12" + - python-version: "3.13" steps: - uses: actions/checkout@v4 @@ -42,7 +41,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.12"] + python-version: ["3.13"] tox-job: ["twine-check"] steps: diff --git a/pyproject.toml b/pyproject.toml index 960e4c2..1966f4e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,14 +15,14 @@ classifiers = [ "Natural Language :: English", "Operating System :: OS Independent", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", ] -requires-python = ">=3.8" +requires-python = ">=3.9" dynamic = ["version"] [tool.setuptools.dynamic] diff --git a/tox.ini b/tox.ini index b6c2a9b..41c62a9 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py38,py39,py310,py311,py312,twine-check +envlist = py39,py310,py311,py312,py313,twine-check [testenv] deps = From 68a40a32c4a3a1550f0bbe0a8600164db94397ff Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 4 Nov 2024 18:36:45 +0500 Subject: [PATCH 2/3] Add pre-commit. --- .github/workflows/test.yml | 2 +- .pre-commit-config.yaml | 22 +++++++++++++++++++ pyproject.toml | 7 ++++++ setup.cfg | 4 ++++ tests/normalizer/test_normalizer.py | 2 +- .../test_query_removal_except.py | 2 +- tests/subpathRemoval/test_subpath_removal.py | 2 +- tests/test_rules.py | 7 +++++- tox.ini | 6 ++++- 9 files changed, 48 insertions(+), 6 deletions(-) create mode 100644 .pre-commit-config.yaml create mode 100644 setup.cfg diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 835d098..11a5c88 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,7 +42,7 @@ jobs: fail-fast: false matrix: python-version: ["3.13"] - tox-job: ["twine-check"] + tox-job: ["linters", "twine-check"] steps: - uses: actions/checkout@v4 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..1eb6339 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,22 @@ +repos: +- repo: https://github.com/PyCQA/isort + rev: 5.13.2 + hooks: + - id: isort +- repo: https://github.com/psf/black + rev: 24.10.0 + hooks: + - id: black +- repo: https://github.com/pycqa/flake8 + rev: 7.1.1 + hooks: + - id: flake8 + additional_dependencies: + - flake8-docstrings + - flake8-print +- repo: https://github.com/adamchainz/blacken-docs + rev: 1.19.1 + hooks: + - id: blacken-docs + additional_dependencies: + - black==24.10.0 diff --git a/pyproject.toml b/pyproject.toml index 1966f4e..a55df0b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,3 +33,10 @@ Source = "https://github.com/zytedata/duplicate-url-discarder-rules" [tool.setuptools.package-data] 'duplicate_url_discarder_rules' = ['**/*.json'] + +[tool.black] +target-version = ["py39", "py310", "py311", "py312", "py313"] + +[tool.isort] +profile = "black" +multi_line_output = 3 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..50c143a --- /dev/null +++ b/setup.cfg @@ -0,0 +1,4 @@ +[flake8] +ignore = E203, E266, E501, W503, C901 +max-line-length = 88 +select = B,C,E,F,W,T4 diff --git a/tests/normalizer/test_normalizer.py b/tests/normalizer/test_normalizer.py index b65fa9a..c520993 100644 --- a/tests/normalizer/test_normalizer.py +++ b/tests/normalizer/test_normalizer.py @@ -1,5 +1,5 @@ -from duplicate_url_discarder.url_canonicalizer import UrlCanonicalizer from duplicate_url_discarder.processors import NormalizerProcessor +from duplicate_url_discarder.url_canonicalizer import UrlCanonicalizer from duplicate_url_discarder_rules import RULE_PATHS diff --git a/tests/queryRemovalExcept/test_query_removal_except.py b/tests/queryRemovalExcept/test_query_removal_except.py index c8af3d1..cfbddf5 100644 --- a/tests/queryRemovalExcept/test_query_removal_except.py +++ b/tests/queryRemovalExcept/test_query_removal_except.py @@ -1,5 +1,5 @@ -from duplicate_url_discarder.url_canonicalizer import UrlCanonicalizer from duplicate_url_discarder.processors import QueryRemovalExceptProcessor +from duplicate_url_discarder.url_canonicalizer import UrlCanonicalizer from duplicate_url_discarder_rules import RULE_PATHS diff --git a/tests/subpathRemoval/test_subpath_removal.py b/tests/subpathRemoval/test_subpath_removal.py index 456f241..8284d55 100644 --- a/tests/subpathRemoval/test_subpath_removal.py +++ b/tests/subpathRemoval/test_subpath_removal.py @@ -1,5 +1,5 @@ -from duplicate_url_discarder.url_canonicalizer import UrlCanonicalizer from duplicate_url_discarder.processors import SubpathRemovalProcessor +from duplicate_url_discarder.url_canonicalizer import UrlCanonicalizer from duplicate_url_discarder_rules import RULE_PATHS diff --git a/tests/test_rules.py b/tests/test_rules.py index b35098b..3e391d7 100644 --- a/tests/test_rules.py +++ b/tests/test_rules.py @@ -2,7 +2,12 @@ from url_matcher import Patterns -from duplicate_url_discarder_rules import RULE_PATHS, RULE_PATHS_ARTICLE, RULE_PATHS_PRODUCT, RULE_PATHS_COMMON +from duplicate_url_discarder_rules import ( + RULE_PATHS, + RULE_PATHS_ARTICLE, + RULE_PATHS_COMMON, + RULE_PATHS_PRODUCT, +) def test_rule_validity(): diff --git a/tox.ini b/tox.ini index 41c62a9..217c2db 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py39,py310,py311,py312,py313,twine-check +envlist = py39,py310,py311,py312,py313,linters,twine-check [testenv] deps = @@ -11,6 +11,10 @@ commands = py.test \ {posargs:duplicate_url_discarder_rules tests} +[testenv:linters] +deps = pre-commit +commands = pre-commit run --all-files --show-diff-on-failure + [testenv:twine-check] deps = build From 11d565a8d0af412a93c68460ca032eb50c9db1b7 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 4 Nov 2024 18:52:46 +0500 Subject: [PATCH 3/3] Add mypy. --- .github/workflows/test.yml | 2 +- duplicate_url_discarder_rules/__init__.py | 2 +- pyproject.toml | 12 ++++++++++++ tests/normalizer/test_normalizer.py | 1 + .../queryRemovalExcept/test_query_removal_except.py | 1 + tests/subpathRemoval/test_subpath_removal.py | 1 + tests/test_rules.py | 2 ++ tox.ini | 8 +++++++- 8 files changed, 26 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 11a5c88..c90fd99 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,7 +42,7 @@ jobs: fail-fast: false matrix: python-version: ["3.13"] - tox-job: ["linters", "twine-check"] + tox-job: ["linters", "mypy", "twine-check"] steps: - uses: actions/checkout@v4 diff --git a/duplicate_url_discarder_rules/__init__.py b/duplicate_url_discarder_rules/__init__.py index b6bb37b..26ae940 100644 --- a/duplicate_url_discarder_rules/__init__.py +++ b/duplicate_url_discarder_rules/__init__.py @@ -16,7 +16,7 @@ RULE_PATHS_ARTICLE: List[str] = [] RULE_PATHS_PRODUCT: List[str] = [] -for path in RULE_PATHS: +for path in RULE_PATHS or []: filename = Path(path).name if filename == "article.json": RULE_PATHS_ARTICLE.append(path) diff --git a/pyproject.toml b/pyproject.toml index a55df0b..9bc7664 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,3 +40,15 @@ target-version = ["py39", "py310", "py311", "py312", "py313"] [tool.isort] profile = "black" multi_line_output = 3 + +[tool.mypy] +strict = true +implicit_reexport = true + +[[tool.mypy.overrides]] +module = "tests.*.*" +disallow_untyped_defs = false + +[[tool.mypy.overrides]] +module = "url_matcher" +ignore_missing_imports = true diff --git a/tests/normalizer/test_normalizer.py b/tests/normalizer/test_normalizer.py index c520993..5c12f11 100644 --- a/tests/normalizer/test_normalizer.py +++ b/tests/normalizer/test_normalizer.py @@ -5,6 +5,7 @@ def test_normalizer_main_rules(): + assert RULE_PATHS is not None rule_path = [path for path in RULE_PATHS if path.endswith("normalizer/main.json")] assert len(rule_path) == 1 diff --git a/tests/queryRemovalExcept/test_query_removal_except.py b/tests/queryRemovalExcept/test_query_removal_except.py index cfbddf5..f855e36 100644 --- a/tests/queryRemovalExcept/test_query_removal_except.py +++ b/tests/queryRemovalExcept/test_query_removal_except.py @@ -5,6 +5,7 @@ def test_query_removal_except_product_rules(): + assert RULE_PATHS is not None rule_path = [ path for path in RULE_PATHS if path.endswith("queryRemovalExcept/product.json") ] diff --git a/tests/subpathRemoval/test_subpath_removal.py b/tests/subpathRemoval/test_subpath_removal.py index 8284d55..16f4337 100644 --- a/tests/subpathRemoval/test_subpath_removal.py +++ b/tests/subpathRemoval/test_subpath_removal.py @@ -5,6 +5,7 @@ def test_subpath_removal_product_rules(): + assert RULE_PATHS is not None rule_path = [ path for path in RULE_PATHS if path.endswith("subpathRemoval/product.json") ] diff --git a/tests/test_rules.py b/tests/test_rules.py index 3e391d7..b774bfd 100644 --- a/tests/test_rules.py +++ b/tests/test_rules.py @@ -11,6 +11,7 @@ def test_rule_validity(): + assert RULE_PATHS is not None for path in RULE_PATHS: try: with open(path, "r") as f: @@ -45,6 +46,7 @@ def test_rule_validity(): def test_rules_concat(): + assert RULE_PATHS is not None all_rules = RULE_PATHS_COMMON + RULE_PATHS_ARTICLE + RULE_PATHS_PRODUCT assert isinstance(all_rules, list) for path in all_rules: diff --git a/tox.ini b/tox.ini index 217c2db..a28e146 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py39,py310,py311,py312,py313,linters,twine-check +envlist = py39,py310,py311,py312,py313,linters,mypy,twine-check [testenv] deps = @@ -15,6 +15,12 @@ commands = deps = pre-commit commands = pre-commit run --all-files --show-diff-on-failure +[testenv:mypy] +deps = + mypy==1.13.0 + duplicate-url-discarder +commands = mypy duplicate_url_discarder_rules tests + [testenv:twine-check] deps = build