Add Python 3.13, drop Python 3.8, update tool versions (#211)

.github/workflows/publish.yml

@@ -17,7 +17,7 @@ jobs:
  - name: Set up Python
  uses: actions/setup-python@v5
  with:
- python-version: '3.12'
+ python-version: '3.13'
  - name: Install dependencies
  run: |
  python -m pip install --upgrade pip

.github/workflows/test.yml

@@ -16,20 +16,20 @@ jobs:
  fail-fast: false
  matrix:
  include:
- - python-version: "3.8"
+ - python-version: "3.9"
  toxenv: "min"
- - python-version: "3.8"
+ - python-version: "3.9"
  toxenv: "pinned-scrapy-2x7"
- - python-version: "3.8"
+ - python-version: "3.9"
  toxenv: "pinned-scrapy-2x8"
- - python-version: "3.8"
+ - python-version: "3.9"
  toxenv: "asyncio-min"
- - python-version: "3.8"
  - python-version: "3.9"
  - python-version: "3.10"
  - python-version: "3.11"
  - python-version: "3.12"
- - python-version: "3.12"
+ - python-version: "3.13"
+ - python-version: "3.13"
  toxenv: "asyncio"
 
  steps:
@@ -54,7 +54,7 @@ jobs:
  strategy:
  fail-fast: false
  matrix:
- python-version: ['3.12']
+ python-version: ['3.12'] # Keep in sync with .readthedocs.yml
  tox-job: ["mypy", "docs", "linters", "twinecheck"]
 
  steps:

.pre-commit-config.yaml

@@ -3,12 +3,12 @@ repos:
  - id: black
  language_version: python3
  repo: https://github.com/ambv/black
- rev: 22.12.0
+ rev: 24.10.0
  - hooks:
  - id: isort
  language_version: python3
  repo: https://github.com/PyCQA/isort
- rev: 5.11.5
+ rev: 5.13.2
  - hooks:
  - id: flake8
  language_version: python3
@@ -19,4 +19,4 @@ repos:
  - flake8-docstrings
  - flake8-string-format
  repo: https://github.com/pycqa/flake8
- rev: 6.1.0
+ rev: 7.1.1

.readthedocs.yml

@@ -6,7 +6,7 @@ sphinx:
 build:
  os: ubuntu-22.04
  tools:
- python: "3.12" # Keep in sync with .github/workflows/tests.yml
+ python: "3.12" # Keep in sync with .github/workflows/test.yml
 
 python:
  install:

MANIFEST.in

@@ -1,4 +1,4 @@
-include CHANGES.rst
+include CHANGELOG.rst
 include LICENSE
 include README.rst
 

README.rst

@@ -48,7 +48,7 @@ Installation
 
  pip install scrapy-poet
 
-Requires **Python 3.8+** and **Scrapy >= 2.6.0**.
+Requires **Python 3.9+** and **Scrapy >= 2.6.0**.
 
 Usage in a Scrapy Project
 =========================

docs/intro/install.rst

@@ -7,7 +7,7 @@ Installation
 Installing scrapy-poet
 ======================
 
-``scrapy-poet`` is a Scrapy extension that runs on Python 3.8 and above.
+``scrapy-poet`` is a Scrapy extension that runs on Python 3.9 and above.
 
 If you’re already familiar with installation of Python packages, you can install
 ``scrapy-poet`` and its dependencies from PyPI with:

docs/providers.rst

@@ -327,8 +327,6 @@ you could implement those limits in the library itself.
 Attaching metadata to dependencies
 ==================================
 
-.. note:: This feature requires Python 3.9+.
-
 Providers can support dependencies with arbitrary metadata attached and use
 that metadata when creating them. Attaching the metadata is done by wrapping
 the dependency class in :data:`typing.Annotated`:

example/example/autoextract.py

@@ -2,6 +2,7 @@
 Example of how to create a PageObject with a very different input data,
 which even requires an API request.
 """
+
 from typing import Any, Dict
 
 import attr

example/example/spiders/books_01.py

@@ -1,6 +1,7 @@
 """
 Baseline: regular Scrapy spider, sweet & easy.
 """
+
 import scrapy
 
 

example/example/spiders/books_02.py

@@ -2,6 +2,7 @@
 Scrapy spider which uses Page Objects to make extraction code more reusable.
 BookPage is now independent of Scrapy.
 """
+
 import scrapy
 from web_poet import WebPage
 

example/example/spiders/books_02_1.py

@@ -3,6 +3,7 @@
 BookPage is now independent of Scrapy. callback_for is used to reduce
 boilerplate.
 """
+
 import scrapy
 from web_poet import WebPage
 

example/example/spiders/books_02_2.py

@@ -10,6 +10,7 @@
 has problems now, it is used in the latter examples, because as an API
 it is better than defining callback explicitly.
 """
+
 import scrapy
 from web_poet import WebPage
 

example/example/spiders/books_02_3.py

@@ -7,6 +7,7 @@
 Page object is used instead of callback below. It doesn't work now,
 but it can be implemented, with Scrapy support.
 """
+
 import scrapy
 from web_poet import WebPage
 

example/example/spiders/books_03.py

@@ -1,6 +1,7 @@
 """
 Scrapy spider which uses AutoExtract API, to extract books as products.
 """
+
 import scrapy
 from example.autoextract import ProductPage
 

example/example/spiders/books_04.py

@@ -1,6 +1,7 @@
 """
 Scrapy spider which uses Page Objects both for crawling and extraction.
 """
+
 import scrapy
 from web_poet import WebPage
 

example/example/spiders/books_04_overrides_01.py

@@ -5,6 +5,7 @@
 
 The default configured PO logic contains the logic for books.toscrape.com
 """
+
 import scrapy
 from web_poet import ApplyRule, WebPage
 

example/example/spiders/books_04_overrides_02.py

@@ -6,6 +6,7 @@
 No configured default logic: if used for an unregistered domain, no logic
 at all is applied.
 """
+
 import scrapy
 from web_poet import WebPage
 from web_poet.rules import ApplyRule

example/example/spiders/books_04_overrides_03.py

@@ -10,6 +10,7 @@
 difference is that this example is using the ``@handle_urls`` decorator to
 store the rules in web-poet's registry.
 """
+
 import scrapy
 from web_poet import WebPage, default_registry, handle_urls
 

example/example/spiders/books_05.py

@@ -2,6 +2,7 @@
 Scrapy spider which uses Page Objects both for crawling and extraction.
 You can mix various page types freely.
 """
+
 import scrapy
 from example.autoextract import ProductPage
 from web_poet import WebPage

scrapy_poet/_request_fingerprinter.py

@@ -10,7 +10,7 @@
  import json
  from functools import cached_property
  from logging import getLogger
- from typing import Callable, Dict, List, Optional, get_args, get_origin
+ from typing import Annotated, Callable, Dict, List, Optional, get_args, get_origin
  from weakref import WeakKeyDictionary
 
  from andi import CustomBuilder
@@ -37,10 +37,6 @@
  def _serialize_dep(cls):
  if isinstance(cls, CustomBuilder):
  cls = cls.result_class_or_fn
- try:
- from typing import Annotated
- except ImportError:
- pass
  else:
  if get_origin(cls) is Annotated:
  annotated, *annotations = get_args(cls)

scrapy_poet/downloadermiddlewares.py

@@ -2,6 +2,7 @@
 responsible for injecting Page Input dependencies before the request callbacks
 are executed.
 """
+
 import inspect
 import logging
 import warnings

scrapy_poet/page_input_providers.py

@@ -8,6 +8,7 @@
 different providers in order to acquire data from multiple external sources,
 for example, from scrapy-playwright or from an API for automatic extraction.
 """
+
 from typing import Any, Callable, ClassVar, FrozenSet, List, Set, Union
 from warnings import warn
 

setup.py

@@ -19,7 +19,7 @@
  "scrapy.commands": ["savefixture = scrapy_poet.commands:SaveFixtureCommand"]
  },
  package_data={"scrapy_poet": ["VERSION"]},
- python_requires=">=3.8",
+ python_requires=">=3.9",
  install_requires=[
  "andi >= 0.6.0",
  "attrs >= 21.3.0",
@@ -39,10 +39,10 @@
  "Operating System :: OS Independent",
  "Framework :: Scrapy",
  "Programming Language :: Python :: 3",
- "Programming Language :: Python :: 3.8",
  "Programming Language :: Python :: 3.9",
  "Programming Language :: Python :: 3.10",
  "Programming Language :: Python :: 3.11",
  "Programming Language :: Python :: 3.12",
+ "Programming Language :: Python :: 3.13",
  ],
 )

tests/test_commands.py

@@ -6,7 +6,6 @@
 import tempfile
 from pathlib import Path
 
-import pytest
 from twisted.web.resource import Resource
 from web_poet.testing import Fixture
 
@@ -246,9 +245,6 @@ class CustomItemAdapter(ItemAdapter):
  result.assert_outcomes(passed=3)
 
 
-@pytest.mark.skipif(
- sys.version_info < (3, 9), reason="No Annotated support in Python < 3.9"
-)
 def test_savefixture_annotated(pytester) -> None:
  project_name = "foo"
  cwd = Path(pytester.path)