add duplicate-url-discarder[rules]

zytedata · Jul 5, 2024 · ccf94ee · ccf94ee
1 parent 6309eae
commit ccf94ee
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 0 deletions.
diff --git a/requirements.txt b/requirements.txt
@@ -1,3 +1,4 @@
 Scrapy==2.11.2
 scrapy-zyte-api[provider]>=0.17.3
 zyte-spider-templates==0.7.1
+duplicate-url-discarder[rules]>=0.1.0
diff --git a/zyte_spider_templates_project/settings.py b/zyte_spider_templates_project/settings.py
@@ -14,6 +14,7 @@
 CLOSESPIDER_TIMEOUT_NO_ITEM = 900
 ADDONS = {
     "scrapy_zyte_api.Addon": 500,
+    "duplicate_url_discarder.Addon": 600,
 }
 SCHEDULER_DISK_QUEUE = "scrapy.squeues.PickleFifoDiskQueue"
 SCHEDULER_MEMORY_QUEUE = "scrapy.squeues.FifoMemoryQueue"