Merge pull request #20 from zytedata/dud-upgrade

use new duplicate-url-discarder==0.2.0 item pipeline
zytedata · Jul 23, 2024 · 74d81c8 · 74d81c8
2 parents 358342a + 3e0cf2c
commit 74d81c8
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 1 deletion.
diff --git a/requirements.txt b/requirements.txt
@@ -1,4 +1,4 @@
 Scrapy==2.11.2
 scrapy-zyte-api[provider]>=0.17.3
 zyte-spider-templates==0.7.2
-duplicate-url-discarder[rules]>=0.1.0
+duplicate-url-discarder[rules]>=0.2.0
diff --git a/zyte_spider_templates_project/settings.py b/zyte_spider_templates_project/settings.py
@@ -30,3 +30,19 @@
     "zyte_spider_templates.pages",
     "zyte_spider_templates_project.pages",
 ]
+
+# duplicate-url-discarder
+DUD_ATTRIBUTES_PER_ITEM = {
+    "zyte_common_items.Product": [
+        "canonicalUrl",
+        "brand",
+        "name",
+        "gtin",
+        "mpn",
+        "productId",
+        "sku",
+        "color",
+        "size",
+        "style"
+    ],
+}