Update

README.rst

@@ -16,7 +16,7 @@ Scrapy_ project built following `Zyte’s web scraping tutorial`_.
 Requirements
 ============
 
-Python 3.8 or higher.
+Python 3.9 or higher.
 
 
 Setup

requirements.txt

@@ -1,4 +1,4 @@
-scrapy
+scrapy==2.11.2
 scrapy-zyte-api
 shub
 zyte-spider-templates

scrapinghub.yml

@@ -1,4 +1,4 @@
 requirements:
   file: requirements.txt
 stacks:
-  default: scrapy:2.11
+  default: scrapy:2.11-20241022

tutorial/settings.py

@@ -1,3 +1,8 @@
+from itemadapter import ItemAdapter
+from zyte_common_items import ZyteItemAdapter
+
+ItemAdapter.ADAPTER_CLASSES.appendleft(ZyteItemAdapter)
+
 # Scrapy settings for tutorial project
 #
 # For simplicity, this file contains only settings considered important or
@@ -96,27 +101,15 @@
 FEED_EXPORT_ENCODING = "utf-8"
 
 # Custom settings
-DOWNLOAD_HANDLERS = {
-    "http": "scrapy_zyte_api.ScrapyZyteAPIDownloadHandler",
-    "https": "scrapy_zyte_api.ScrapyZyteAPIDownloadHandler",
+ADDONS = {
+    "scrapy_zyte_api.Addon": 500,
+    "zyte_spider_templates.Addon": 700,
 }
+ZYTE_API_KEY = "YOUR_API_KEY"
 DOWNLOADER_MIDDLEWARES = {
-    "scrapy_poet.InjectionMiddleware": 543,
-    "scrapy_zyte_api.ScrapyZyteAPIDownloaderMiddleware": 1000,
+    "scrapy.downloadermiddlewares.stats.DownloaderStats": None,
+    "scrapy_poet.DownloaderStatsMiddleware": 850,
 }
-REQUEST_FINGERPRINTER_CLASS = "scrapy_zyte_api.ScrapyZyteAPIRequestFingerprinter"
-ZYTE_API_TRANSPARENT_MODE = True
 SPIDER_MIDDLEWARES = {
-    "scrapy_zyte_api.ScrapyZyteAPISpiderMiddleware": 100,
     "scrapy_poet.RetryMiddleware": 275,
-    "zyte_spider_templates.middlewares.CrawlingLogsMiddleware": 1000,
-}
-SCRAPY_POET_DISCOVER = [
-    "zyte_spider_templates.page_objects",
-]
-SCRAPY_POET_PROVIDERS = {
-    "scrapy_zyte_api.providers.ZyteApiProvider": 1100,
 }
-CLOSESPIDER_TIMEOUT_NO_ITEM = 600
-SCHEDULER_DISK_QUEUE = "scrapy.squeues.PickleFifoDiskQueue"
-SCHEDULER_MEMORY_QUEUE = "scrapy.squeues.FifoMemoryQueue"

tutorial/spiders/quotes_toscrape_com_scroll_capture.py

@@ -0,0 +1,42 @@
+import json
+from base64 import b64decode
+
+from scrapy import Request, Spider
+
+
+class QuotesToScrapeComScrollCaptureSpider(Spider):
+    name = "quotes_toscrape_com_scroll_capture"
+
+    def start_requests(self):
+        yield Request(
+            "http://quotes.toscrape.com/scroll",
+            meta={
+                "zyte_api_automap": {
+                    "browserHtml": True,
+                    "actions": [
+                        {
+                            "action": "scrollBottom",
+                        },
+                    ],
+                    "networkCapture": [
+                        {
+                            "filterType": "url",
+                            "httpResponseBody": True,
+                            "value": "/api/",
+                            "matchType": "contains",
+                        },
+                    ],
+                },
+            },
+        )
+
+    def parse(self, response):
+        for capture in response.raw_api_response["networkCapture"]:
+            text = b64decode(capture["httpResponseBody"]).decode()
+            data = json.loads(text)
+            for quote in data["quotes"]:
+                yield {
+                    "author": quote["author"]["name"],
+                    "tags": quote["tags"],
+                    "text": quote["text"],
+                }