Feat: cheerio experimental queue

packages/actor-scraper/cheerio-scraper/.actor/actor.json

@@ -0,0 +1,6 @@
+{
+	"actorSpecification": 1,
+	"name": "cheerio-scraper",
+	"version": "0.1",
+	"buildTag": "latest"
+}

packages/actor-scraper/cheerio-scraper/package.json

@@ -7,7 +7,8 @@
     "dependencies": {
         "@apify/scraper-tools": "^1.1.1",
         "@crawlee/cheerio": "^3.1.0",
-        "apify": "^3.1.1"
+        "apify": "^3.1.1",
+        "got": "^12.6.0"
     },
     "devDependencies": {
         "@apify/tsconfig": "^0.1.0",

packages/actor-scraper/cheerio-scraper/src/internals/consts.ts

@@ -35,6 +35,7 @@ export interface Input {
     datasetName?: string;
     keyValueStoreName?: string;
     requestQueueName?: string;
+    experimentalQueue?: boolean;
 }
 
 export const enum ProxyRotation {

packages/actor-scraper/cheerio-scraper/src/internals/crawler_setup.ts

@@ -27,6 +27,7 @@ import { IncomingMessage } from 'node:http';
 import { dirname } from 'node:path';
 import { fileURLToPath, URL } from 'node:url';
 import { Input, ProxyRotation } from './consts.js';
+import { createRequestQueue } from './request_queue_experimental.js';
 
 const { SESSION_MAX_USAGE_COUNTS, META_KEY } = scraperToolsConstants;
 const SCHEMA = JSON.parse(await readFile(new URL('../../INPUT_SCHEMA.json', import.meta.url), 'utf8'));
@@ -135,6 +136,7 @@ export class CrawlerSetup implements CrawlerSetupOptions {
         this.requestList = await RequestList.open('CHEERIO_SCRAPER', startUrls);
 
         // RequestQueue
+
         this.requestQueue = await RequestQueue.open(this.requestQueueName);
 
         // Dataset
@@ -204,6 +206,12 @@ export class CrawlerSetup implements CrawlerSetupOptions {
 
         this.crawler = new CheerioCrawler(options);
 
+        if (this.input.experimentalQueue) {
+            // We have to override the queue after crawler creation.
+            // We could possibly extend the original request queue too.
+            this.crawler.requestQueue = await createRequestQueue() as any as RequestQueue;
+        }
+
         return this.crawler;
     }
 

packages/actor-scraper/cheerio-scraper/src/internals/queue.ts

@@ -0,0 +1,27 @@
+export default class Queue<T> {
+    private items: T[] = [];
+
+    enqueue(item: T): void {
+        this.items.push(item);
+    }
+
+    dequeue(): T | undefined {
+        return this.items.shift();
+    }
+
+    isEmpty(): boolean {
+        return this.items.length === 0;
+    }
+
+    size(): number {
+        return this.items.length;
+    }
+
+    toArray(): T[] {
+        return this.items;
+    }
+
+    fromArray(items: T[]) {
+        this.items = items;
+    }
+}