Merge pull request #974 from mendableai/fix-sdk/next-in-when-502

[bug/JS-SDK]Added check for object and trycatch as workaround for 502s
mendableai · Dec 26, 2024 · 37f258b · 37f258b
2 parents c911aad + bcc18e1
commit 37f258b
Show file tree

Hide file tree

Showing 2 changed files with 39 additions and 35 deletions.
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@mendable/firecrawl-js",
-  "version": "1.10.0",
+  "version": "1.10.1",
   "description": "JavaScript SDK for Firecrawl API",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.ts
@@ -470,7 +470,7 @@ export default class FirecrawlApp {
           let statusData = response.data
           if ("data" in statusData) {
             let data = statusData.data;
-            while ('next' in statusData) {
+            while (typeof statusData === 'object' && 'next' in statusData) {
               statusData = (await this.getRequest(statusData.next, headers)).data;
               data = data.concat(statusData.data);
             }
@@ -704,7 +704,7 @@ export default class FirecrawlApp {
           let statusData = response.data
           if ("data" in statusData) {
             let data = statusData.data;
-            while ('next' in statusData) {
+            while (typeof statusData === 'object' && 'next' in statusData) {
               statusData = (await this.getRequest(statusData.next, headers)).data;
               data = data.concat(statusData.data);
             }
@@ -863,42 +863,46 @@ export default class FirecrawlApp {
     headers: AxiosRequestHeaders,
     checkInterval: number
   ): Promise<CrawlStatusResponse | ErrorResponse> {
-    while (true) {
-      let statusResponse: AxiosResponse = await this.getRequest(
-        `${this.apiUrl}/v1/crawl/${id}`,
-        headers
-      );
-      if (statusResponse.status === 200) {
-        let statusData = statusResponse.data;
-          if (statusData.status === "completed") {
-            if ("data" in statusData) {
-              let data = statusData.data;
-              while ('next' in statusData) {
-                statusResponse = await this.getRequest(statusData.next, headers);
-                statusData = statusResponse.data;
-                data = data.concat(statusData.data);
+    try {
+      while (true) {
+        let statusResponse: AxiosResponse = await this.getRequest(
+          `${this.apiUrl}/v1/crawl/${id}`,
+          headers
+        );
+        if (statusResponse.status === 200) {
+          let statusData = statusResponse.data;
+            if (statusData.status === "completed") {
+              if ("data" in statusData) {
+                let data = statusData.data;
+                while (typeof statusData === 'object' && 'next' in statusData) {
+                  statusResponse = await this.getRequest(statusData.next, headers);
+                  statusData = statusResponse.data;
+                  data = data.concat(statusData.data);
+                }
+                statusData.data = data;
+                return statusData;
+              } else {
+                throw new FirecrawlError("Crawl job completed but no data was returned", 500);
               }
-              statusData.data = data;
-              return statusData;
-            } else {
-              throw new FirecrawlError("Crawl job completed but no data was returned", 500);
-            }
-          } else if (
-          ["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)
-        ) {
-          checkInterval = Math.max(checkInterval, 2);
-          await new Promise((resolve) =>
-            setTimeout(resolve, checkInterval * 1000)
-          );
+            } else if (
+            ["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)
+          ) {
+            checkInterval = Math.max(checkInterval, 2);
+            await new Promise((resolve) =>
+              setTimeout(resolve, checkInterval * 1000)
+            );
+          } else {
+            throw new FirecrawlError(
+              `Crawl job failed or was stopped. Status: ${statusData.status}`,
+              500
+            );
+          }
         } else {
-          throw new FirecrawlError(
-            `Crawl job failed or was stopped. Status: ${statusData.status}`,
-            500
-          );
+          this.handleError(statusResponse, "check crawl status");
         }
-      } else {
-        this.handleError(statusResponse, "check crawl status");
       }
+    } catch (error: any) {
+      throw new FirecrawlError(error, 500);
     }
   }