config.ts.example added

.gitignore

@@ -16,3 +16,7 @@ storage
 *.json
 .env
 pnpm-lock.yaml
+
+# exclude working files
+/gpt-crawler-y-upstream_project_summary.*
+/config.ts

.tool-versions

@@ -0,0 +1 @@
+nodejs 20.10.0

config.ts

@@ -4,8 +4,8 @@ export const defaultConfig: Config = {
   // url: "https://www.builder.io/c/docs/developers",
   // match: "https://www.builder.io/c/docs/**",
   url: "https://help.mypurecloud.com/articles/about-cx-cloud-from-genesys-and-salesforce/",
-  match: "https://help.mypurecloud.com/?p**",
-  maxPagesToCrawl: 100,
+  match: ["https://help.mypurecloud.com/?p**","https://help.salesforce.com/**"],
+  maxPagesToCrawl: 300,
   // outputFileName: "output.json",
   outputFileName: "cxcloud.json",
   waitTime: 1000,
@@ -17,5 +17,4 @@ export const defaultConfig: Config = {
   // userAgent: "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   maxTokens: 2000000,
   selector: "#main-content > div",
-  crawlInsideSelector: true,
 };

config.ts.example

@@ -0,0 +1,17 @@
+import { Config } from "./src/config";
+
+export const defaultConfig: Config = {
+  url: "https://www.builder.io/c/docs/developers",
+  match: "https://www.builder.io/c/docs/**",
+  maxPagesToCrawl: 300,
+  outputFileName: "output.json",
+  waitTime: 1000,
+  onVisitPage: async ({ visitPageWaitTime }) => {
+    await new Promise((resolve) =>
+      setTimeout(resolve, visitPageWaitTime ?? 1000),
+    );
+  },
+  // userAgent: "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
+  maxTokens: 2000000,
+  selector: "#main-content > div",
+};