fix(scrapeURL): crawl stuff

mendableai · Nov 5, 2024 · 9144dba · 9144dba
1 parent 8b69ccb
commit 9144dba
Show file tree

Hide file tree

Showing 5 changed files with 9 additions and 2 deletions.
diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.ts
@@ -178,9 +178,10 @@ export async function crawlController(req: Request, res: Response) {
             url,
             mode: "single_urls",
             crawlerOptions,
+            scrapeOptions,
+            internalOptions,
             team_id,
             plan,
-            pageOptions: pageOptions,
             origin: req.body.origin ?? defaultOrigin,
             crawl_id: id,
             sitemapped: true,
@@ -215,6 +216,7 @@ export async function crawlController(req: Request, res: Response) {
         {
           url,
           mode: "single_urls",
+          crawlerOptions,
           scrapeOptions,
           internalOptions,
           team_id,

diff --git a/apps/api/src/controllers/v0/crawlPreview.ts b/apps/api/src/controllers/v0/crawlPreview.ts
@@ -115,6 +115,7 @@ export async function crawlPreviewController(req: Request, res: Response) {
           mode: "single_urls",
           team_id,
           plan: plan!,
+          crawlerOptions,
           scrapeOptions,
           internalOptions,
           origin: "website-preview",
@@ -131,6 +132,7 @@ export async function crawlPreviewController(req: Request, res: Response) {
         mode: "single_urls",
         team_id,
         plan: plan!,
+        crawlerOptions,
         scrapeOptions,
         internalOptions,
         origin: "website-preview",

diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.ts
@@ -42,6 +42,7 @@ export async function crawlController(
 
   const crawlerOptions = {
     ...req.body,
+    url: undefined,
     scrapeOptions: undefined,
   };
   const scrapeOptions = req.body.scrapeOptions;
@@ -147,6 +148,7 @@ export async function crawlController(
         url: req.body.url,
         mode: "single_urls",
         team_id: req.auth.team_id,
+        crawlerOptions,
         scrapeOptions: scrapeOptionsSchema.parse(scrapeOptions),
         plan: req.auth.plan!,
         origin: "api",

diff --git a/apps/api/src/lib/logger.ts b/apps/api/src/lib/logger.ts
@@ -45,7 +45,7 @@ export const logger = winston.createLogger({
       format: winston.format.combine(
         winston.format.timestamp({ format: "YYYY-MM-DD HH:mm:ss" }),
         winston.format.metadata({ fillExcept: ["message", "level", "timestamp"] }),
-        ...(process.env.ENVIRONMENT === "production" && process.env.SENTRY_ENVIRONMENT !== "dev" ? [winston.format.colorize(), logFormat] : []),
+        ...(((process.env.ENV === "production" && process.env.SENTRY_ENVIRONMENT === "dev") || (process.env.ENV !== "production")) ? [winston.format.colorize(), logFormat] : []),
       ),
     }),
   ],

diff --git a/apps/api/src/types.ts b/apps/api/src/types.ts
@@ -25,6 +25,7 @@ export interface IngestResult {
 export interface WebScraperOptions {
   url: string;
   mode: Mode;
+  crawlerOptions?: any;
   scrapeOptions: ScrapeOptions;
   internalOptions?: InternalOptions;
   team_id: string;