added js/ts sdk tests

.github/workflows/fly.yml

@@ -94,6 +94,22 @@ jobs:
         run: |
           npm run test
         working-directory: ./apps/test-suite
+      - name: Set up Python ${{ matrix.python-version }}  
+        uses: actions/setup-python@v4  
+        with:  
+          python-version: ${{ matrix.python-version }}
+      - name: Install Python dependencies  
+        run: |  
+          python -m pip install --upgrade pip  
+          pip install -r requirements.txt
+        working-directory: ./apps/python-sdk
+      - name: Run E2E tests for Python SDK
+        run: |  
+          pytest firecrawl/__tests__/e2e_withAuth/test.py 
+        working-directory: ./apps/python-sdk
+      - name: Run E2E tests for JavaScript SDK
+        run: npm run tests
+        working-directory: ./apps/js-sdk/firecrawl
 
   deploy:
     name: Deploy app

.github/workflows/js-sdk.yml

@@ -0,0 +1,57 @@
+name: Run JavaScript SDK E2E Tests
+
+on:
+  pull_request:
+    branches:
+      - main
+env:
+  ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+  BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}
+  FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
+  HOST: ${{ secrets.HOST }}
+  LLAMAPARSE_API_KEY: ${{ secrets.LLAMAPARSE_API_KEY }}
+  LOGTAIL_KEY: ${{ secrets.LOGTAIL_KEY }}
+  POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}
+  POSTHOG_HOST: ${{ secrets.POSTHOG_HOST }}
+  NUM_WORKERS_PER_QUEUE: ${{ secrets.NUM_WORKERS_PER_QUEUE }}
+  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+  PLAYWRIGHT_MICROSERVICE_URL: ${{ secrets.PLAYWRIGHT_MICROSERVICE_URL }}
+  PORT: ${{ secrets.PORT }}
+  REDIS_URL: ${{ secrets.REDIS_URL }}
+  SCRAPING_BEE_API_KEY: ${{ secrets.SCRAPING_BEE_API_KEY }}
+  SUPABASE_ANON_TOKEN: ${{ secrets.SUPABASE_ANON_TOKEN }}
+  SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }}
+  SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
+  TEST_API_KEY: ${{ secrets.TEST_API_KEY }}
+  HYPERDX_API_KEY: ${{ secrets.HYPERDX_API_KEY }}
+  HDX_NODE_BETA_MODE: 1
+
+jobs:  
+  build:  
+    runs-on: ubuntu-latest  
+    services:
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: "20"
+      - name: Install pnpm
+        run: npm install -g pnpm
+      - name: Install dependencies for API
+        run: pnpm install
+        working-directory: ./apps/api
+      - name: Start the application
+        run: npm start &
+        working-directory: ./apps/api
+      - name: Start workers
+        run: npm run workers &
+        working-directory: ./apps/api
+      - name: Run E2E tests for JavaScript SDK
+        run: npm run tests
+        working-directory: ./apps/js-sdk/firecrawl

.github/workflows/python-tests.yml → .github/workflows/python-sdk.yml

@@ -61,12 +61,12 @@ jobs:
         uses: actions/setup-python@v4  
         with:  
           python-version: ${{ matrix.python-version }}
-      - name: Install dependencies  
+      - name: Install Python dependencies  
         run: |  
           python -m pip install --upgrade pip  
           pip install -r requirements.txt
         working-directory: ./apps/python-sdk
-      - name: Test with pytest  
+      - name: Run E2E tests for Python SDK  
         run: |  
           pytest firecrawl/__tests__/e2e_withAuth/test.py 
         working-directory: ./apps/python-sdk

apps/js-sdk/firecrawl/src/__tests__/e2e_withAuth/index.test.ts

@@ -61,7 +61,6 @@ describe('FirecrawlApp E2E Tests', () => {
   test('should return successful response for valid scrape with PDF file without explicit extension', async () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
     const response = await app.scrapeUrl('https://arxiv.org/pdf/astro-ph/9301001');
-    await new Promise(resolve => setTimeout(resolve, 6000)); // wait for 6 seconds
     expect(response).not.toBeNull();
     expect(response.data.content).toContain('We present spectrophotometric observations of the Broad Line Radio Galaxy');
   }, 30000); // 30 seconds timeout