Scrape project board items (experimental) (#514)

ohcnetwork · Sep 9, 2024 · 805e922 · 805e922
1 parent 1489657
commit 805e922
Show file tree

Hide file tree

Showing 5 changed files with 383 additions and 170 deletions.
diff --git a/.github/workflows/scraper-dry-run.yaml b/.github/workflows/scraper-dry-run.yaml
@@ -37,26 +37,27 @@ jobs:
         working-directory: scraper
 
       - name: Scrape data from GitHub
-        run: pnpm start ${{ github.repository_owner }} ../data/github 
+        run: pnpm start ${{ github.repository_owner }} ../data/github
         working-directory: scraper
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.SCRAPER_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
+          PROJECTS_BOARD_ID: PVT_kwDOA7JD8804yA
 
       - run: mkdir contributors
 
       - name: Generate markdown files for new contributors
         run: node scripts/generateNewContributors.js
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          
+
       - uses: actions/upload-artifact@v4
         with:
           name: output
           retention-days: 5
           path: |
             data
             contributors
-            
+
       - name: Get pnpm store directory
         shell: bash
         run: |

diff --git a/.github/workflows/scraper.yaml b/.github/workflows/scraper.yaml
@@ -9,7 +9,14 @@ on:
         type: string
         default: main
         description: A specific branch of scraper that is to be used.
+      projects-board-id:
+        required: false
+        type: string
+        description: Experimental; Works only with particular project board structure. Requires `read:project` scope for the SCRAPER_GITHUB_TOKEN. Disabled if not configured.
     secrets:
+      SCRAPER_GITHUB_TOKEN:
+        required: false
+        description: Allows overriding the default GITHUB_TOKEN for providing scopes beyond what GITHUB_TOKEN can offer.
       GIT_ACCESS_TOKEN:
         required: true
       SLACK_API_TOKEN:
@@ -47,7 +54,8 @@ jobs:
         run: pnpm start ${{ github.repository_owner }} ../../data/github
         working-directory: scraper
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.SCRAPER_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
+          PROJECTS_BOARD_ID: ${{ inputs.projects-board-id }}
 
       - uses: actions/setup-python@v3
         with: