diff --git a/.github/workflows/scraper-dry-run.yaml b/.github/workflows/scraper-dry-run.yaml index 2ff353f2..a8e6e8e1 100644 --- a/.github/workflows/scraper-dry-run.yaml +++ b/.github/workflows/scraper-dry-run.yaml @@ -37,10 +37,11 @@ jobs: working-directory: scraper - name: Scrape data from GitHub - run: pnpm start ${{ github.repository_owner }} ../data/github + run: pnpm start ${{ github.repository_owner }} ../data/github working-directory: scraper env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.SCRAPER_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + PROJECTS_BOARD_ID: PVT_kwDOA7JD8804yA - run: mkdir contributors @@ -48,7 +49,7 @@ jobs: run: node scripts/generateNewContributors.js env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - + - uses: actions/upload-artifact@v4 with: name: output @@ -56,7 +57,7 @@ jobs: path: | data contributors - + - name: Get pnpm store directory shell: bash run: | diff --git a/.github/workflows/scraper.yaml b/.github/workflows/scraper.yaml index 2a334939..98130785 100644 --- a/.github/workflows/scraper.yaml +++ b/.github/workflows/scraper.yaml @@ -9,7 +9,14 @@ on: type: string default: main description: A specific branch of scraper that is to be used. + projects-board-id: + required: false + type: string + description: Experimental; Works only with particular project board structure. Requires `read:project` scope for the SCRAPER_GITHUB_TOKEN. Disabled if not configured. secrets: + SCRAPER_GITHUB_TOKEN: + required: false + description: Allows overriding the default GITHUB_TOKEN for providing scopes beyond what GITHUB_TOKEN can offer. GIT_ACCESS_TOKEN: required: true SLACK_API_TOKEN: @@ -47,7 +54,8 @@ jobs: run: pnpm start ${{ github.repository_owner }} ../../data/github working-directory: scraper env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.SCRAPER_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + PROJECTS_BOARD_ID: ${{ inputs.projects-board-id }} - uses: actions/setup-python@v3 with: diff --git a/scraper/pnpm-lock.yaml b/scraper/pnpm-lock.yaml index 52c99eac..8abc1bae 100644 --- a/scraper/pnpm-lock.yaml +++ b/scraper/pnpm-lock.yaml @@ -1,33 +1,182 @@ -lockfileVersion: '6.0' +lockfileVersion: '9.0' settings: autoInstallPeers: true excludeLinksFromLockfile: false -dependencies: - date-fns: - specifier: ^3.6.0 - version: 3.6.0 - dotenv: - specifier: ^16.4.5 - version: 16.4.5 - octokit: - specifier: ^4.0.2 - version: 4.0.2 - -devDependencies: - '@types/node': - specifier: ^20.14.12 - version: 20.14.12 - typescript: - specifier: ^4.9.5 - version: 4.9.5 +importers: + + .: + dependencies: + date-fns: + specifier: ^3.6.0 + version: 3.6.0 + dotenv: + specifier: ^16.4.5 + version: 16.4.5 + octokit: + specifier: ^4.0.2 + version: 4.0.2 + devDependencies: + '@types/node': + specifier: ^20.14.12 + version: 20.14.12 + typescript: + specifier: ^4.9.5 + version: 4.9.5 packages: - /@octokit/app@15.1.0: + '@octokit/app@15.1.0': resolution: {integrity: sha512-TkBr7QgOmE6ORxvIAhDbZsqPkF7RSqTY4pLTtUQCvr6dTXqvi2fFo46q3h1lxlk/sGMQjqyZ0kEahkD/NyzOHg==} engines: {node: '>= 18'} + + '@octokit/auth-app@7.1.0': + resolution: {integrity: sha512-cazGaJPSgeZ8NkVYeM/C5l/6IQ5vZnsI8p1aMucadCkt/bndI+q+VqwrlnWbASRmenjOkf1t1RpCKrif53U8gw==} + engines: {node: '>= 18'} + + '@octokit/auth-oauth-app@8.1.1': + resolution: {integrity: sha512-5UtmxXAvU2wfcHIPPDWzVSAWXVJzG3NWsxb7zCFplCWEmMCArSZV0UQu5jw5goLQXbFyOr5onzEH37UJB3zQQg==} + engines: {node: '>= 18'} + + '@octokit/auth-oauth-device@7.1.1': + resolution: {integrity: sha512-HWl8lYueHonuyjrKKIup/1tiy0xcmQCdq5ikvMO1YwkNNkxb6DXfrPjrMYItNLyCP/o2H87WuijuE+SlBTT8eg==} + engines: {node: '>= 18'} + + '@octokit/auth-oauth-user@5.1.1': + resolution: {integrity: sha512-rRkMz0ErOppdvEfnemHJXgZ9vTPhBuC6yASeFaB7I2yLMd7QpjfrL1mnvRPlyKo+M6eeLxrKanXJ9Qte29SRsw==} + engines: {node: '>= 18'} + + '@octokit/auth-token@5.1.1': + resolution: {integrity: sha512-rh3G3wDO8J9wSjfI436JUKzHIxq8NaiL0tVeB2aXmG6p/9859aUOAjA9pmSPNGGZxfwmaJ9ozOJImuNVJdpvbA==} + engines: {node: '>= 18'} + + '@octokit/auth-unauthenticated@6.1.0': + resolution: {integrity: sha512-zPSmfrUAcspZH/lOFQnVnvjQZsIvmfApQH6GzJrkIunDooU1Su2qt2FfMTSVPRp7WLTQyC20Kd55lF+mIYaohQ==} + engines: {node: '>= 18'} + + '@octokit/core@6.1.2': + resolution: {integrity: sha512-hEb7Ma4cGJGEUNOAVmyfdB/3WirWMg5hDuNFVejGEDFqupeOysLc2sG6HJxY2etBp5YQu5Wtxwi020jS9xlUwg==} + engines: {node: '>= 18'} + + '@octokit/endpoint@10.1.1': + resolution: {integrity: sha512-JYjh5rMOwXMJyUpj028cu0Gbp7qe/ihxfJMLc8VZBMMqSwLgOxDI1911gV4Enl1QSavAQNJcwmwBF9M0VvLh6Q==} + engines: {node: '>= 18'} + + '@octokit/graphql@8.1.1': + resolution: {integrity: sha512-ukiRmuHTi6ebQx/HFRCXKbDlOh/7xEV6QUXaE7MJEKGNAncGI/STSbOkl12qVXZrfZdpXctx5O9X1AIaebiDBg==} + engines: {node: '>= 18'} + + '@octokit/oauth-app@7.1.3': + resolution: {integrity: sha512-EHXbOpBkSGVVGF1W+NLMmsnSsJRkcrnVmDKt0TQYRBb6xWfWzoi9sBD4DIqZ8jGhOWO/V8t4fqFyJ4vDQDn9bg==} + engines: {node: '>= 18'} + + '@octokit/oauth-authorization-url@7.1.1': + resolution: {integrity: sha512-ooXV8GBSabSWyhLUowlMIVd9l1s2nsOGQdlP2SQ4LnkEsGXzeCvbSbCPdZThXhEFzleGPwbapT0Sb+YhXRyjCA==} + engines: {node: '>= 18'} + + '@octokit/oauth-methods@5.1.2': + resolution: {integrity: sha512-C5lglRD+sBlbrhCUTxgJAFjWgJlmTx5bQ7Ch0+2uqRjYv7Cfb5xpX4WuSC9UgQna3sqRGBL9EImX9PvTpMaQ7g==} + engines: {node: '>= 18'} + + '@octokit/openapi-types@22.2.0': + resolution: {integrity: sha512-QBhVjcUa9W7Wwhm6DBFu6ZZ+1/t/oYxqc2tp81Pi41YNuJinbFRx8B133qVOrAaBbF7D/m0Et6f9/pZt9Rc+tg==} + + '@octokit/openapi-webhooks-types@8.3.0': + resolution: {integrity: sha512-vKLsoR4xQxg4Z+6rU/F65ItTUz/EXbD+j/d4mlq2GW8TsA4Tc8Kdma2JTAAJ5hrKWUQzkR/Esn2fjsqiVRYaQg==} + + '@octokit/plugin-paginate-graphql@5.2.2': + resolution: {integrity: sha512-7znSVvlNAOJisCqAnjN1FtEziweOHSjPGAuc5W58NeGNAr/ZB57yCsjQbXDlWsVryA7hHQaEQPcBbJYFawlkyg==} + engines: {node: '>= 18'} + peerDependencies: + '@octokit/core': '>=6' + + '@octokit/plugin-paginate-rest@11.3.3': + resolution: {integrity: sha512-o4WRoOJZlKqEEgj+i9CpcmnByvtzoUYC6I8PD2SA95M+BJ2x8h7oLcVOg9qcowWXBOdcTRsMZiwvM3EyLm9AfA==} + engines: {node: '>= 18'} + peerDependencies: + '@octokit/core': '>=6' + + '@octokit/plugin-rest-endpoint-methods@13.2.4': + resolution: {integrity: sha512-gusyAVgTrPiuXOdfqOySMDztQHv6928PQ3E4dqVGEtOvRXAKRbJR4b1zQyniIT9waqaWk/UDaoJ2dyPr7Bk7Iw==} + engines: {node: '>= 18'} + peerDependencies: + '@octokit/core': '>=6' + + '@octokit/plugin-retry@7.1.1': + resolution: {integrity: sha512-G9Ue+x2odcb8E1XIPhaFBnTTIrrUDfXN05iFXiqhR+SeeeDMMILcAnysOsxUpEWcQp2e5Ft397FCXTcPkiPkLw==} + engines: {node: '>= 18'} + peerDependencies: + '@octokit/core': '>=6' + + '@octokit/plugin-throttling@9.3.1': + resolution: {integrity: sha512-Qd91H4liUBhwLB2h6jZ99bsxoQdhgPk6TdwnClPyTBSDAdviGPceViEgUwj+pcQDmB/rfAXAXK7MTochpHM3yQ==} + engines: {node: '>= 18'} + peerDependencies: + '@octokit/core': ^6.0.0 + + '@octokit/request-error@6.1.4': + resolution: {integrity: sha512-VpAhIUxwhWZQImo/dWAN/NpPqqojR6PSLgLYAituLM6U+ddx9hCioFGwBr5Mi+oi5CLeJkcAs3gJ0PYYzU6wUg==} + engines: {node: '>= 18'} + + '@octokit/request@9.1.3': + resolution: {integrity: sha512-V+TFhu5fdF3K58rs1pGUJIDH5RZLbZm5BI+MNF+6o/ssFNT4vWlCh/tVpF3NxGtP15HUxTTMUbsG5llAuU2CZA==} + engines: {node: '>= 18'} + + '@octokit/types@13.5.0': + resolution: {integrity: sha512-HdqWTf5Z3qwDVlzCrP8UJquMwunpDiMPt5er+QjGzL4hqr/vBVY/MauQgS1xWxCDT1oMx1EULyqxncdCY/NVSQ==} + + '@octokit/webhooks-methods@5.1.0': + resolution: {integrity: sha512-yFZa3UH11VIxYnnoOYCVoJ3q4ChuSOk2IVBBQ0O3xtKX4x9bmKb/1t+Mxixv2iUhzMdOl1qeWJqEhouXXzB3rQ==} + engines: {node: '>= 18'} + + '@octokit/webhooks@13.3.0': + resolution: {integrity: sha512-TUkJLtI163Bz5+JK0O+zDkQpn4gKwN+BovclUvCj6pI/6RXrFqQvUMRS2M+Rt8Rv0qR3wjoMoOPmpJKeOh0nBg==} + engines: {node: '>= 18'} + + '@types/aws-lambda@8.10.142': + resolution: {integrity: sha512-wy2y/2hQKrS6myOS++koXg3N1Hg+LLyPjaggCFajczSHZPqBnOMuT2sdH3kiASrmdBYyM3pmjyz5SoWraRllCQ==} + + '@types/node@20.14.12': + resolution: {integrity: sha512-r7wNXakLeSsGT0H1AU863vS2wa5wBOK4bWMjZz2wj+8nBx+m5PeIn0k8AloSLpRuiwdRQZwarZqHE4FNArPuJQ==} + + before-after-hook@3.0.2: + resolution: {integrity: sha512-Nik3Sc0ncrMK4UUdXQmAnRtzmNQTAAXmXIopizwZ1W1t8QmfJj+zL4OA2I7XPTPW5z5TDqv4hRo/JzouDJnX3A==} + + bottleneck@2.19.5: + resolution: {integrity: sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw==} + + date-fns@3.6.0: + resolution: {integrity: sha512-fRHTG8g/Gif+kSh50gaGEdToemgfj74aRX3swtiouboip5JDLAyDE9F11nHMIcvOaXeOC6D7SpNhi7uFyB7Uww==} + + dotenv@16.4.5: + resolution: {integrity: sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==} + engines: {node: '>=12'} + + lru-cache@10.4.3: + resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} + + octokit@4.0.2: + resolution: {integrity: sha512-wbqF4uc1YbcldtiBFfkSnquHtECEIpYD78YUXI6ri1Im5OO2NLo6ZVpRdbJpdnpZ05zMrVPssNiEo6JQtea+Qg==} + engines: {node: '>= 18'} + + typescript@4.9.5: + resolution: {integrity: sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==} + engines: {node: '>=4.2.0'} + hasBin: true + + undici-types@5.26.5: + resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} + + universal-github-app-jwt@2.2.0: + resolution: {integrity: sha512-G5o6f95b5BggDGuUfKDApKaCgNYy2x7OdHY0zSMF081O0EJobw+1130VONhrA7ezGSV2FNOGyM+KQpQZAr9bIQ==} + + universal-user-agent@7.0.2: + resolution: {integrity: sha512-0JCqzSKnStlRRQfCdowvqy3cy0Dvtlb8xecj/H8JFZuCze4rwjPZQOgvFvn0Ws/usCHQFGpyr+pB9adaGwXn4Q==} + +snapshots: + + '@octokit/app@15.1.0': dependencies: '@octokit/auth-app': 7.1.0 '@octokit/auth-unauthenticated': 6.1.0 @@ -36,11 +185,8 @@ packages: '@octokit/plugin-paginate-rest': 11.3.3(@octokit/core@6.1.2) '@octokit/types': 13.5.0 '@octokit/webhooks': 13.3.0 - dev: false - /@octokit/auth-app@7.1.0: - resolution: {integrity: sha512-cazGaJPSgeZ8NkVYeM/C5l/6IQ5vZnsI8p1aMucadCkt/bndI+q+VqwrlnWbASRmenjOkf1t1RpCKrif53U8gw==} - engines: {node: '>= 18'} + '@octokit/auth-app@7.1.0': dependencies: '@octokit/auth-oauth-app': 8.1.1 '@octokit/auth-oauth-user': 5.1.1 @@ -50,56 +196,38 @@ packages: lru-cache: 10.4.3 universal-github-app-jwt: 2.2.0 universal-user-agent: 7.0.2 - dev: false - /@octokit/auth-oauth-app@8.1.1: - resolution: {integrity: sha512-5UtmxXAvU2wfcHIPPDWzVSAWXVJzG3NWsxb7zCFplCWEmMCArSZV0UQu5jw5goLQXbFyOr5onzEH37UJB3zQQg==} - engines: {node: '>= 18'} + '@octokit/auth-oauth-app@8.1.1': dependencies: '@octokit/auth-oauth-device': 7.1.1 '@octokit/auth-oauth-user': 5.1.1 '@octokit/request': 9.1.3 '@octokit/types': 13.5.0 universal-user-agent: 7.0.2 - dev: false - /@octokit/auth-oauth-device@7.1.1: - resolution: {integrity: sha512-HWl8lYueHonuyjrKKIup/1tiy0xcmQCdq5ikvMO1YwkNNkxb6DXfrPjrMYItNLyCP/o2H87WuijuE+SlBTT8eg==} - engines: {node: '>= 18'} + '@octokit/auth-oauth-device@7.1.1': dependencies: '@octokit/oauth-methods': 5.1.2 '@octokit/request': 9.1.3 '@octokit/types': 13.5.0 universal-user-agent: 7.0.2 - dev: false - /@octokit/auth-oauth-user@5.1.1: - resolution: {integrity: sha512-rRkMz0ErOppdvEfnemHJXgZ9vTPhBuC6yASeFaB7I2yLMd7QpjfrL1mnvRPlyKo+M6eeLxrKanXJ9Qte29SRsw==} - engines: {node: '>= 18'} + '@octokit/auth-oauth-user@5.1.1': dependencies: '@octokit/auth-oauth-device': 7.1.1 '@octokit/oauth-methods': 5.1.2 '@octokit/request': 9.1.3 '@octokit/types': 13.5.0 universal-user-agent: 7.0.2 - dev: false - /@octokit/auth-token@5.1.1: - resolution: {integrity: sha512-rh3G3wDO8J9wSjfI436JUKzHIxq8NaiL0tVeB2aXmG6p/9859aUOAjA9pmSPNGGZxfwmaJ9ozOJImuNVJdpvbA==} - engines: {node: '>= 18'} - dev: false + '@octokit/auth-token@5.1.1': {} - /@octokit/auth-unauthenticated@6.1.0: - resolution: {integrity: sha512-zPSmfrUAcspZH/lOFQnVnvjQZsIvmfApQH6GzJrkIunDooU1Su2qt2FfMTSVPRp7WLTQyC20Kd55lF+mIYaohQ==} - engines: {node: '>= 18'} + '@octokit/auth-unauthenticated@6.1.0': dependencies: '@octokit/request-error': 6.1.4 '@octokit/types': 13.5.0 - dev: false - /@octokit/core@6.1.2: - resolution: {integrity: sha512-hEb7Ma4cGJGEUNOAVmyfdB/3WirWMg5hDuNFVejGEDFqupeOysLc2sG6HJxY2etBp5YQu5Wtxwi020jS9xlUwg==} - engines: {node: '>= 18'} + '@octokit/core@6.1.2': dependencies: '@octokit/auth-token': 5.1.1 '@octokit/graphql': 8.1.1 @@ -108,28 +236,19 @@ packages: '@octokit/types': 13.5.0 before-after-hook: 3.0.2 universal-user-agent: 7.0.2 - dev: false - /@octokit/endpoint@10.1.1: - resolution: {integrity: sha512-JYjh5rMOwXMJyUpj028cu0Gbp7qe/ihxfJMLc8VZBMMqSwLgOxDI1911gV4Enl1QSavAQNJcwmwBF9M0VvLh6Q==} - engines: {node: '>= 18'} + '@octokit/endpoint@10.1.1': dependencies: '@octokit/types': 13.5.0 universal-user-agent: 7.0.2 - dev: false - /@octokit/graphql@8.1.1: - resolution: {integrity: sha512-ukiRmuHTi6ebQx/HFRCXKbDlOh/7xEV6QUXaE7MJEKGNAncGI/STSbOkl12qVXZrfZdpXctx5O9X1AIaebiDBg==} - engines: {node: '>= 18'} + '@octokit/graphql@8.1.1': dependencies: '@octokit/request': 9.1.3 '@octokit/types': 13.5.0 universal-user-agent: 7.0.2 - dev: false - /@octokit/oauth-app@7.1.3: - resolution: {integrity: sha512-EHXbOpBkSGVVGF1W+NLMmsnSsJRkcrnVmDKt0TQYRBb6xWfWzoi9sBD4DIqZ8jGhOWO/V8t4fqFyJ4vDQDn9bg==} - engines: {node: '>= 18'} + '@octokit/oauth-app@7.1.3': dependencies: '@octokit/auth-oauth-app': 8.1.1 '@octokit/auth-oauth-user': 5.1.1 @@ -139,154 +258,87 @@ packages: '@octokit/oauth-methods': 5.1.2 '@types/aws-lambda': 8.10.142 universal-user-agent: 7.0.2 - dev: false - /@octokit/oauth-authorization-url@7.1.1: - resolution: {integrity: sha512-ooXV8GBSabSWyhLUowlMIVd9l1s2nsOGQdlP2SQ4LnkEsGXzeCvbSbCPdZThXhEFzleGPwbapT0Sb+YhXRyjCA==} - engines: {node: '>= 18'} - dev: false + '@octokit/oauth-authorization-url@7.1.1': {} - /@octokit/oauth-methods@5.1.2: - resolution: {integrity: sha512-C5lglRD+sBlbrhCUTxgJAFjWgJlmTx5bQ7Ch0+2uqRjYv7Cfb5xpX4WuSC9UgQna3sqRGBL9EImX9PvTpMaQ7g==} - engines: {node: '>= 18'} + '@octokit/oauth-methods@5.1.2': dependencies: '@octokit/oauth-authorization-url': 7.1.1 '@octokit/request': 9.1.3 '@octokit/request-error': 6.1.4 '@octokit/types': 13.5.0 - dev: false - /@octokit/openapi-types@22.2.0: - resolution: {integrity: sha512-QBhVjcUa9W7Wwhm6DBFu6ZZ+1/t/oYxqc2tp81Pi41YNuJinbFRx8B133qVOrAaBbF7D/m0Et6f9/pZt9Rc+tg==} - dev: false + '@octokit/openapi-types@22.2.0': {} - /@octokit/openapi-webhooks-types@8.3.0: - resolution: {integrity: sha512-vKLsoR4xQxg4Z+6rU/F65ItTUz/EXbD+j/d4mlq2GW8TsA4Tc8Kdma2JTAAJ5hrKWUQzkR/Esn2fjsqiVRYaQg==} - dev: false + '@octokit/openapi-webhooks-types@8.3.0': {} - /@octokit/plugin-paginate-graphql@5.2.2(@octokit/core@6.1.2): - resolution: {integrity: sha512-7znSVvlNAOJisCqAnjN1FtEziweOHSjPGAuc5W58NeGNAr/ZB57yCsjQbXDlWsVryA7hHQaEQPcBbJYFawlkyg==} - engines: {node: '>= 18'} - peerDependencies: - '@octokit/core': '>=6' + '@octokit/plugin-paginate-graphql@5.2.2(@octokit/core@6.1.2)': dependencies: '@octokit/core': 6.1.2 - dev: false - /@octokit/plugin-paginate-rest@11.3.3(@octokit/core@6.1.2): - resolution: {integrity: sha512-o4WRoOJZlKqEEgj+i9CpcmnByvtzoUYC6I8PD2SA95M+BJ2x8h7oLcVOg9qcowWXBOdcTRsMZiwvM3EyLm9AfA==} - engines: {node: '>= 18'} - peerDependencies: - '@octokit/core': '>=6' + '@octokit/plugin-paginate-rest@11.3.3(@octokit/core@6.1.2)': dependencies: '@octokit/core': 6.1.2 '@octokit/types': 13.5.0 - dev: false - /@octokit/plugin-rest-endpoint-methods@13.2.4(@octokit/core@6.1.2): - resolution: {integrity: sha512-gusyAVgTrPiuXOdfqOySMDztQHv6928PQ3E4dqVGEtOvRXAKRbJR4b1zQyniIT9waqaWk/UDaoJ2dyPr7Bk7Iw==} - engines: {node: '>= 18'} - peerDependencies: - '@octokit/core': '>=6' + '@octokit/plugin-rest-endpoint-methods@13.2.4(@octokit/core@6.1.2)': dependencies: '@octokit/core': 6.1.2 '@octokit/types': 13.5.0 - dev: false - /@octokit/plugin-retry@7.1.1(@octokit/core@6.1.2): - resolution: {integrity: sha512-G9Ue+x2odcb8E1XIPhaFBnTTIrrUDfXN05iFXiqhR+SeeeDMMILcAnysOsxUpEWcQp2e5Ft397FCXTcPkiPkLw==} - engines: {node: '>= 18'} - peerDependencies: - '@octokit/core': '>=6' + '@octokit/plugin-retry@7.1.1(@octokit/core@6.1.2)': dependencies: '@octokit/core': 6.1.2 '@octokit/request-error': 6.1.4 '@octokit/types': 13.5.0 bottleneck: 2.19.5 - dev: false - /@octokit/plugin-throttling@9.3.1(@octokit/core@6.1.2): - resolution: {integrity: sha512-Qd91H4liUBhwLB2h6jZ99bsxoQdhgPk6TdwnClPyTBSDAdviGPceViEgUwj+pcQDmB/rfAXAXK7MTochpHM3yQ==} - engines: {node: '>= 18'} - peerDependencies: - '@octokit/core': ^6.0.0 + '@octokit/plugin-throttling@9.3.1(@octokit/core@6.1.2)': dependencies: '@octokit/core': 6.1.2 '@octokit/types': 13.5.0 bottleneck: 2.19.5 - dev: false - /@octokit/request-error@6.1.4: - resolution: {integrity: sha512-VpAhIUxwhWZQImo/dWAN/NpPqqojR6PSLgLYAituLM6U+ddx9hCioFGwBr5Mi+oi5CLeJkcAs3gJ0PYYzU6wUg==} - engines: {node: '>= 18'} + '@octokit/request-error@6.1.4': dependencies: '@octokit/types': 13.5.0 - dev: false - /@octokit/request@9.1.3: - resolution: {integrity: sha512-V+TFhu5fdF3K58rs1pGUJIDH5RZLbZm5BI+MNF+6o/ssFNT4vWlCh/tVpF3NxGtP15HUxTTMUbsG5llAuU2CZA==} - engines: {node: '>= 18'} + '@octokit/request@9.1.3': dependencies: '@octokit/endpoint': 10.1.1 '@octokit/request-error': 6.1.4 '@octokit/types': 13.5.0 universal-user-agent: 7.0.2 - dev: false - /@octokit/types@13.5.0: - resolution: {integrity: sha512-HdqWTf5Z3qwDVlzCrP8UJquMwunpDiMPt5er+QjGzL4hqr/vBVY/MauQgS1xWxCDT1oMx1EULyqxncdCY/NVSQ==} + '@octokit/types@13.5.0': dependencies: '@octokit/openapi-types': 22.2.0 - dev: false - /@octokit/webhooks-methods@5.1.0: - resolution: {integrity: sha512-yFZa3UH11VIxYnnoOYCVoJ3q4ChuSOk2IVBBQ0O3xtKX4x9bmKb/1t+Mxixv2iUhzMdOl1qeWJqEhouXXzB3rQ==} - engines: {node: '>= 18'} - dev: false + '@octokit/webhooks-methods@5.1.0': {} - /@octokit/webhooks@13.3.0: - resolution: {integrity: sha512-TUkJLtI163Bz5+JK0O+zDkQpn4gKwN+BovclUvCj6pI/6RXrFqQvUMRS2M+Rt8Rv0qR3wjoMoOPmpJKeOh0nBg==} - engines: {node: '>= 18'} + '@octokit/webhooks@13.3.0': dependencies: '@octokit/openapi-webhooks-types': 8.3.0 '@octokit/request-error': 6.1.4 '@octokit/webhooks-methods': 5.1.0 - dev: false - /@types/aws-lambda@8.10.142: - resolution: {integrity: sha512-wy2y/2hQKrS6myOS++koXg3N1Hg+LLyPjaggCFajczSHZPqBnOMuT2sdH3kiASrmdBYyM3pmjyz5SoWraRllCQ==} - dev: false + '@types/aws-lambda@8.10.142': {} - /@types/node@20.14.12: - resolution: {integrity: sha512-r7wNXakLeSsGT0H1AU863vS2wa5wBOK4bWMjZz2wj+8nBx+m5PeIn0k8AloSLpRuiwdRQZwarZqHE4FNArPuJQ==} + '@types/node@20.14.12': dependencies: undici-types: 5.26.5 - dev: true - /before-after-hook@3.0.2: - resolution: {integrity: sha512-Nik3Sc0ncrMK4UUdXQmAnRtzmNQTAAXmXIopizwZ1W1t8QmfJj+zL4OA2I7XPTPW5z5TDqv4hRo/JzouDJnX3A==} - dev: false + before-after-hook@3.0.2: {} - /bottleneck@2.19.5: - resolution: {integrity: sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw==} - dev: false + bottleneck@2.19.5: {} - /date-fns@3.6.0: - resolution: {integrity: sha512-fRHTG8g/Gif+kSh50gaGEdToemgfj74aRX3swtiouboip5JDLAyDE9F11nHMIcvOaXeOC6D7SpNhi7uFyB7Uww==} - dev: false + date-fns@3.6.0: {} - /dotenv@16.4.5: - resolution: {integrity: sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==} - engines: {node: '>=12'} - dev: false + dotenv@16.4.5: {} - /lru-cache@10.4.3: - resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} - dev: false + lru-cache@10.4.3: {} - /octokit@4.0.2: - resolution: {integrity: sha512-wbqF4uc1YbcldtiBFfkSnquHtECEIpYD78YUXI6ri1Im5OO2NLo6ZVpRdbJpdnpZ05zMrVPssNiEo6JQtea+Qg==} - engines: {node: '>= 18'} + octokit@4.0.2: dependencies: '@octokit/app': 15.1.0 '@octokit/core': 6.1.2 @@ -298,22 +350,11 @@ packages: '@octokit/plugin-throttling': 9.3.1(@octokit/core@6.1.2) '@octokit/request-error': 6.1.4 '@octokit/types': 13.5.0 - dev: false - /typescript@4.9.5: - resolution: {integrity: sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==} - engines: {node: '>=4.2.0'} - hasBin: true - dev: true + typescript@4.9.5: {} - /undici-types@5.26.5: - resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} - dev: true + undici-types@5.26.5: {} - /universal-github-app-jwt@2.2.0: - resolution: {integrity: sha512-G5o6f95b5BggDGuUfKDApKaCgNYy2x7OdHY0zSMF081O0EJobw+1130VONhrA7ezGSV2FNOGyM+KQpQZAr9bIQ==} - dev: false + universal-github-app-jwt@2.2.0: {} - /universal-user-agent@7.0.2: - resolution: {integrity: sha512-0JCqzSKnStlRRQfCdowvqy3cy0Dvtlb8xecj/H8JFZuCze4rwjPZQOgvFvn0Ws/usCHQFGpyr+pB9adaGwXn4Q==} - dev: false + universal-user-agent@7.0.2: {} diff --git a/scraper/src/github-scraper/index.ts b/scraper/src/github-scraper/index.ts index 1db519a3..e8ae8274 100644 --- a/scraper/src/github-scraper/index.ts +++ b/scraper/src/github-scraper/index.ts @@ -1,10 +1,11 @@ -import { formatISO, parseISO, startOfDay, subDays } from "date-fns"; +import { formatISO, parseISO, subDays } from "date-fns"; import { fetchMergeEvents, fetchOpenPulls } from "./fetchUserData.js"; import { IGitHubEvent, ProcessData } from "./types.js"; import { fetchEvents } from "./fetchEvents.js"; import { parseEvents } from "./parseEvents.js"; import { mergedData } from "./saveData.js"; import { scrapeDiscussions } from "./discussion.js"; +import scrapeProjectBoardItems from "./projectItems.js"; let processedData: ProcessData = {}; @@ -78,6 +79,10 @@ const main = async () => { await mergedData(dataDir, processedData); await scrapeDiscussions(orgName, dataDir, endDate, startDate); + if (process.env.PROJECTS_BOARD_ID) { + await scrapeProjectBoardItems(process.env.PROJECTS_BOARD_ID, dataDir); + } + console.log("Done"); }; diff --git a/scraper/src/github-scraper/projectItems.ts b/scraper/src/github-scraper/projectItems.ts new file mode 100644 index 00000000..5a709fb7 --- /dev/null +++ b/scraper/src/github-scraper/projectItems.ts @@ -0,0 +1,158 @@ +/** + * This is experimental. Needs to be made configurable as this may not work + * for pulling items from project board's of other orgs. + * + * Notes: + * - SCRAPER_GITHUB_TOKEN requires `read:project` scope. + * - Status change events can't be extracted atm. using GraphQL API, however should be in future. (Ref: Ref: https://github.com/orgs/community/discussions/5859#discussioncomment-4679381) + */ + +import { mkdir, readFile, writeFile } from "fs/promises"; +import { octokit } from "./config.js"; +import path from "path"; + +async function getProjectBoardItems(projectId: string) { + const data: any = await octokit.graphql( + `query getProjectItems($projectId: ID!) { + node(id: $projectId) { + ... on ProjectV2 { + updatedAt + items(first: 100, orderBy: {field: POSITION, direction: DESC}) { + nodes { + id + createdAt + updatedAt + fieldValues(first: 10) { + nodes { + ... on ProjectV2ItemFieldTextValue { + text + field { + ... on ProjectV2FieldCommon { + name + } + } + } + ... on ProjectV2ItemFieldIterationValue { + title + startDate + duration + field { + ... on ProjectV2FieldCommon { + name + } + } + } + ... on ProjectV2ItemFieldNumberValue { + number + field { + ... on ProjectV2FieldCommon { + name + } + } + } + ... on ProjectV2ItemFieldSingleSelectValue { + name + description + color + field { + ... on ProjectV2FieldCommon { + name + } + } + } + } + } + content { + ... on DraftIssue { + assignees(last: 10) { + nodes { + login + } + } + } + ... on Issue { + url + closedAt + author { + login + } + assignees(last: 10) { + nodes { + login + } + } + } + ... on PullRequest { + url + closedAt + author { + login + } + assignees(last: 10) { + nodes { + login + } + } + } + } + } + } + } + } + }`, + { projectId }, + ); + + return Object.fromEntries( + data.node.items.nodes.map((node: any) => { + const get = (fieldName: string) => { + return node.fieldValues.nodes.find( + (fieldValue: any) => fieldValue.field?.name === fieldName, + ); + }; + + return [ + node.id, + { + url: node.content.url, + title: get("Title").text, + createdAt: node.createdAt, + updatedAt: node.updatedAt, + closedAt: node.content.closedAt, + sprint: get("Sprint")?.title, + category: get("Category")?.name, + status: get("Status")?.name, + storyPoints: get("Story Points")?.number, + priority: get("Priority")?.name, + author: node.content.author?.login, + assignees: node.content.assignees.nodes.map( + (user: any) => user.login, + ), + focus: get("Focus")?.name, + }, + ]; + }), + ); +} + +async function readExistingItems(filePath: string): Promise { + try { + const contents = await readFile(filePath); + return JSON.parse(contents.toString()); + } catch (e) {} + return {}; +} + +async function upsertItems(items: object, filePath: string) { + const existing = await readExistingItems(filePath); + await writeFile(filePath, JSON.stringify({ ...existing, ...items }, null, 2)); +} + +export default async function scrapeProjectBoardItems( + projectId: string, + rootDir: string, +) { + await mkdir(path.join(rootDir, "project-boards"), { recursive: true }); + const file = path.join(rootDir, "project-boards", `${projectId}.json`); + await upsertItems(await getProjectBoardItems(projectId), file); +}