AG-10005 - Improve accuracy of Nx cache hashes. #3820
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: CI | |
on: | |
push: | |
branches: [latest] | |
pull_request: | |
branches: [latest] | |
workflow_dispatch: | |
inputs: | |
clean_checkout: | |
description: 'Disable all caching' | |
type: 'choice' | |
required: true | |
default: 'false' | |
options: | |
- 'true' | |
- 'false' | |
nx_command: | |
type: 'choice' | |
required: true | |
default: 'affected' | |
options: | |
- 'affected' | |
- 'run-many' | |
env: | |
NX_REJECT_UNKNOWN_LOCAL_CACHE: 0 | |
NX_NO_CLOUD: true | |
NX_CLOUD_ACCESS_TOKEN: x | |
NX_BRANCH: ${{ github.ref }} | |
concurrency: | |
group: ${{ github.workflow }}-${{ github.ref }} | |
cancel-in-progress: ${{ github.ref != 'refs/heads/latest' }} | |
jobs: | |
execute_blt: | |
runs-on: ubuntu-latest | |
outputs: | |
nx_base: ${{ steps.nx_config.outputs.base }} | |
build: ${{ steps.build.outcome || '' }} | |
lint: ${{ steps.lint.outcome || '' }} | |
format: ${{ steps.format.outcome || '' }} | |
test_count: ${{ steps.test_count.outputs.test_count }} | |
steps: | |
- id: init_format | |
name: Update Status (format) | |
uses: LouisBrunner/[email protected] | |
with: | |
token: ${{ secrets.GITHUB_TOKEN }} | |
name: format | |
status: queued | |
- id: init_lint | |
name: Update Status (lint) | |
uses: LouisBrunner/[email protected] | |
with: | |
token: ${{ secrets.GITHUB_TOKEN }} | |
name: lint | |
status: queued | |
- id: init_build | |
name: Update Status (build) | |
uses: LouisBrunner/[email protected] | |
with: | |
token: ${{ secrets.GITHUB_TOKEN }} | |
name: build | |
status: queued | |
- name: Checkout | |
id: checkout | |
uses: actions/checkout@v3 | |
with: | |
fetch-depth: 0 | |
- name: Fetch Refs | |
run: git fetch origin latest-success latest | |
- name: Nx Affected SHA Setup | |
id: nx_config | |
run: | | |
if [[ $(git branch --show-current) == 'latest' ]] ; then | |
echo "base=latest-success" >> $GITHUB_OUTPUT | |
echo "NX_BASE=latest-success" >> $GITHUB_ENV | |
else | |
echo "base=origin/latest" >> $GITHUB_OUTPUT | |
echo "NX_BASE=origin/latest" >> $GITHUB_ENV | |
fi | |
git fetch origin latest-success latest | |
- name: Cache node_modules | |
if: github.event.inputs.clean_checkout != 'true' | |
id: cache | |
uses: actions/cache@v3 | |
with: | |
path: | | |
node_modules/ | |
packages/*/node_modules/ | |
plugins/*/node_modules/ | |
libraries/*/node_modules/ | |
key: node_modules-${{hashFiles('yarn.lock','patches/*.patch', 'packages/*/package.json', 'plugins/*/package.json')}} | |
restore-keys: node_modules- # Take any latest cache if failed to find it for current yarn.lock | |
- name: Cache Nx | |
if: github.event.inputs.clean_checkout != 'true' | |
uses: actions/cache@v3 | |
with: | |
path: .nx/cache | |
key: cache-nx-v17-${{ hashFiles('yarn.lock','patches/*.patch', 'packages/*/package.json', 'plugins/*/package.json') }}-${{ github.sha }} | |
restore-keys: cache-nx-v17-${{ hashFiles('yarn.lock','patches/*.patch', 'packages/*/package.json', 'plugins/*/package.json') }}- | |
- name: Setup Node.js | |
id: setup_node | |
uses: actions/setup-node@v4 | |
with: | |
node-version: '18' | |
cache: ${{ github.event.inputs.clean_checkout != 'true' && steps.cache.outputs.cache-hit != 'true' && 'yarn' || '' }} | |
- name: yarn install | |
if: steps.cache.outputs.cache-hit != 'true' | |
id: yarn_install | |
run: (yarn check --integrity && yarn postinstall) || yarn install --ci --prefer-offline | |
- name: yarn postinstall | |
if: steps.cache.outputs.cache-hit == 'true' | |
run: yarn postinstall | |
- name: nx format:check | |
id: format | |
if: steps.yarn_install.outcome == 'success' || steps.yarn_install.outcome == 'skipped' | |
run: | | |
if [[ "${{ github.event.inputs.nx_command || 'affected' }}" == "run-many" ]] ; then | |
yarn nx format:check --all | |
else | |
yarn nx format:check --base ${{ steps.nx_config.outputs.base }} | |
fi | |
- name: Update Status (format) | |
uses: LouisBrunner/[email protected] | |
if: always() | |
with: | |
token: ${{ secrets.GITHUB_TOKEN }} | |
check_id: ${{ steps.init_format.outputs.check_id }} | |
conclusion: ${{ steps.format.outcome }} | |
- name: nx lint | |
id: lint | |
if: steps.yarn_install.outcome == 'success' || steps.yarn_install.outcome == 'skipped' | |
run: yarn nx ${{ github.event.inputs.nx_command || 'affected' }} -t lint --parallel=3 --exclude all | |
- name: Update Status (lint) | |
uses: LouisBrunner/[email protected] | |
if: always() | |
with: | |
token: ${{ secrets.GITHUB_TOKEN }} | |
check_id: ${{ steps.init_lint.outputs.check_id }} | |
conclusion: ${{ steps.lint.outcome }} | |
- name: nx build | |
id: build | |
if: steps.yarn_install.outcome == 'success' || steps.yarn_install.outcome == 'skipped' | |
run: yarn nx ${{ github.event.inputs.nx_command || 'affected' }} -t build --parallel=3 --exclude all | |
- name: Update Status (build) | |
uses: LouisBrunner/[email protected] | |
if: always() | |
with: | |
token: ${{ secrets.GITHUB_TOKEN }} | |
check_id: ${{ steps.init_build.outputs.check_id }} | |
conclusion: ${{ steps.build.outcome }} | |
- name: Count test targets to run | |
id: test_count | |
run: | | |
if [[ "${{ github.event.inputs.nx_command || 'affected' }}" == "run-many" ]] ; then | |
echo "test_count=1000" >> $GITHUB_OUTPUT | |
else | |
echo "test_count=$(yarn -s nx show projects --affected --base ${{ steps.nx_config.outputs.base }} -t test | wc -l)" >> $GITHUB_OUTPUT | |
fi | |
test: | |
runs-on: ubuntu-latest | |
needs: execute_blt | |
if: needs.execute_blt.outputs.test_count > 0 | |
strategy: | |
matrix: | |
shard: [1, 2, 3, 4] | |
name: test (${{ matrix.shard }}/${{ strategy.job-total }}) | |
env: | |
NX_PARALLEL: 1 | |
NX_BASE: ${{ needs.execute_blt.outputs.nx_base }} | |
steps: | |
- name: Checkout | |
id: checkout | |
uses: actions/checkout@v3 | |
with: | |
fetch-depth: 0 | |
- name: Cache node_modules | |
if: github.event.inputs.clean_checkout != 'true' | |
id: cache | |
uses: actions/cache@v3 | |
with: | |
path: | | |
node_modules/ | |
packages/*/node_modules/ | |
plugins/*/node_modules/ | |
libraries/*/node_modules/ | |
key: node_modules-${{hashFiles('yarn.lock','patches/*.patch', 'packages/*/package.json', 'plugins/*/package.json')}} | |
restore-keys: node_modules- # Take any latest cache if failed to find it for current yarn.lock | |
- name: Cache Nx | |
if: github.event.inputs.clean_checkout != 'true' | |
uses: actions/cache@v3 | |
with: | |
path: .nx/cache | |
key: cache-nx-v17-${{ hashFiles('yarn.lock','patches/*.patch', 'packages/*/package.json', 'plugins/*/package.json') }}-${{ github.sha }} | |
restore-keys: cache-nx-v17-${{ hashFiles('yarn.lock','patches/*.patch', 'packages/*/package.json', 'plugins/*/package.json') }}- | |
- name: Setup Node.js | |
id: setup_node | |
uses: actions/setup-node@v4 | |
with: | |
node-version: '18' | |
cache: ${{ github.event.inputs.clean_checkout != 'true' && steps.cache.outputs.cache-hit != 'true' && 'yarn' || '' }} | |
- name: yarn install | |
if: steps.cache.outputs.cache-hit != 'true' | |
id: yarn_install | |
run: (yarn check --integrity && yarn postinstall) || yarn install --ci --prefer-offline | |
- name: yarn postinstall | |
if: steps.cache.outputs.cache-hit == 'true' | |
run: yarn postinstall | |
- name: nx test | |
id: test | |
run: yarn nx ${{ github.event.inputs.nx_command || 'affected' }} -t test --configuration=ci --exclude all --shard=${{ matrix.shard }}/${{ strategy.job-total }} | |
- name: Perist test results | |
if: always() | |
uses: actions/upload-artifact@v3 | |
with: | |
name: test-results-${{matrix.shard}} | |
path: | | |
reports/ | |
packages/**/__diff_output__/* | |
report: | |
runs-on: ubuntu-latest | |
needs: [execute_blt, test] | |
if: cancelled() != true | |
steps: | |
- name: Checkout | |
id: checkout | |
uses: actions/checkout@v3 | |
with: | |
fetch-depth: 0 | |
- name: Fetch Refs | |
run: git fetch origin latest-success latest | |
- uses: actions/download-artifact@v3 | |
with: | |
path: test-results/ | |
- name: Merge JUnit Report XMLs | |
run: | | |
yarn global add junit-report-merger | |
reports=$(find test-results/ -name \*.xml -type f -exec basename \{\} \; | sort | uniq) | |
mkdir -p reports/ | |
echo "$reports" | (while read name ; do | |
yarn exec -s jrm reports/${name} "test-results/**/${name}" | |
done) | |
- name: Test Report | |
uses: dorny/test-reporter@v1 | |
if: needs.test.result == 'success' || needs.test.result == 'failure' | |
id: testReport | |
continue-on-error: true | |
with: | |
name: 'Tests Results' | |
path: reports/*.xml | |
reporter: jest-junit | |
- name: Check last job status | |
id: lastJobStatus | |
if: always() && github.ref == 'refs/heads/latest' | |
run: | | |
WORKFLOW_STATUS="success" | |
if [[ "${{ needs.execute_blt.result }}" == "failure" ]] ; then | |
WORKFLOW_STATUS="failure" | |
elif [ "${{ needs.test.result }}" == "failure" ] ; then | |
WORKFLOW_STATUS="failure" | |
fi | |
echo "workflowStatus=${WORKFLOW_STATUS}" >> $GITHUB_OUTPUT | |
LAST_WORKFLOW_STATUS=$(gh run list --workflow .github/workflows/ci.yml -b latest | grep -oh "completed.*" | grep -v "cancelled" | head -1 | awk '{print $2}') | |
if [ "$GITHUB_RUN_ATTEMPT" -ge 2 ]; then | |
# Handle re-run cases - there is no way to query the previous run status, so we assume the most | |
# common scenario will be re-run after failure. | |
LAST_WORKFLOW_STATUS="failure" | |
fi | |
if [ "$LAST_WORKFLOW_STATUS" != "$WORKFLOW_STATUS" ]; then | |
echo "status changed from $LAST_WORKFLOW_STATUS to $WORKFLOW_STATUS" | |
echo "changedState=true" >> $GITHUB_OUTPUT | |
else | |
echo "status is still $WORKFLOW_STATUS" | |
echo "changedState=false" >> $GITHUB_OUTPUT | |
fi | |
env: | |
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
- name: Commit History | |
id: commits | |
if: always() && job.status != 'cancelled' && github.ref == 'refs/heads/latest' && steps.lastJobStatus.outputs.changedState == 'true' | |
run: | | |
GIT_LOG=$(git log HEAD ^${{ needs.execute_blt.outputs.nx_base }} --format="%an (%h) %s") | |
echo "GIT_LOG<<EOF" >> $GITHUB_ENV | |
echo "$GIT_LOG" >> $GITHUB_ENV | |
echo "EOF" >> $GITHUB_ENV | |
- name: Tag Latest Successful Commit | |
if: success() && github.ref == 'refs/heads/latest' && steps.lastJobStatus.outputs.workflowStatus == 'success' | |
uses: EndBug/latest-tag@latest | |
with: | |
ref: latest-success | |
description: Latest commit to pass GitHub Actions workflow on latest branch. | |
- name: Slack Notification | |
uses: rtCamp/action-slack-notify@v2 | |
if: always() && job.status != 'cancelled' && github.ref == 'refs/heads/latest' && steps.lastJobStatus.outputs.changedState == 'true' | |
env: | |
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} | |
SLACK_COLOR: ${{ steps.lastJobStatus.outputs.workflowStatus }} | |
SLACK_ICON: https://avatars.slack-edge.com/2020-11-25/1527503386626_319578f21381f9641cd8_192.png | |
SLACK_USERNAME: ag-charts CI | |
SLACK_FOOTER: '' | |
SLACK_MESSAGE: > | |
Build: ${{ needs.execute_blt.outputs.build == 'success' && '✅' || needs.execute_blt.outputs.build == 'failure' && '❌' || 'NA' }} | |
Lint: ${{ needs.execute_blt.outputs.lint == 'success' && '✅' || needs.execute_blt.outputs.lint == 'failure' && '❌' || 'NA' }} | |
Format: ${{ needs.execute_blt.outputs.format == 'success' && '✅' || needs.execute_blt.outputs.format == 'failure' && '❌' || 'NA' }} | |
Test: ${{ needs.test.result == 'success' && '✅' || needs.test.result == 'failure' && '❌' || 'NA' }} | |
*Changes:* | |
${{ env.GIT_LOG }} |