Skip to content

E2E matrix

E2E matrix #1364

Workflow file for this run

name: E2E matrix
on:
schedule:
- cron: "0 5 * * *"
workflow_dispatch:
inputs:
debug_enabled:
type: boolean
description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
required: false
default: false
env:
CYPRESS_CACHE_FOLDER: ${{ github.workspace }}/.cypress
permissions: { }
jobs:
preinstall:
if: ${{ github.repository_owner == 'nrwl' }}
runs-on: ${{ matrix.os }}
strategy:
matrix:
os:
- ubuntu-latest
# - macos-latest
node_version:
- 20
# - 18
# exclude:
# run just node v20 on macos
# - os: macos-latest
# node_version: 18
name: Cache install (${{ matrix.os }}, node v${{ matrix.node_version }})
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Install PNPM
run: |
npm install -g @pnpm/[email protected]
- name: Set node
uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node_version }}
cache: 'pnpm'
- name: Cache node_modules
id: cache-modules
uses: actions/cache@v3
with:
lookup-only: true
path: '**/node_modules'
key: ${{ runner.os }}-modules-${{ matrix.node_version }}-${{ github.run_id }}
- name: Ensure Python setuptools Installed on Macos
if: ${{ matrix.os == 'macos-latest' }}
id: brew-install-python-setuptools
run: brew install python-setuptools
- name: Install packages
if: steps.cache-modules.outputs.cache-hit != 'true'
run: pnpm install --frozen-lockfile
- name: Homebrew cache directory path
if: ${{ matrix.os == 'macos-latest' }}
id: homebrew-cache-dir-path
run: echo "dir=$(brew --cache)" >> $GITHUB_OUTPUT
- name: Cache Homebrew
if: ${{ matrix.os == 'macos-latest' }}
uses: actions/cache@v3
with:
lookup-only: true
path: ${{ steps.homebrew-cache-dir-path.outputs.dir }}
key: brew-${{ matrix.node_version }}
restore-keys: |
brew-
- name: Cache Cypress
id: cache-cypress
uses: actions/cache@v3
with:
lookup-only: true
path: '${{ github.workspace }}/.cypress'
key: ${{ runner.os }}-cypress
- name: Install Cypress
if: steps.cache-cypress.outputs.cache-hit != 'true'
run: npx cypress install
e2e:
if: ${{ github.repository_owner == 'nrwl' }}
needs: preinstall
permissions:
contents: read
runs-on: ${{ matrix.os }}
timeout-minutes: 90
strategy:
matrix:
os:
- ubuntu-latest
# - macos-latest
node_version:
- 20
# - 18
package_manager:
- npm
- yarn
- pnpm
project:
- e2e-angular
# - e2e-cypress
# - e2e-detox
# - e2e-esbuild
# - e2e-eslint
# - e2e-expo
# - e2e-gradle
# - e2e-jest
# - e2e-js
# - e2e-lerna-smoke-tests
# - e2e-next
# - e2e-node
# - e2e-nuxt
# - e2e-nx-init
# - e2e-nx
# - e2e-playwright
# - e2e-plugin
# - e2e-react
# - e2e-react-native
# - e2e-release
# - e2e-remix
# - e2e-rollup
# - e2e-storybook
# - e2e-vite
# - e2e-vue
# - e2e-web
# - e2e-webpack
# - e2e-workspace-create
# include:
# # os short names
# - os: ubuntu-latest
# os_name: 'Linux'
# - os: macos-latest
# os_name: 'MacOS'
# # test timeouts
# - os: ubuntu-latest
# os_timeout: 60
# - os: macos-latest
# os_timeout: 90
# # codeowner groups
# - project: e2e-angular
# codeowners: 'S04SS457V38'
# - project: e2e-cypress
# codeowners: 'S04T16BTJJY'
# - project: e2e-detox
# codeowners: 'S04TNCNJG5N'
# - project: e2e-esbuild
# codeowners: 'S04SJ6HHP0X'
# - project: e2e-expo
# codeowners: 'S04TNCNJG5N'
# - project: e2e-gradle
# codeowners: 'S04TNCNJG5N'
# - project: e2e-jest
# codeowners: 'S04T16BTJJY'
# - project: e2e-js
# codeowners: 'S04SJ6HHP0X'
# - project: e2e-lerna-smoke-tests
# codeowners: 'S04TNCVEETS'
# - project: e2e-eslint
# codeowners: 'S04SYJGKSCT'
# - project: e2e-next
# codeowners: 'S04TNCNJG5N'
# - project: e2e-node
# codeowners: 'S04SJ6HHP0X'
# - project: e2e-nx-init
# codeowners: 'S04SYHYKGNP'
# - project: e2e-nx
# codeowners: 'S04SYHYKGNP'
# - project: e2e-plugin
# codeowners: 'S04SYHYKGNP'
# - project: e2e-release
# codeowners: 'S04SYHYKGNP'
# - project: e2e-react
# codeowners: 'S04TNCNJG5N'
# - project: e2e-react-native
# codeowners: 'S04TNCNJG5N'
# - project: e2e-web
# codeowners: 'S04SJ6PL98X'
# - project: e2e-rollup
# codeowners: 'S04SJ6PL98X'
# - project: e2e-storybook
# codeowners: 'S04SVQ8H0G5'
# - project: e2e-playwright
# codeowners: 'S04SVQ8H0G5'
# - project: e2e-remix
# codeowners: 'S04SVQ8H0G5'
# - project: e2e-vite
# codeowners: 'S04SJ6PL98X'
# - project: e2e-vue
# codeowners: 'S04SJ6PL98X'
# - project: e2e-nuxt
# codeowners: 'S04SJ6PL98X'
# - project: e2e-webpack
# codeowners: 'S04SJ6PL98X'
# - project: e2e-workspace-create
# codeowners: 'S04SYHYKGNP'
# exclude:
# # exclude react-native tests from ubuntu
# - os: ubuntu-latest
# project: e2e-react-native
# - os: ubuntu-latest
# project: e2e-detox
# - os: ubuntu-latest
# project: e2e-expo
# # exclude non-CNW/Lerna tests from non-LTS node versions
# - node_version: 18
# project: e2e-angular
# - node_version: 18
# project: e2e-cypress
# - node_version: 18
# project: e2e-detox
# - node_version: 18
# project: e2e-esbuild
# - node_version: 18
# project: e2e-expo
# - node_version: 18
# project: e2e-gradle
# - node_version: 18
# project: e2e-jest
# - node_version: 18
# project: e2e-js
# - node_version: 18
# project: e2e-eslint
# - node_version: 18
# project: e2e-next
# - node_version: 18
# project: e2e-node
# - node_version: 18
# project: e2e-nuxt
# - node_version: 18
# project: e2e-nx-init
# - node_version: 18
# project: e2e-nx
# - node_version: 18
# project: e2e-plugin
# - node_version: 18
# project: e2e-playwright
# - node_version: 18
# project: e2e-react
# - node_version: 18
# project: e2e-react-native
# - node_version: 18
# project: e2e-web
# - node_version: 18
# project: e2e-remix
# - node_version: 18
# project: e2e-rollup
# - node_version: 18
# project: e2e-storybook
# - node_version: 18
# project: e2e-vite
# - node_version: 18
# project: e2e-vue
# - node_version: 18
# project: e2e-webpack
# # run just npm v20 on macos
# - os: macos-latest
# package_manager: yarn
# - os: macos-latest
# package_manager: pnpm
# - os: macos-latest
# node_version: 18
fail-fast: false
name: ${{ matrix.os_name }}/${{ matrix.package_manager }}/${{ matrix.node_version }} ${{ join(matrix.project) }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Prepare dir for output
run: mkdir -p outputs
- name: Install PNPM
run: |
npm install -g @pnpm/[email protected]
- name: Use Node.js ${{ matrix.node_version }}
uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node_version }}
cache: 'pnpm'
- name: Cache node_modules
id: cache-modules
uses: actions/cache@v3
with:
path: '**/node_modules'
key: ${{ runner.os }}-modules-${{ matrix.node_version }}-${{ github.run_id }}
- name: Install packages
run: pnpm install --frozen-lockfile
- name: Cleanup
if: ${{ matrix.os == 'ubuntu-latest' }}
run: |
# Workaround to provide additional free space for testing.
# https://github.com/actions/virtual-environments/issues/2840
sudo rm -rf /usr/share/dotnet
sudo rm -rf /opt/ghc
sudo rm -rf "/usr/local/share/boost"
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
sudo apt-get install lsof
echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p
- name: Homebrew cache directory path
if: ${{ matrix.os == 'macos-latest' }}
id: homebrew-cache-dir-path
run: echo "dir=$(brew --cache)" >> $GITHUB_OUTPUT
- name: Cache Homebrew
if: ${{ matrix.os == 'macos-latest' }}
uses: actions/cache@v3
with:
path: ${{ steps.homebrew-cache-dir-path.outputs.dir }}
key: brew-${{ matrix.node_version }}
restore-keys: |
brew-
- name: Cache Cypress
id: cache-cypress
uses: actions/cache@v3
with:
path: '${{ github.workspace }}/.cypress'
key: ${{ runner.os }}-cypress
- name: Install Cypress
if: steps.cache-cypress.outputs.cache-hit != 'true'
run: npx cypress install
- name: Install applesimutils, reset ios simulators
if: ${{ matrix.os == 'macos-latest' }}
run: |
HOMEBREW_NO_AUTO_UPDATE=1 brew tap wix/brew >/dev/null
HOMEBREW_NO_AUTO_UPDATE=1 brew install applesimutils >/dev/null
xcrun simctl shutdown all && xcrun simctl erase all
- name: Configure git metadata (needed for lerna smoke tests)
run: |
git config --global user.email [email protected]
git config --global user.name "Test Test"
- name: Set starting timestamp
id: before-e2e
run: |
echo "timestamp=$(date +%s)" >> $GITHUB_OUTPUT
- name: Run e2e tests
id: e2e-run
run: pnpm nx run-many -t e2e-local -p ${{ matrix.project }}
timeout-minutes: ${{ matrix.os_timeout }}
env:
GIT_AUTHOR_EMAIL: [email protected]
GIT_AUTHOR_NAME: Test
GIT_COMMITTER_EMAIL: [email protected]
GIT_COMMITTER_NAME: Test
NX_E2E_CI_CACHE_KEY: e2e-gha-${{ matrix.os }}-${{ matrix.node_version }}-${{ matrix.package_manager }}
NODE_OPTIONS: --max_old_space_size=8192
SELECTED_PM: ${{ matrix.package_manager }}
npm_config_registry: http://localhost:4872
YARN_REGISTRY: http://localhost:4872
NX_CACHE_DIRECTORY: 'tmp'
NX_E2E_SKIP_CLEANUP: 'true'
NX_E2E_RUN_E2E: 'true'
NX_E2E_VERBOSE_LOGGING: 'true'
NX_PERF_LOGGING: 'false'
NX_DAEMON: 'true'
NX_SKIP_LOG_GROUPING: 'true'
- name: Save matrix config in file
if: ${{ always() }}
id: save-matrix
shell: bash
run: |
before=${{ steps.before-e2e.outputs.timestamp }}
now=$(date +%s)
delta=$(($now - $before))
matrix=$((
echo '${{ toJSON(matrix) }}'
) | jq --argjson delta $delta -c '. + { "status": "${{ steps.e2e-run.outcome}}", "duration": $delta }')
echo "$matrix" > matrix
path=outputs/${{ matrix.os_name}}-${{ matrix.node_version}}-${{ matrix.package_manager}}-${{ matrix.project }}
echo "path=$path" >> $GITHUB_OUTPUT
echo "$matrix" > $path
- name: Upload matrix config
uses: actions/upload-artifact@v3
if: ${{ always() }}
with:
name: outputs
path: ${{ steps.save-matrix.outputs.path }}
- name: Setup tmate session
if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled && failure() }}
uses: mxschmitt/[email protected]
timeout-minutes: 15
# process-result:
# if: ${{ always() && github.repository_owner == 'nrwl' }}
# runs-on: ubuntu-latest
# needs: e2e
# outputs:
# message: ${{ steps.process-json.outputs.SLACK_MESSAGE }}
# proj-duration: ${{ steps.process-json.outputs.SLACK_PROJ_DURATION }}
# pm-duration: ${{ steps.process-json.outputs.SLACK_PM_DURATION }}
# codeowners: ${{ steps.process-json.outputs.CODEOWNERS }}
# steps:
# - name: Load outputs
# uses: actions/download-artifact@v3
# with:
# name: outputs
# path: outputs
# - name: Join and stringify matrix configs
# id: combine-json
# run: |
# combined=$((jq -s . outputs/*) | jq tostring)
# echo "combined=$combined" >> $GITHUB_OUTPUT
# - name: Make slack outputs
# id: process-json
# uses: actions/github-script@v6
# env:
# GITHUB_TOKEN: ${{ github.token }}
# with:
# script: |
# const combined = JSON.parse(${{ steps.combine-json.outputs.combined }});
# const failedProjects = combined.filter(c => c.status === 'failure').sort((a, b) => a.project.localeCompare(b.project));
# // codeowners
# const codeowners = new Set();
# failedProjects.forEach(c => {
# codeowners.add(c.codeowners);
# });
# core.setOutput('CODEOWNERS', Array.from(codeowners).join(','));
# function trimSpace(res) {
# return res.split('\n').map((l) => l.trim()).join('\n');
# }
# // failed message
# let lastProject;
# let result = `
# \`\`\`
# | Failed project | PM | OS | Node |
# |--------------------------------|------|-------|------|`;
# failedProjects.forEach(matrix => {
# const project = matrix.project !== lastProject ? matrix.project : '...';
# result += `\n| ${project.padEnd(30)} | ${matrix.package_manager.padEnd(4)} | ${matrix.os_name} | v${matrix.node_version} |`
# lastProject = matrix.project;
# });
# result += `\`\`\``;
# core.setOutput('SLACK_MESSAGE', trimSpace(result));
# function humanizeDuration(num) {
# let res = '';
# const hours = Math.floor(num / 3600);
# if (hours) {
# res += `${hours}h `;
# }
# const mins = Math.floor((num % 3600) / 60);
# if (mins) {
# res += `${mins}m `;
# }
# const sec = num % 60;
# if (sec) {
# res += `${sec}s`
# }
# return res;
# }
# // duration message
# const timeReport = {};
# const pmReport = {
# npm: 0,
# yarn: 0,
# pnpm: 0
# };
# const macosProjects = ['e2e-detox', 'e2e-expo', 'e2e-react-native'];
# combined.forEach((matrix) => {
# if (matrix.os_name === 'Linux' && matrix.node_version === 18) {
# pmReport[matrix.package_manager] += matrix.duration;
# }
# if (matrix.os_name === 'Linux' || macosProjects.includes(matrix.project)) {
# if (timeReport[matrix.project]) {
# if (matrix.duration > timeReport[matrix.project].max) {
# timeReport[matrix.project].max = matrix.duration;
# timeReport[
# matrix.project
# ].maxEnv = `${matrix.os_name}, ${matrix.package_manager}`;
# }
# if (matrix.duration < timeReport[matrix.project].min) {
# timeReport[matrix.project].min = matrix.duration;
# timeReport[
# matrix.project
# ].minEnv = `${matrix.os_name}, ${matrix.package_manager}`;
# }
# } else {
# timeReport[matrix.project] = {
# min: matrix.duration,
# max: matrix.duration,
# minEnv: `${matrix.os_name}, ${matrix.package_manager}`,
# maxEnv: `${matrix.os_name}, ${matrix.package_manager}`,
# };
# }
# }
# });
# // project time report
# let resultPkg = `
# \`\`\`
# | Project | Time |
# |--------------------------------|---------------------------|`;
# function mapProjectTime(proj, section) {
# let res = '';
# res += `${humanizeDuration(timeReport[proj][section])}`;
# res += ` (${timeReport[proj][section + 'Env']})`
# return res;
# }
# function durationIcon(proj, section) {
# if (timeReport[proj][section] < 12 * 60) {
# return `${section} ✅`;
# }
# if (timeReport[proj][section] < 15 * 60) {
# return `${section} ❗`;
# }
# return `${section} ❌`;
# }
# Object.keys(timeReport).forEach(proj => {
# resultPkg += `\n| ${proj.padEnd(30)} | |`;
# resultPkg += `\n| ${durationIcon(proj, 'min').padStart(29)} | ${mapProjectTime(proj, 'min').padEnd(25)} |`;
# resultPkg += `\n| ${durationIcon(proj, 'max').padStart(29)} | ${mapProjectTime(proj, 'max').padEnd(25)} |`;
# });
# resultPkg += `\`\`\``;
# core.setOutput('SLACK_PROJ_DURATION', trimSpace(resultPkg));
# // Print project duration report inline to allow reviewing on manual runs (when no slack message will be sent)
# console.log(trimSpace(resultPkg));
# let resultPm = `
# \`\`\`
# | PM | Total time |
# |------|-------------|`;
# Object.keys(pmReport).forEach(pm => {
# resultPm += `\n| ${pm.padEnd(4)} | ${humanizeDuration(pmReport[pm]).padEnd(11)} |`
# });
# resultPm += `\`\`\``;
# core.setOutput('SLACK_PM_DURATION', trimSpace(resultPm));
# // Print package manager duration report inline to allow reviewing on manual runs (when no slack message will be sent)
# console.log(trimSpace(resultPm));
# report-failure:
# if: ${{ failure() && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }}
# needs: process-result
# runs-on: ubuntu-latest
# name: Report failure
# steps:
# - name: Send notification
# uses: ravsamhq/notify-slack-action@v2
# with:
# status: 'failure'
# message_format: '{emoji} Workflow has {status_message} ${{ needs.process-result.outputs.message }}'
# notification_title: '{workflow}'
# footer: '<{run_url}|View Run> / Last commit <{commit_url}|{commit_sha}>'
# mention_groups: ${{ needs.process-result.outputs.codeowners }}
# env:
# SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
# report-success:
# if: ${{ success() && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }}
# needs: e2e
# runs-on: ubuntu-latest
# name: Report status
# steps:
# - name: Send notification
# uses: ravsamhq/notify-slack-action@v2
# with:
# status: ${{ needs.e2e.result }}
# message_format: '{emoji} Workflow has {status_message}'
# notification_title: '{workflow}'
# footer: '<{run_url}|View Run> / Last commit <{commit_url}|{commit_sha}>'
# env:
# SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
# report-pm-time:
# if: ${{ always() && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }}
# needs: process-result
# runs-on: ubuntu-latest
# name: Report duration per package manager
# steps:
# - name: Send notification
# uses: ravsamhq/notify-slack-action@v2
# with:
# status: 'skipped'
# message_format: '${{ needs.process-result.outputs.pm-duration }}'
# notification_title: 'Total duration per package manager (ubuntu only)'
# env:
# SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
# report-proj-time:
# if: ${{ always() && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }}
# needs: process-result
# runs-on: ubuntu-latest
# name: Report duration per package manager
# steps:
# - name: Send notification
# uses: ravsamhq/notify-slack-action@v2
# with:
# status: 'skipped'
# message_format: '${{ needs.process-result.outputs.proj-duration }}'
# notification_title: 'E2E Project duration stats'
# env:
# SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}