test.yaml

name: Tests
on:
  pull_request:
    branches: [ "master", "release/**" ]
  push:
    # we trigger runs on master branch, but we do not run spread on master 
    # branch, the master branch runs are just for unit tests + codecov.io
    branches: [ "master","release/**" ]

concurrency:
  group: ${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

jobs:
  snap-builds:
    runs-on: ubuntu-20.04
    # only build the snap for pull requests, it's not needed on release branches
    # or on master since we have launchpad build recipes which do this already
    if: ${{ github.event_name == 'pull_request' }}
    steps:
    - name: Checkout code
      uses: actions/checkout@v2

    - name: Build snapd snap
      uses: snapcore/action-build@v1
      with:
        snapcraft-channel: 4.x/candidate

    - name: Check built artifact
      run: |
        unsquashfs snapd*.snap meta/snap.yaml usr/lib/snapd/info
        if cat squashfs-root/meta/snap.yaml | grep -q "version:.*dirty.*"; then
          echo "PR produces dirty snapd snap version"
          cat squashfs-root/usr/lib/snapd/dirty-git-tree-info.txt
          exit 1
        elif cat squashfs-root/usr/lib/snapd/info | grep -q "VERSION=.*dirty.*"; then
          echo "PR produces dirty internal snapd info version"
          cat squashfs-root/usr/lib/snapd/info
          exit 1
        fi

    - name: Uploading snapd snap artifact
      uses: actions/upload-artifact@v2
      with:
        name: snap-files
        path: "*.snap"

  cache-build-deps:
    runs-on: ubuntu-20.04
    steps:
    - name: Checkout code
      uses: actions/checkout@v2
      with:
        # needed for git commit history
        fetch-depth: 0
        # NOTE: checkout the code in a fixed location, even for forks, as this
        # is relevant for go's import system.
        path: ./src/github.com/snapcore/snapd

    # Fetch base ref, needed for golangci-lint
    - name: Fetching base ref ${{ github.base_ref }}
      run: |
        cd ${{ github.workspace }}/src/github.com/snapcore/snapd
        git fetch origin ${{ github.base_ref }}:${{ github.base_ref }}
        # golang latest ensures things work on the edge

    - name: Download Debian dependencies
      run: |
          sudo apt clean
          sudo apt update
          sudo apt build-dep -d -y ${{ github.workspace }}/src/github.com/snapcore/snapd

    - name: Copy dependencies
      run: |
        sudo tar cvf cached-apt.tar /var/cache/apt

    - name: upload Debian dependencies
      uses: actions/upload-artifact@v2
      with:
        name: debian-dependencies
        path: ./cached-apt.tar

  static-checks:
    runs-on: ubuntu-20.04
    needs: [cache-build-deps]
    env:
      GOPATH: ${{ github.workspace }}
      # Set PATH to ignore the load of magic binaries from /usr/local/bin And
      # to use the go snap automatically. Note that we install go from the
      # snap in a step below. Without this we get the GitHub-controlled latest
      # version of go.
      PATH: /snap/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:${{ github.workspace }}/bin
      GOROOT: ""
      GITHUB_PULL_REQUEST: ${{ github.event.number }}

    strategy:
      # we cache successful runs so it's fine to keep going
      fail-fast: false      
      matrix:
        gochannel:
          - 1.13
          - latest/stable

    steps:
    - name: Checkout code
      uses: actions/checkout@v2
      with:
        # needed for git commit history
        fetch-depth: 0
        # NOTE: checkout the code in a fixed location, even for forks, as this
        # is relevant for go's import system.
        path: ./src/github.com/snapcore/snapd

    # Fetch base ref, needed for golangci-lint
    - name: Fetching base ref ${{ github.base_ref }}
      run: |
        cd ${{ github.workspace }}/src/github.com/snapcore/snapd
        git fetch origin ${{ github.base_ref }}:${{ github.base_ref }}

    - name: Download Debian dependencies
      uses: actions/download-artifact@v3
      with:
        name: debian-dependencies
        path: ./debian-deps/

    - name: Copy dependencies
      run: |
        test -f ./debian-deps/cached-apt.tar
        sudo tar xvf ./debian-deps/cached-apt.tar -C /

    - name: Install Debian dependencies
      run: |
          sudo apt update
          sudo apt build-dep -y ${{ github.workspace }}/src/github.com/snapcore/snapd

    # golang latest ensures things work on the edge
    - name: Install the go snap
      run: |
          sudo snap install --classic --channel=${{ matrix.gochannel }} go

    - name: Install ShellCheck as a snap
      run: |
          sudo apt-get remove --purge shellcheck
          sudo snap install shellcheck

    - name: Get C vendoring
      run: |
        cd ${{ github.workspace }}/src/github.com/snapcore/snapd/c-vendor && ./vendor.sh

    - name: golangci-lint
      if: ${{ matrix.gochannel == 'latest/stable' }}
      uses: golangci/golangci-lint-action@v2
      with:
        # version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest`
        # to use the latest version
        version: v1.45.2
        working-directory: ./src/github.com/snapcore/snapd
        # show only new issues
        # use empty path prefix to make annotations work
        args: --new-from-rev=${{ github.base_ref }} --path-prefix=
        # skip all additional steps
        skip-go-installation: true
        skip-pkg-cache: true
        skip-build-cache: true
        # XXX: does no work with working-directory
        # only-new-issues: true

    - name: Get changed files
      id: changed-files
      uses: tj-actions/changed-files@v19
      with:
        path: ./src/github.com/snapcore/snapd

    - name: Save changes files
      run: |
          CHANGED_FILES="${{ steps.changed-files.outputs.all_changed_files }}"
          echo "CHANGED_FILES=$CHANGED_FILES" >> $GITHUB_ENV
          echo "The changed files found are: $CHANGED_FILES"

    - name: Run static checks
      run: |
          cd ${{ github.workspace }}/src/github.com/snapcore/snapd || exit 1
          # run gofmt checks only with Go 1.13
          if ! echo "${{ matrix.gochannel }}" | grep -E '1\.13' ; then
              # and skip with other versions
              export SKIP_GOFMT=1
              echo "Formatting checks will be skipped due to the use of Go version ${{ matrix.gochannel }}"
          else
              # misspell and infessassign do not build with Go 1.13
              export SKIP_MISSPELL=1
              export SKIP_INEFFASSIGN=1
          fi
          sudo apt-get install -y python3-yamlordereddictloader
          ./run-checks --static

  unit-tests:
    needs: [static-checks]
    runs-on: ubuntu-20.04
    env:
      GOPATH: ${{ github.workspace }}
      # Set PATH to ignore the load of magic binaries from /usr/local/bin And
      # to use the go snap automatically. Note that we install go from the
      # snap in a step below. Without this we get the GitHub-controlled latest
      # version of go.
      PATH: /snap/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:${{ github.workspace }}/bin
      GOROOT: ""
      GITHUB_PULL_REQUEST: ${{ github.event.number }}
    strategy:
      # we cache successful runs so it's fine to keep going
      fail-fast: false      
      matrix:
        gochannel:
          - 1.13
          - latest/stable
        unit-scenario:
          - normal
          - snapd_debug
          - withbootassetstesting
          - nosecboot
          - faultinject

    steps:
    - name: Checkout code
      uses: actions/checkout@v2
      with:
        # needed for git commit history
        fetch-depth: 0
        # NOTE: checkout the code in a fixed location, even for forks, as this
        # is relevant for go's import system.
        path: ./src/github.com/snapcore/snapd

    # Fetch base ref, needed for golangci-lint
    - name: Fetching base ref ${{ github.base_ref }}
      run: |
        cd ${{ github.workspace }}/src/github.com/snapcore/snapd
        git fetch origin ${{ github.base_ref }}:${{ github.base_ref }}

    - name: Download Debian dependencies
      uses: actions/download-artifact@v3
      with:
        name: debian-dependencies
        path: ./debian-deps/

    - name: Copy dependencies
      run: |
        test -f ./debian-deps/cached-apt.tar
        sudo tar xvf ./debian-deps/cached-apt.tar -C /

    - name: Install Debian dependencies
      run: |
          sudo apt update
          sudo apt build-dep -y ${{ github.workspace }}/src/github.com/snapcore/snapd

    # golang latest ensures things work on the edge
    - name: Install the go snap
      run: |
          sudo snap install --classic --channel=${{ matrix.gochannel }} go

    - name: Build C
      run: |
          cd ${{ github.workspace }}/src/github.com/snapcore/snapd/cmd/
          ./autogen.sh
          make -j2

    - name: Get deps
      run: |
          cd ${{ github.workspace }}/src/github.com/snapcore/snapd/ && ./get-deps.sh

    - name: Build Go
      run: |
          go build github.com/snapcore/snapd/...

    - name: Test C
      run: |
          cd ${{ github.workspace }}/src/github.com/snapcore/snapd/cmd/ && make check

    - name: Reset code coverage data
      run: |
          rm -rf ${{ github.workspace }}/.coverage/

    - name: Test Go
      if: ${{ matrix.unit-scenario == 'normal' }}
      run: |
        cd ${{ github.workspace }}/src/github.com/snapcore/snapd || exit 1
        ./run-checks --unit

    - name: Test Go (SNAPD_DEBUG=1)
      if: ${{ matrix.unit-scenario == 'snapd_debug' }}
      run: |
        cd ${{ github.workspace }}/src/github.com/snapcore/snapd || exit 1
        SKIP_DIRTY_CHECK=1 SNAPD_DEBUG=1 ./run-checks --unit

    - name: Test Go (withbootassetstesting)
      if: ${{ matrix.unit-scenario == 'withbootassetstesting' }}
      run: |
        cd ${{ github.workspace }}/src/github.com/snapcore/snapd || exit 1
        SKIP_DIRTY_CHECK=1 GO_BUILD_TAGS=withbootassetstesting ./run-checks --unit

    - name: Test Go (nosecboot)
      if: ${{ matrix.unit-scenario == 'nosecboot' }}
      run: |
        cd ${{ github.workspace }}/src/github.com/snapcore/snapd || exit 1
        echo "Dropping github.com/snapcore/secboot"
        # use govendor remove so that a subsequent govendor sync does not
        # install secboot again
        # ${{ github.workspace }}/bin/govendor remove github.com/snapcore/secboot
        # ${{ github.workspace }}/bin/govendor remove +unused
        SKIP_DIRTY_CHECK=1 GO_BUILD_TAGS=nosecboot ./run-checks --unit

    - name: Test Go (faultinject)
      if: ${{ matrix.unit-scenario == 'faultinject' }}
      run: |
        cd ${{ github.workspace }}/src/github.com/snapcore/snapd || exit 1
        SKIP_DIRTY_CHECK=1 GO_BUILD_TAGS=faultinject ./run-checks --unit

    - name: Upload the coverage results
      if: ${{ matrix.gochannel != 'latest/stable' }}
      uses: actions/upload-artifact@v2
      with:
        name: coverage-files
        path: "${{ github.workspace }}/src/github.com/snapcore/snapd/.coverage/coverage*.cov"

  code-coverage:
    needs: [unit-tests]
    runs-on: ubuntu-20.04
    env:
      GOPATH: ${{ github.workspace }}
      # Set PATH to ignore the load of magic binaries from /usr/local/bin And
      # to use the go snap automatically. Note that we install go from the
      # snap in a step below. Without this we get the GitHub-controlled latest
      # version of go.
      PATH: /snap/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:${{ github.workspace }}/bin
      GOROOT: ""
      GITHUB_PULL_REQUEST: ${{ github.event.number }}
    steps:
    - name: Download the coverage files
      uses: actions/download-artifact@v3
      with:
        name: coverage-files
        path: .coverage/

    - name: Merge coverage results
      run: |
        echo "List the coverage files downloaded"
        ls -l .coverage/

        echo "Install covertool and merge the results"
        go install github.com/makholm/covertool@latest
        covertool merge -o .coverage/all.cov .coverage/coverage*.cov

    - name: Upload coverage to Codecov
      uses: codecov/codecov-action@v2
      # uploading to codecov occasionally fails, so continue running the test
      # workflow regardless of the upload
      continue-on-error: true
      with:
        fail_ci_if_error: true
        flags: unittests
        name: codecov-umbrella
        files: .coverage/all.cov
        verbose: true

  spread:
    needs: [unit-tests]
    # have spread jobs run on master on PRs only, but on both PRs and pushes to
    # release branches
    if: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
    runs-on: self-hosted
    strategy:
      # FIXME: enable fail-fast mode once spread can cancel an executing job.
      # Disable fail-fast mode as it doesn't function with spread. It seems
      # that cancelling tasks requires short, interruptible actions and
      # interrupting spread, notably, does not work today. As such disable
      # fail-fast while we tackle that problem upstream.
      fail-fast: false
      matrix:
        system:
        - amazon-linux-2-64
        - arch-linux-64
        - centos-7-64
        - centos-8-64
        - debian-10-64
        - debian-11-64
        - debian-sid-64
        - fedora-35-64
        - fedora-36-64
        - opensuse-15.3-64
        - opensuse-15.4-64
        - opensuse-tumbleweed-64
        - ubuntu-14.04-64
        - ubuntu-16.04-64
        - ubuntu-18.04-32
        - ubuntu-18.04-64
        - ubuntu-20.04-64
        - ubuntu-21.10-64
        - ubuntu-22.04-64
        - ubuntu-core-16-64
        - ubuntu-core-18-64
        - ubuntu-core-20-64
        - ubuntu-core-22-64
        - ubuntu-secboot-20.04-64
    steps:
    - name: Cleanup job workspace
      id: cleanup-job-workspace
      run: |
          rm -rf "${{ github.workspace }}"
          mkdir "${{ github.workspace }}"

    - name: Checkout code
      uses: actions/checkout@v2
      with:
        # spread uses tags as delta reference
        fetch-depth: 0

    - name: Cache snapd test results
      id: cache-snapd-test-results
      uses: pat-s/always-upload-cache@v2.1.5
      with:
        path: "${{ github.workspace }}/.test-results"
        key: "${{ github.run_id }}-${{ github.job }}-${{ matrix.system }}-results"

    - name: Check cached test results
      id: cached-results
      run: |
          TEST_RESULTS_DIR="${{ github.workspace }}/.test-results"
          mkdir -p "$TEST_RESULTS_DIR"
          echo "TEST_RESULTS_DIR=$TEST_RESULTS_DIR" >> $GITHUB_ENV

          # Save the var with the failed tests file
          echo "FAILED_TESTS_FILE=$TEST_RESULTS_DIR/${{ matrix.system }}-failed-tests" >> $GITHUB_ENV

    - name: Check failed tests to run
      if: "!contains(github.event.pull_request.labels.*.name, 'Run all')"
      run: |
          # Save previous failed test results in FAILED_TESTS env var
          FAILED_TESTS=""
          if [ -f "$FAILED_TESTS_FILE" ]; then
              echo "Failed tests file found"
              FAILED_TESTS="$(cat $FAILED_TESTS_FILE)"
              if [ -n "$FAILED_TESTS" ]; then
                  echo "Failed tests to run: $FAILED_TESTS"
                  echo "FAILED_TESTS=$FAILED_TESTS" >> $GITHUB_ENV
              fi
          fi

    - name: Run spread tests
      if: "!contains(github.event.pull_request.labels.*.name, 'Skip spread')"
      env:
          SPREAD_GOOGLE_KEY: ${{ secrets.SPREAD_GOOGLE_KEY }}
      run: |
          # Register a problem matcher to highlight spread failures
          echo "::add-matcher::.github/spread-problem-matcher.json"

          # Save previous failed test results in FAILED_TESTS env var
          RUN_TESTS="google:${{ matrix.system }}:tests/..."
          if [ -n "$FAILED_TESTS" ]; then
              RUN_TESTS="$FAILED_TESTS"
          fi
          # Run spread tests
          # "pipefail" ensures that a non-zero status from the spread is
          # propagated; and we use a subshell as this option could trigger
          # undesired changes elsewhere
          (set -o pipefail; spread -abend $RUN_TESTS | tee spread.log)

    - name: Discard spread workers
      if: always()
      run: |
        shopt -s nullglob;
        for r in .spread-reuse.*.yaml; do
          spread -discard -reuse-pid="$(echo "$r" | grep -o -E '[0-9]+')";
        done

    - name: report spread errors
      if: always()
      run: |
        if [ -e spread.log ]; then
            echo "Running spread log analyzer"
            issues_metadata='{"source_url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"}'
            ./tests/lib/external/snapd-testing-tools/utils/log-parser spread.log --print-detail error-debug --output spread-results.json --cut 100
            while IFS= read -r line; do
                if [ ! -z "$line" ]; then
                    echo "Reporting spread error..."
                    ./tests/lib/tools/report-mongodb --db-name snapd --db-collection spread_errors --metadata "$issues_metadata" "$line"
                fi
            done <<< $(jq -cr '.[] | select( .type == "info") | select( (.info_type == "Error") or (.info_type == "Debug"))' spread-results.json)
        else
            echo "No spread log found, skipping errors reporting"
        fi

    - name: save spread results
      if: always()
      run: |
          if [ -f spread.log ]; then
              echo "Running spread log parser"
              ./tests/lib/external/snapd-testing-tools/utils/log-parser spread.log --output spread-results.json

              echo "Determining which tests were executed"
              RUN_TESTS="google:${{ matrix.system }}:tests/..."
              if [ -n "$FAILED_TESTS" ]; then
                  RUN_TESTS="$FAILED_TESTS"
              fi

              echo "Running spread log analyzer"
              ./tests/lib/external/snapd-testing-tools/utils/log-analyzer list-reexecute-tasks "$RUN_TESTS" spread-results.json > "$FAILED_TESTS_FILE"
          else
              echo "No spread log found, saving empty list of failed tests"
              touch "$FAILED_TESTS_FILE"
          fi

  spread-nested:
    needs: [unit-tests]
    # have spread jobs run on master on PRs only, but on both PRs and pushes to
    # release branches
    if: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
    runs-on: self-hosted
    strategy:
      # FIXME: enable fail-fast mode once spread can cancel an executing job.
      # Disable fail-fast mode as it doesn't function with spread. It seems
      # that cancelling tasks requires short, interruptible actions and
      # interrupting spread, notably, does not work today. As such disable
      # fail-fast while we tackle that problem upstream.
      fail-fast: false
      matrix:
        system:
        - ubuntu-16.04-64
        - ubuntu-18.04-64
        - ubuntu-20.04-64
        - ubuntu-22.04-64
    steps:
    - name: Cleanup job workspace
      id: cleanup-job-workspace
      run: |
          rm -rf "${{ github.workspace }}"
          mkdir "${{ github.workspace }}"

    - name: Checkout code
      uses: actions/checkout@v2

    - name: Cache snapd test results
      id: cache-snapd-test-results
      uses: pat-s/always-upload-cache@v2.1.5
      with:
        path: "${{ github.workspace }}/.test-results"
        key: "${{ github.run_id }}-${{ github.job }}-${{ matrix.system }}-results"

    - name: Check cached test results
      id: cached-results
      run: |
          TEST_RESULTS_DIR="${{ github.workspace }}/.test-results"
          mkdir -p "$TEST_RESULTS_DIR"
          echo "TEST_RESULTS_DIR=$TEST_RESULTS_DIR" >> $GITHUB_ENV

          # Save the var with the failed tests file
          echo "FAILED_TESTS_FILE=$TEST_RESULTS_DIR/${{ matrix.system }}-failed-tests" >> $GITHUB_ENV

    - name: Check failed tests to run
      if: "!contains(github.event.pull_request.labels.*.name, 'Run all')"
      run: |
          # Save previous failed test results in FAILED_TESTS env var
          FAILED_TESTS=""
          if [ -f "$FAILED_TESTS_FILE" ]; then
              echo "Failed tests file found"
              FAILED_TESTS="$(cat $FAILED_TESTS_FILE)"
              if [ -n "$FAILED_TESTS" ]; then
                  echo "Failed tests to run: $FAILED_TESTS"
                  echo "FAILED_TESTS=$FAILED_TESTS" >> $GITHUB_ENV
              fi
          fi

    - name: Run spread tests
      # run if the commit is pushed to the release/* branch or there is a 'Run
      # nested' label set on the PR
      if: "contains(github.event.pull_request.labels.*.name, 'Run nested') || contains(github.ref, 'refs/heads/release/')"
      env:
          SPREAD_GOOGLE_KEY: ${{ secrets.SPREAD_GOOGLE_KEY }}
      run: |
          # Register a problem matcher to highlight spread failures
          echo "::add-matcher::.github/spread-problem-matcher.json"
          export NESTED_BUILD_SNAPD_FROM_CURRENT=true
          export NESTED_ENABLE_KVM=true
          RUN_TESTS="google-nested:${{ matrix.system }}:tests/nested/..."
          if [ -n "$FAILED_TESTS" ]; then
              RUN_TESTS="$FAILED_TESTS"
          fi
          # Run spread tests
          # "pipefail" ensures that a non-zero status from the spread is
          # propagated; and we use a subshell as this option could trigger
          # undesired changes elsewhere
          (set -o pipefail; spread -abend $RUN_TESTS | tee spread.log)

    - name: Discard spread workers
      if: always()
      run: |
        shopt -s nullglob;
        for r in .spread-reuse.*.yaml; do
          spread -discard -reuse-pid="$(echo "$r" | grep -o -E '[0-9]+')";
        done

    - name: report spread errors
      if: always()
      run: |
        if [ -e spread.log ]; then
            echo "Running spread log analyzer"
            issues_metadata='{"source_url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"}'
            ./tests/lib/external/snapd-testing-tools/utils/log-parser spread.log --print-detail error --output spread-results.json --cut 100
            while IFS= read -r line; do
                if [ ! -z "$line" ]; then
                    echo "Reporting spread error..."
                    ./tests/lib/tools/report-mongodb --db-name snapd --db-collection spread_errors --metadata "$issues_metadata" "$line"
                fi
            done <<< $(jq -cr '.[] | select( .type == "info") | select( .info_type == "Error")' spread-results.json)
        else
            echo "No spread log found, skipping errors reporting"
        fi

    - name: save spread results
      if: always()
      run: |
          if [ -f spread.log ]; then
              echo "Running spread log parser"
              ./tests/lib/external/snapd-testing-tools/utils/log-parser spread.log --output spread-results.json
    
              echo "Determining which tests were executed"
              RUN_TESTS="google-nested:${{ matrix.system }}:tests/nested/..."
              if [ -n "$FAILED_TESTS" ]; then
                  RUN_TESTS="$FAILED_TESTS"
              fi

              echo "Running spread log analyzer"
              ./tests/lib/external/snapd-testing-tools/utils/log-analyzer list-reexecute-tasks "$RUN_TESTS" spread-results.json > "$FAILED_TESTS_FILE"
          else
              echo "No spread log found, saving empty list of failed tests"
              touch "$FAILED_TESTS_FILE"
          fi