pipeline.yaml

name: Tests

on:
  push:
    # branches-ignore:
      # - main
      # - dev
  workflow_dispatch:

env:
  REGISTRY: ghcr.io
  IMAGE_NAME: ${{ github.repository }}

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  static-analysis:
    strategy:
      matrix:
        python-version: ["3.9", "3.10", "3.11", "3.12"]
      fail-fast: false
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install Dependencies and library
        shell: bash
        run: |
          set -ux
          python -m pip install --upgrade pip
          pip install -e ".[submodules,docs,lint,testing]"
      - name: Run mypy
        shell: bash
        run: mypy fastagency tests

      - name: Run bandit
        shell: bash
        run: bandit -c pyproject.toml -r fastagency

      - name: Run Semgrep
        shell: bash
        run: semgrep scan --config auto --error

  test-without-llms:
    strategy:
      matrix:
        python-version: ["3.9", "3.10", "3.11", "3.12"]
      fail-fast: false
    uses: ./.github/workflows/test.yaml
    with:
      python-version: ${{ matrix.python-version }}
      environment: null
      use-llms: ""
    secrets: inherit  # pragma: allowlist secret

  test-playwright-without-llms:
    strategy:
      matrix:
        python-version: ["3.10", "3.11", "3.12"]
      fail-fast: false
    uses: ./.github/workflows/test-playwright.yaml
    with:
      python-version: ${{ matrix.python-version }}
      environment: null
      use-llms: ""
    secrets: inherit  # pragma: allowlist secret


  test-with-anthropic:
    uses: ./.github/workflows/test.yaml
    with:
      python-version: "3.9"
      environment: testing
      use-llms: "anthropic"
    secrets: inherit  # pragma: allowlist secret

  test-with-azure_oai:
    uses: ./.github/workflows/test.yaml
    with:
      python-version: "3.9"
      environment: testing
      use-llms: "azure_oai"
    secrets: inherit  # pragma: allowlist secret

  test-with-openai:
    uses: ./.github/workflows/test.yaml
    with:
      python-version: "3.9"
      environment: testing
      use-llms: "openai"
    secrets: inherit  # pragma: allowlist secret

  test-with-togetherai:
    uses: ./.github/workflows/test.yaml
    with:
      python-version: "3.9"
      environment: testing
      use-llms: "togetherai"
    secrets: inherit  # pragma: allowlist secret

  test-with-llm:
    uses: ./.github/workflows/test.yaml
    with:
      python-version: "3.9"
      environment: testing
      use-llms: "llm"
    secrets: inherit  # pragma: allowlist secret

  test-macos-latest:
    if: github.event.pull_request.draft == false
    runs-on: macos-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"
          cache: "pip"
          cache-dependency-path: pyproject.toml

      - name: Install Dependencies
        if: steps.cache.outputs.cache-hit != 'true'
        run: pip install .[submodules,docs,testing]
      - name: Test
        run: bash scripts/test.sh -m "not (nats or anthropic or azure_oai or openai or togetherai or llm)"

  test-windows-latest:
    if: github.event.pull_request.draft == false
    runs-on: windows-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"
          cache: "pip"
          cache-dependency-path: pyproject.toml

      - name: Install Dependencies
        if: steps.cache.outputs.cache-hit != 'true'
        run: pip install .[submodules,docs,testing]
      - name: Test
        run: bash scripts/test.sh -m "not (nats or anthropic or azure_oai or openai or togetherai or llm)"

  coverage-combine:
    needs:
      - test-without-llms
      - test-with-llm
      - test-playwright-without-llms
      - test-with-anthropic
      - test-with-azure_oai
      - test-with-openai
      - test-with-togetherai
      - test-windows-latest
      - test-macos-latest
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: "3.9"
          cache: "pip"
          cache-dependency-path: pyproject.toml

      - name: Get coverage files
        uses: actions/download-artifact@v4
        with:
          pattern: .coverage*
          path: coverage
          merge-multiple: true

      - run: pip install coverage[toml]

      - run: ls -la coverage
      - run: coverage combine coverage
      - run: coverage report -i
      - run: coverage html -i --show-contexts --title "FastAgency coverage for ${{ github.sha }}"

      - name: Store coverage html
        uses: actions/upload-artifact@v4
        with:
          name: coverage-html
          path: htmlcov

      - name: Upload coverage reports to Codecov
        uses: codecov/codecov-action@v5
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          slug: airtai/fastagency

  pre-commit-check:
    runs-on: ubuntu-latest
    env:
      SKIP: "static-analysis, test"
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.10"
      - name: Set $PY environment variable
        run: echo "PY=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV
      - uses: actions/cache@v4
        with:
          path: ~/.cache/pre-commit
          key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }}
      - name: Install Dependencies
        run: pip install .[submodules,docs,testing]
      - uses: pre-commit/action@v3.0.1

  # https://github.com/marketplace/actions/alls-green#why
  check: # This job does nothing and is only used for the branch protection
    # from: https://github.com/re-actors/alls-green
    # Important: For this to work properly, it is a must to have the job always
    # run, otherwise GitHub will make it skipped when any of the dependencies
    # fail. In some contexts, skipped is interpreted as success which may lead
    # to undersired, unobvious and even dangerous (as in security breach
    # "dangerous") side-effects.
    if: always()

    needs:
      - static-analysis
      - pre-commit-check
      - coverage-combine
      - test-macos-latest
      - test-windows-latest

    runs-on: ubuntu-latest

    steps:
      - name: Decide whether the needed jobs succeeded or failed
        uses: re-actors/alls-green@release/v1 # nosemgrep
        with:
          # allowed-failures: docs, linters
          # allowed-skips: non-voting-flaky-job
          jobs: ${{ toJSON(needs) }}