Introduce Nightly Regression Pipeline (#100)

* Updated tests, dockerfile, and created new nightly workflow.

* Enabling manual trigger on branch for testing.

* Fixed depends for GLIBC.

* Triggering single test.

* Trigger on branch.

* Moving to bash.

* Adding bash.

* Running in container.

* Adding installer build.

* Adding debug.

* Adding debug 2.

* Moving to Ubuntu 22.04.

* Downloading from main's release binary.

* Incrementing version.

* Building installer and then running integration tests.

* Adding execution permissions to cover-agent.

* Fixing OpenAI token and cache.

* Enabling all tests.

* Adding cache and moving to Python version 3.12.

* Moving regrassion testing pipeline to nightly.

* Adding Docs.

Resolves #99 #84 #65 #38

.github/workflows/ci_pipeline.yml

@@ -38,7 +38,19 @@ jobs:
  - name: Set up Python
  uses: actions/setup-python@v2
  with:
- python-version: '3.x'
+ python-version: '3.12'
+
+ # Step to cache Poetry dependencies
+ - name: Cache Poetry dependencies
+ uses: actions/cache@v2
+ with:
+ path: |
+ ~/.cache/pypoetry
+ ~/.cache/pip
+ key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
+ restore-keys: |
+ ${{ runner.os }}-poetry-
+
  - name: Install Poetry
  run: pip install poetry
  - name: Install dependencies using Poetry
@@ -71,7 +83,19 @@ jobs:
  - name: Set up Python
  uses: actions/setup-python@v2
  with:
- python-version: '3.x'
+ python-version: '3.12'
+
+ # Step to cache Poetry dependencies
+ - name: Cache Poetry dependencies
+ uses: actions/cache@v2
+ with:
+ path: |
+ ~/.cache/pypoetry
+ ~/.cache/pip
+ key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
+ restore-keys: |
+ ${{ runner.os }}-poetry-
+
  - name: Install Poetry
  run: pip install poetry
  - name: Install dependencies using Poetry
@@ -88,7 +112,7 @@ jobs:
  if: needs.changes.outputs.relevant_changes == 'true'
  strategy:
  matrix:
- os: [ubuntu-latest, windows-latest, macos-latest]
+ os: [ubuntu-22.04, windows-latest, macos-latest]
 
  runs-on: ${{ matrix.os }}
  steps:
@@ -97,6 +121,18 @@ jobs:
  uses: actions/setup-python@v2
  with:
  python-version: '3.12'
+
+ # Step to cache Poetry dependencies
+ - name: Cache Poetry dependencies
+ uses: actions/cache@v2
+ with:
+ path: |
+ ~/.cache/pypoetry
+ ~/.cache/pip
+ key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
+ restore-keys: |
+ ${{ runner.os }}-poetry-
+
  - name: Install Dependencies
  run: |
  pip install poetry wandb
@@ -130,8 +166,8 @@ jobs:
  - name: Download executables (Ubuntu)
  uses: actions/download-artifact@v2
  with:
- name: cover-agent-ubuntu-latest
- path: dist/ubuntu-latest
+ name: cover-agent-ubuntu-22.04
+ path: dist/ubuntu-22.04
  - uses: actions/download-artifact@v2
  with:
  name: cover-agent-windows-latest
@@ -159,7 +195,7 @@ jobs:
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  with:
  upload_url: ${{ steps.create_release.outputs.upload_url }}
- asset_path: dist/ubuntu-latest/cover-agent
+ asset_path: dist/ubuntu-22.04/cover-agent
  asset_name: cover-agent-ubuntu
  asset_content_type: application/octet-stream
  - name: Upload Release Asset (Windows)

.github/workflows/nightly_regression.yml

@@ -0,0 +1,76 @@
+name: Nightly Integration Tests
+
+# Define the events that will trigger this workflow
+on:
+ # Schedule the workflow to run nightly at midnight
+ schedule:
+ - cron: '0 0 * * *'
+ # Allow manual triggering of the workflow
+ workflow_dispatch:
+
+jobs:
+ build-executable:
+ runs-on: ubuntu-22.04
+ steps:
+ - uses: actions/checkout@v2
+ - name: Set up Python
+ uses: actions/setup-python@v2
+ with:
+ python-version: '3.12'
+
+ # Step to cache Poetry dependencies
+ - name: Cache Poetry dependencies
+ uses: actions/cache@v2
+ with:
+ path: |
+ ~/.cache/pypoetry
+ ~/.cache/pip
+ key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
+ restore-keys: |
+ ${{ runner.os }}-poetry-
+
+ - name: Install Dependencies
+ run: |
+ pip install poetry wandb
+ poetry install
+ - name: Build Executable
+ run: make installer
+ - name: Upload Executable
+ uses: actions/upload-artifact@v2
+ with:
+ name: cover-agent
+ path: dist/cover-agent*
+
+ run-integration-tests:
+ needs: build-executable
+ runs-on: ubuntu-latest
+ container:
+ image: docker:latest
+ options: --privileged # Required for Docker-in-Docker
+
+ env:
+ OPENAI_API_KEY: ${{ secrets.OPENAI_KEY }}
+
+ steps:
+ # Step 1: Check out the repository code
+ - name: Checkout repository
+ uses: actions/checkout@v2
+
+ # Step 2: Download the artifact
+ - name: Download artifact
+ uses: actions/download-artifact@v2
+ with:
+ name: cover-agent
+ path: dist/
+
+ # Step 3: Run integration tests
+ - name: Run integration tests
+ run: |
+ # Set permissions for dist/cover-agent
+ chmod +x dist/cover-agent
+
+ # Install bash
+ apk add --no-cache bash
+ 
+ # Execute the shell script with bash
+ bash tests_integration/test_all.sh

Dockerfile

@@ -1,5 +1,5 @@
 # Dockerfile to build installer for Cover Agent
-FROM python:3.12
+FROM python:3.12-bullseye
 
 WORKDIR /app
 

cover_agent/version.txt

@@ -1 +1 @@
-0.1.42
+0.1.43

templated_tests/java_spring_calculator/Dockerfile

@@ -3,9 +3,13 @@ FROM maven:latest
 # Set the working directory inside the container
 WORKDIR /app
 
+# Install glibc
+RUN apt-get update && \
+ apt-get install -y libc6
+
 # Copy the pom.xml file and the source code
 COPY pom.xml .
 COPY src ./src
 
 # Package the application and run tests
-RUN mvn clean verify
+RUN mvn clean verify

templated_tests/java_spring_calculator/src/test/java/com/example/calculator/controller/CalculatorControllerTest.java

@@ -45,23 +45,4 @@ public void testSubtract() throws Exception {
  .andExpect(content().string("1.0"));
  }
 
- @Test
- public void testMultiply() throws Exception {
- mockMvc.perform(get("/multiply?a=2&b=3"))
- .andExpect(status().isOk())
- .andExpect(content().string("6.0"));
- }
-
- @Test
- public void testDivide() throws Exception {
- mockMvc.perform(get("/divide?a=6&b=3"))
- .andExpect(status().isOk())
- .andExpect(content().string("2.0"));
- }
-
- // @Test
- // public void testDivideByZero() throws Exception {
- // mockMvc.perform(get("/divide?a=1&b=0"))
- // .andExpect(status().isBadRequest());
- // }
 }

templated_tests/js_vanilla/Dockerfile

@@ -0,0 +1,12 @@
+FROM node:18
+
+WORKDIR /app
+
+# Install glibc
+RUN apt-get update && \
+ apt-get install -y libc6
+
+COPY . ./
+
+RUN npm install
+RUN npm run test:coverage

tests_integration/README.md

@@ -1,62 +1,97 @@
-# Integration tests for Cover Agent
-This folder contains end-to-end integration tests for Cover Agent
+# Integration Tests for Cover Agent
+This folder contains end-to-end integration tests for Cover Agent.
 
 ## Prerequisites
-Before running any of these tests you will need to build the installer package by running the following command from the root of the repository:
+Before running any of these tests, you will need to build the installer package by running the following command from the root of the repository:
 ```
 make installer
 ```
 
 You will also need [Docker](https://www.docker.com/) installed.
 
-__Note:__ These scripts were written for Linux but they have been tested on a Windows system using WSL 2 and Docker for Desktop.
+__Note:__ These scripts were written for Linux but have been tested on a Windows system using WSL 2 and Docker for Desktop.
 
-Since the targets live in Linux you'll need to build the installer in Linux (versus on Windows and MacOS). This can be done automatically in the `sh tests_integration/test_all.sh` script by add the `--run-installer` flag.
+Since the targets live in Linux, you'll need to build the installer in Linux (versus on Windows and MacOS). This can be done automatically in the `sh tests_integration/test_all.sh` script by adding the `--run-installer` flag.
 
-## How to run
-You can run these example test suites using a locally hosted LLM or in the cloud just as you would normally with Cover Agent
+## How to Run
+You can run these example test suites using a locally hosted LLM or in the cloud just as you would normally with Cover Agent.
 
 ### Running the Tests
-To run the full test suite simply run the following command from the root of the repository:
+To run the full test suite, simply run the following command from the root of the repository:
 ```
 sh tests_integration/test_all.sh
 ```
 
 Or run each test individually:
 #### Python Fast API Example
-````
+```
 sh tests_integration/test_with_docker.sh \
  --dockerfile "templated_tests/python_fastapi/Dockerfile" \
  --source-file-path "app.py" \
  --test-file-path "test_app.py" \
  --test-command "pytest --cov=. --cov-report=xml --cov-report=term"
-````
+```
 
 #### Go Webservice Example
-````
+```
 sh tests_integration/test_with_docker.sh \
  --dockerfile "templated_tests/go_webservice/Dockerfile" \
  --source-file-path "app.go" \
  --test-file-path "app_test.go" \
  --test-command "go test -coverprofile=coverage.out && gocov convert coverage.out | gocov-xml > coverage.xml" \
  --model "gpt-4o"
-````
+```
+
+#### Java Gradle Example
+```
+sh tests_integration/test_with_docker.sh \
+ --dockerfile "templated_tests/java_gradle/Dockerfile" \
+ --source-file-path "src/main/java/com/davidparry/cover/SimpleMathOperations.java" \
+ --test-file-path "src/test/groovy/com/davidparry/cover/SimpleMathOperationsSpec.groovy" \
+ --test-command "./gradlew clean test jacocoTestReport" \
+ --coverage-type "jacoco" \
+ --code-coverage-report-path "build/reports/jacoco/test/jacocoTestReport.csv" \
+ --model "gpt-4o"
+```
+
+#### Java Spring Calculator Example
+```
+sh tests_integration/test_with_docker.sh \
+ --dockerfile "templated_tests/java_spring_calculator/Dockerfile" \
+ --source-file-path "src/main/java/com/example/calculator/controller/CalculatorController.java" \
+ --test-file-path "src/test/java/com/example/calculator/controller/CalculatorControllerTest.java" \
+ --test-command "mvn verify" \
+ --coverage-type "jacoco" \
+ --code-coverage-report-path "target/site/jacoco/jacoco.csv" \
+ --model "gpt-4o"
+```
+
+#### VanillaJS Example
+```
+sh tests_integration/test_with_docker.sh \
+ --dockerfile "templated_tests/js_vanilla/Dockerfile" \
+ --source-file-path "ui.js" \
+ --test-file-path "ui.test.js" \
+ --test-command "npm run test:coverage" \
+ --code-coverage-report-path "coverage/coverage.xml" \
+ --model "gpt-4o"
+```
 
 ### Using Different LLMs
 You can use a different LLM by passing in the `--model` and `--api-base` parameters. For example, to use a locally hosted LLM with Ollama you can pass in:
 ```
 --model "ollama/mistral" --api-base "http://host.docker.internal:11434"
 ```
 
-For any other LLM that requires more environment variables to be set you will need to update the shell script and pass in the variables within the Docker command.
+For any other LLM that requires more environment variables to be set, you will need to update the shell script and pass in the variables within the Docker command.
 
-## When to run
-This test suite is intended to run with real LLMs (either locally hosted on online). If choosing cloud provided LLMs keep in mind that there is a cost associated with running these tests.
+## When to Run
+This test suite is intended to run with real LLMs (either locally hosted or online). If choosing cloud-provided LLMs, keep in mind that there is a cost associated with running these tests.
 
 These tests should **absolutely** be run before a massive refactor or any major changes to the LLM/prompting logic of the code.
 
-## How it works
-The integration tests run within Docker containers in order to prove out complete isolation from any external or existing environment
+## How It Works
+The integration tests run within Docker containers to ensure complete isolation from any external or existing environment.
 
 # Increasing Coverage Iteratively
 The `increase_coverage.py` script attempts to run Cover Agent for all files within the `cover_agent` directory. You'll need to call a Poetry shell first before running like so: