Implementing BDN Perf pipeline (#758)

* The CI scripts, YML and config files to check perf using BDN benchmarking.  Also, a fix in ClusterBench for and renamed Lua Runner benchmarking

* Cleaned up the debug message and updated couple expected values

* Updated SAddRem - windows - to 130 from 118.  Also removed unncesssary build statement in YML

* Proof of concept for Performance run on ADO using BDN

* Comment out number of Cores check so can try proof of concept on ADO perf machines

* Getting close to having BDN working

* Fixed RespLuaStress

* Fixed issue in Lua Runner Stress

* Updated config values, set charting and commit comment as separate tasks

* Updated config values

* Minor change to YML (remove pull requests) and updated LuaRunner config value

* One more config value fix

* Added a task to do the push

* Putting back to push chart in the chart task and not a separate one

* Moved the permissions to full workflow

* Just minor to add commits

* Adding actions: write permit since actions default to read only

* Set up the commit comment for BDN as well as summary to show current vs previous BDN perf numbers

* A few fixes

* Updated BDN perf YML, config files and ps1 script

* Versioning issue with BDN test files, fixed now

* Bump threshold to 35% (135%) to trigger perf failure

* Set to Linux only to reduce number of consectutive tests which caused push collisions with GitHub

* Just Windows only test run

* Putting it back to both OS for the tests

* Set threshhold to 50% (150%) as seeing BDN ran twice on same code sometimes is in 40%+ range of difference.

* Big change where using continuousbenchmark branch for all data for continuous benchmarking and also changed to where script is only checking allocated values and let github-action-benchmark compare mean values

* Updated ClusterBench to get main changes

* Fix issue with Cluster BDN is not able to access folder so setting to tmp which does have access

* Bumping threshold to 50% (150%) as mean value in RespParseStress can vary into the 40% from run to run.

* First update

* Hiding the Gen0 column for Lua benchmark

* Updated BDN test infrastructure to match the new BDN org.

* Fix bug in ObjectOperations

* Set max items on chart to 50 ... can modify later if want

* Added the run on push to main

* Put Performance Pipeline back

* Fixed the default case when no parameters sent to ps1

* Cleaned up config files:
have shorter names for values
not have configuration or framework (moved to ps1 file)
not have windows / linux differnce in expected
Moved acceptable allocated range to script

modified the script:
to only load values for the specific test instead of looking for all values
changes to match the changes in config (shorter names, etc)

* Big update to where moved to a single config file and the ps1 file does not need to be updated when new BDNs are added.

* Added workflow_dispatch: to deploy website yml so we can manually deploy website if we want

* Updated alert-threshold from 150% to 140%. If too many false positives we can raise it back. Also, removed darrenge from the alert CC line as no need for that.

* Removed the commit comment / comparison to previous commit task. Still will get  summary data in the specific workflow run and still uploads data to charts (when pushing to main).

* Minor update - comments only

.github/workflows/ci-bdnbenchmark.yml

@@ -1,18 +1,23 @@
 name: Garnet CI BDN.benchmark
 on:
   workflow_dispatch:
-
+  push:
+    branches:
+      - main
+
 env:
   DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
   DOTNET_NOLOGO: true
 
+permissions:
+  actions: write
+  deployments: write  # permission to deploy GitHub pages website
+  contents: write  # permission to update benchmark contents in continuousbenchmark branch
+
 jobs:
   changes:
     name: Check for changes
-    runs-on: ubuntu-latest # don't need matrix to test where the changes were made in code
-    permissions:
-      pull-requests: read
-      contents: read
+    runs-on: ubuntu-latest
     steps:
     - name: Check out code
       uses: actions/checkout@v4
@@ -25,3 +30,61 @@ jobs:
             - 'website/**'
           bdnbenchmark:
             - '!((*.md)|(website/**))'
+  
+  # Job to build and run BDN.benchmark
+  build-run-bdnbenchmark:
+    name: BDNBenchmark
+    needs: [changes]
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ ubuntu-latest, windows-latest ]
+        framework: [ 'net8.0' ]
+        configuration: [ 'Release' ]
+        test: [ 'Operations.BasicOperations', 'Operations.ObjectOperations', 'Cluster.ClusterMigrate', 'Cluster.ClusterOperations', 'Lua.LuaScripts' ]
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v4
+        with:
+          dotnet-version: 8.0.x
+      - name: Install dependencies
+        run: dotnet restore
+
+      - name: Run BDN.benchmark Perf Test
+        run: ./test/BDNPerfTests/run_bdnperftest.ps1 ${{ matrix.test }}
+        shell: pwsh
+        continue-on-error: false
+
+      - name: Upload test results to artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: Results-${{ matrix.os }}-${{ matrix.framework }}-${{ matrix.configuration }}-${{ matrix.test }}
+          path: |
+             ./test/BDNPerfTests/results
+        if: ${{ always() }}
+
+      - name: Upload Error Log to artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: ErrorLog-${{ matrix.os }}-${{ matrix.framework }}-${{ matrix.configuration }}-${{ matrix.test }}
+          path: |
+            ./test/BDNPerfTests/errorlog
+        if: ${{ always() }}
+
+      # Run `github-action-benchmark` action for the Continuous Benchmarking Charts (https://microsoft.github.io/garnet/charts/)
+      - name: Store benchmark result for charts
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: ${{matrix.test}} (${{matrix.os}}  ${{matrix.framework}} ${{matrix.configuration}})
+          tool: 'benchmarkdotnet'
+          output-file-path: ./test/BDNPerfTests/BenchmarkDotNet.Artifacts/results/BDN.benchmark.${{ matrix.test }}-report-full-compressed.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          summary-always: true
+          gh-pages-branch: 'continuousbenchmark'
+          benchmark-data-dir-path: 'website/static/charts'
+          auto-push: true
+          max-items-in-chart: 50
+

.github/workflows/deploy-website.yml

@@ -1,6 +1,7 @@
 name: Deploy to GitHub Pages
 
 on:
+  workflow_dispatch:
   push:
     branches:
       - main

benchmark/BDN.benchmark/Lua/LuaScripts.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT license.
 
 using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Columns;
 using Garnet.server;
 
 namespace BDN.benchmark.Lua
@@ -10,6 +11,7 @@ namespace BDN.benchmark.Lua
     /// Benchmark for Lua
     /// </summary>
     [MemoryDiagnoser]
+    [HideColumns(Column.Gen0)]
     public unsafe class LuaScripts
     {
         /// <summary>

test/BDNPerfTests/BDN_Benchmark_Config.json

@@ -0,0 +1,48 @@
+{
+  "_comment": "NOTE: If adding a new BDN perf test to this file, you will need to add the name to the test: line in ci-bdnbenchmark.yml (~line 45). This ensures all the tests are not ran sequentially when ran as github action.",
+  "_comment2": "NOTE: The BDN Perf name in the ci-bdnbenchmark.yml file test: line should NOT have the BDN.benchmark prefix and the .* ending. It needs to be this way for the reporting part of the BDN Charts.",
+
+  "BDN.benchmark.Operations.BasicOperations.*": {
+    "expected_InlinePing_ACL": 0,
+    "expected_InlinePing_AOF": 0,
+    "expected_InlinePing_None": 0
+  },
+  "BDN.benchmark.Operations.ObjectOperations.*": {
+    "expected_ZAddRem_ACL": 17.97,
+    "expected_LPushPop_ACL": 14.06,
+    "expected_SAddRem_ACL": 12.5,
+    "expected_HSetDel_ACL": 42.19,
+    "expected_ZAddRem_AOF": 17.97,
+    "expected_LPushPop_AOF": 14.06,
+    "expected_SAddRem_AOF": 12.5,
+    "expected_HSetDel_AOF": 42.19,
+    "expected_ZAddRem_None": 17.97,
+    "expected_LPushPop_None": 14.06,
+    "expected_SAddRem_None": 12.5,
+    "expected_HSetDel_None": 42.19
+  },
+  "BDN.benchmark.Cluster.ClusterOperations.*": {
+    "expected_Get_DSV": 0,
+    "expected_Set_DSV": 0,
+    "expected_MGet_DSV": 0,
+    "expected_MSet_DSV": 0,
+    "expected_CPBSet_DSV": 0,
+    "expected_Get_None": 0,
+    "expected_Set_None": 0,
+    "expected_MGet_None": 0,
+    "expected_MSet_None": 0,
+    "expected_CPBSet_None": 0
+  },
+  "BDN.benchmark.Cluster.ClusterMigrate.*": {
+    "expected_Get_None": 0,
+    "expected_Set_None": 0,
+    "expected_MGet_None": 0,
+    "expected_MSet_None": 0
+  },
+  "BDN.benchmark.Lua.LuaScripts.*": {
+    "expected_Script1_None": 24,
+    "expected_Script2_None": 144,
+    "expected_Script3_None": 240,
+    "expected_Script4_None": 776
+  }
+}