test: Add CRI benchmarks for container operations (#1569)

* test: Add CRI benchmarks for container operations Add CRI API benchmarks to `cri-containerd.test.exe`, and update helper functions in `cri-containerd` to accept `tb testing.TB` instead of `t *testing.T`. Switched `cri-containerd` to `test\internal\flag` so that `test\internal\require` can be used to check if features are present since that implementation already mirrors what `cri-containerd` does, but accepts a `testing.TB`, and deleted old `test/internal/stringsetflag.go` file. Renamed `scripts/Test-Functionl.ps1` to `scripts/Run-Tests.ps1`, and updated it to run both functional and CRI tests and benchmarks. Signed-off-by: Hamza El-Saawy <[email protected]> * PR: clarifying comment Signed-off-by: Hamza El-Saawy <[email protected]> --------- Signed-off-by: Hamza El-Saawy <[email protected]>
microsoft · Jan 31, 2023 · db7a679 · db7a679
1 parent d310213
commit db7a679
Show file tree

Hide file tree

Showing 17 changed files with 544 additions and 327 deletions.
diff --git a/scripts/Test-Functional.ps1 → scripts/Run-Tests.ps1 b/scripts/Test-Functional.ps1 → scripts/Run-Tests.ps1
@@ -1,4 +1,4 @@
-# ex: .\scripts\Test-Functional.ps1 -Action Bench -Count 2 -BenchTime "2x"
+# ex: .\scripts\Run-Tests.ps1 -vb -TestExe Functional -Count 2 -BenchTime "2x"
 
 [CmdletBinding()]
 param (
@@ -8,12 +8,25 @@ param (
  $Action = 'Bench',
 
  [string]
- $Note = '',
+ $Note,
 
  [string]
  $OutDirectory = '.\test\results',
 
+ [string]
+ $TestDirectory = '.\bin\test',
+
+ [ValidateSet('Functional', 'CRI')]
+ [string]
+ $TestExe = 'CRI',
+
+ [string]
+ $BenchstatPath = 'benchstat.exe',
+
  # test parameters
+ [switch]
+ $Shuffle,
+
  [int]
  $Count = 1,
 
@@ -28,34 +41,41 @@ param (
  $TestVerbose,
 
  [string]
- $Run = '',
+ $Run,
 
- [string]
- $Feature = ''
+ [string[]]
+ $Features
 )
-
+$ErrorActionPreference = 'Stop'
 Import-Module ( Join-Path $PSScriptRoot Testing.psm1 ) -Force
 
+$exe = Switch ($TestExe) {
+ 'Functional' { 'functional.test.exe'; break }
+ 'CRI' { 'cri-containerd.test.exe'; break }
+}
+$test = Join-Path $TestDirectory $exe
+
 $date = Get-Date
 $testcmd, $out = New-TestCommand `
  -Action $Action `
- -Path .\bin\test\functional.exe `
- -Name functional `
+ -Path $test `
+ -Name ($exe -replace '.test.exe$', '') `
  -OutDirectory $OutDirectory `
  -Date $date `
  -Note $Note `
+ -Shuffle:$Shuffle `
  -TestVerbose:$TestVerbose `
  -Count $Count `
  -BenchTime $BenchTime `
  -Timeout $Timeout `
  -Run $Run `
- -Feature $Feature `
+ -Features $Features `
  -Verbose:$Verbose
 
 Invoke-TestCommand `
  -TestCmd $testcmd `
  -OutputFile $out `
- -OutputCmd (&{ if ( $Action -eq 'Bench' ) { 'benchstat' } }) `
+ -OutputCmd (& { if ( $Action -eq 'Bench' ) { $BenchstatPath } }) `
  -Preamble `
  -Date $Date `
  -Note $Note `

diff --git a/scripts/Test-LCOW-UVM.ps1 b/scripts/Test-LCOW-UVM.ps1
@@ -1,4 +1,4 @@
-#ex: .\scripts\Test-LCOW-UVM.ps1 -vb -Action Bench -BootFilesPath C:\ContainerPlat\LinuxBootFiles\ -MountGCSTest -Count 2 -Benchtime '3s'
+#ex: .\scripts\Test-LCOW-UVM.ps1 -vb -Count 2 -Benchtime '3s'
 # benchstat via `go install golang.org/x/perf/cmd/benchstat@latest`
 
 [CmdletBinding()]
@@ -12,6 +12,9 @@ param (
  $Note = '',
 
  # test parameters
+ [switch]
+ $Shuffle,
+
  [int]
  $Count = 1,
 
@@ -26,16 +29,16 @@ param (
  $TestVerbose,
 
  [string]
- $Run = '',
-
- [string]
- $CodePath = '.',
+ $Run,
 
  [string]
  $OutDirectory = '.\test\results',
 
  # uvm parameters
 
+ [string]
+ $UVMBootPath = '.\bin\cmd\uvmboot.exe',
+
  [string]
  $BootFilesPath = 'C:\ContainerPlat\LinuxBootFiles',
 
@@ -61,19 +64,18 @@ param (
  $GCSTestPath = '.\bin\test\gcs.test',
 
  [switch]
- $MountGCSTest,
+ $SkipGCSTestMount,
 
- [string]
- $Feature = ''
+ [string[]]
+ $Features
 )
-
+$ErrorActionPreference = 'Stop'
 Import-Module ( Join-Path $PSScriptRoot Testing.psm1 ) -Force
 
-$CodePath = Resolve-Path $CodePath
-$OutDirectory = Resolve-Path $OutDirectory
 $BootFilesPath = Resolve-Path $BootFilesPath
 $ContainerRootFSPath = Resolve-Path $ContainerRootFSPath
 $GCSTestPath = Resolve-Path $GCSTestPath
+$UVMBootPath = Resolve-Path $UVMBootPath
 
 $shell = ( $Action -eq 'Shell' )
 
@@ -83,7 +85,7 @@ if ( $shell ) {
  $date = Get-Date
  $waitfiles = "$ContainerRootFSMount"
  $gcspath = 'gcs.test'
- if ( $MountGCSTest ) {
+ if ( -not $SkipGCSTestMount ) {
  $waitfiles += ",$GCSTestMount"
  $gcspath = "$GCSTestMount/gcs.test"
  }
@@ -104,26 +106,27 @@ if ( $shell ) {
  -OutDirectory $OutDirectory `
  -Date $date `
  -Note $Note `
+ -Shuffle:$Shuffle `
  -TestVerbose:$TestVerbose `
  -Count $Count `
  -BenchTime $BenchTime `
  -Timeout $Timeout `
  -Run $Run `
- -Feature $Feature `
+ -Features $Features `
  -Verbose:$Verbose
 
  $testcmd += " `'-rootfs-path=$ContainerRootFSMount`' "
  $cmd = $pre + $testcmd
 }
 
-$boot = '.\bin\tool\uvmboot.exe -gcs lcow ' + `
+$boot = "$UVMBootPath -gcs lcow " + `
  '-fwd-stdout -fwd-stderr -output-handling stdout ' + `
  "-boot-files-path $BootFilesPath " + `
  "-root-fs-type $BootFSType " + `
  '-kernel-file vmlinux ' + `
  "-mount-scsi `"$ContainerRootFSPath,$ContainerRootFSMount`" "
 
-if ( $MountGCSTest ) {
+if ( -not $SkipGCSTestMount ) {
  $boot += "-share `"$GCSTestPath,$GCSTestMount`" "
 }
 
@@ -140,8 +143,8 @@ $boot += " -exec `"$cmd`" "
 Invoke-TestCommand `
  -TestCmd $boot `
  -TestCmdPreamble $testcmd `
- -OutputFile (&{ if ( $Action -ne 'Shell' ) { $out } }) `
- -OutputCmd (&{ if ( $Action -eq 'Bench' ) { 'benchstat' } }) `
+ -OutputFile (& { if ( $Action -ne 'Shell' ) { $out } }) `
+ -OutputCmd (& { if ( $Action -eq 'Bench' ) { 'benchstat' } }) `
  -Preamble `
  -Date $Date `
  -Note $Note `

diff --git a/scripts/Testing.psm1 b/scripts/Testing.psm1
@@ -1,10 +1,10 @@
 function New-TestCommand {
  [CmdletBinding()]
  param (
+ [Parameter(Mandatory)]
  [ValidateSet('Test', 'Bench', 'List')]
- [alias('a')]
  [string]
- $Action = 'Bench',
+ $Action,
 
  [Parameter(Mandatory)]
  [string]
@@ -16,73 +16,82 @@ function New-TestCommand {
 
  [Parameter(Mandatory)]
  [string]
- $OutDirectory ,
+ $OutDirectory,
+
+ [string]
+ $OutFile,
 
  [DateTime]
  $Date = (Get-Date),
 
  [string]
- $Note = '',
+ $Note,
 
  # test parameters
- [alias('tv')]
  [switch]
- $TestVerbose = $false,
+ $Shuffle,
+
+ [switch]
+ $TestVerbose,
 
  [int]
- $Count = 1,
+ $Count,
 
  [string]
- $BenchTime = '5s',
+ $BenchTime,
 
  [string]
- $Timeout = '10m',
+ $Timeout,
 
  [string]
- $Run = '',
+ $Run,
 
- [string]
- $Feature = ''
+ [string[]]
+ $Features
  )
-
- $OutDirectory = Resolve-Path $OutDirectory
  Write-Verbose "creating $OutDirectory"
-
  New-Item -ItemType 'directory' -Path $OutDirectory -Force > $null
 
- $testcmd = "$Path `'-test.timeout=$Timeout`' `'-test.shuffle=on`' `'-test.count=$Count`' "
+ $testcmd = "$Path `'-test.timeout=$Timeout`' `'-test.count=$Count`' "
+
+ if ( $Shuffle ) {
+ $testcmd += '''-test.shuffle=on'' '
+ }
 
  if ( $TestVerbose ) {
- $testcmd += ' ''-test.v'' '
+ $testcmd += '''-test.v'' '
  }
 
  switch ( $Action ) {
  'List' {
- if ( $Run -eq '' ) {
+ if ( -not $Run ) {
  $Run = '.'
  }
- $testcmd += " `'-test.list=$Run`' "
+ $testcmd += "`'-test.list=$Run`' "
  }
  'Test' {
- if ( $Run -ne '' ) {
- $testcmd += " `'-test.run=$Run`' "
+ if ( $Run ) {
+ $testcmd += "`'-test.run=$Run`' "
  }
  }
  'Bench' {
- if ( $Run -eq '' ) {
+ if ( -not $Run ) {
  $Run = '.'
  }
- $testcmd += ' ''-test.run=^#'' ''-test.benchmem'' ' + `
- " `'-test.bench=$Run`' `'-test.benchtime=$BenchTime`' "
+ $testcmd += '''-test.run=^#'' ''-test.benchmem'' ' + `
+ "`'-test.bench=$Run`' `'-test.benchtime=$BenchTime`' "
  }
  }
 
- if ( $Feature -ne '' ) {
- $testcmd += " `'-feature=$Feature`' "
+ foreach ( $Feature in $Features ) {
+ $Feature = $Feature -replace ' ', ''
+ if ( $Feature ) {
+ $testcmd += "`'-feature=$Feature`' "
+ }
  }
 
- $f = $Name + '-' + $Action
- if ($Note -ne '' ) {
+ $f = $Name + '-' + ($Action.ToLower())
+ if ( $Note ) {
  $f += '-' + $Note
  }
  $out = Join-Path $OutDirectory "$f-$(Get-Date -Date $date -Format FileDateTime).txt"
@@ -101,7 +110,7 @@ function Invoke-TestCommand {
  $TestCmdPreamble = $TestCmd,
 
  [string]
- $OutputFile = 'nul',
+ $OutputFile = '',
 
  [string]
  $OutputCmd,
@@ -115,30 +124,33 @@ function Invoke-TestCommand {
  [string]
  $Note
  )
+ Write-Verbose "Running command: $TestCmd"
 
- if ($OutputFile -eq '' ) {
+ if ( -not $OutputFile ) {
  $OutputFile = 'nul'
+ } else {
+ Write-Verbose "Saving output to: $OutputFile"
  }
 
- Write-Verbose "Saving output to: $OutputFile"
+
  if ( $Preamble ) {
  & {
  Write-Output "test.date: $(Get-Date -Date $Date -UFormat '%FT%R%Z' -AsUTC)"
- if ( $Note -ne '' ) {
+ if ( $Note ) {
  Write-Output "note: $Note"
  }
  Write-Output "test.command: $TestCmdPreamble"
- Write-Output "pkg.commit: $(git rev-parse HEAD)"
- } | Tee-Object -Append -FilePath $OutputFile
+ if ( Get-Command -ErrorAction Ignore 'git' ) {
+ Write-Output "pkg.commit: $(git rev-parse HEAD 2>$null)"
+ }
+ } | Tee-Object -Encoding utf8 -FilePath $OutputFile
  }
+ Invoke-Expression $TestCmd |
+ Tee-Object -Encoding utf8 -Append -FilePath $OutputFile
 
- Write-Verbose "Running command: $TestCmd"
- Invoke-Expression $TestCmd | Tee-Object -Append -FilePath $OutputFile
-
- if ( $OutputCmd -ne '' -and $OutputFile -ne 'nul' ) {
+ if ( $OutputCmd -and $OutputFile -ne 'nul' ) {
  $oc = "$OutputCmd $OutputFile"
  Write-Verbose "Running command: $oc"
  Invoke-Expression $oc
  }
-
-}
+}