From 01f602195983451bc83e72f4214af2cbc495aa94 Mon Sep 17 00:00:00 2001 From: Billy O'Neal Date: Fri, 24 May 2024 22:22:47 -0700 Subject: [PATCH] Patch Tuesday for May 2024 (Again) (#38891) * Update AzCopy, PowerShell Core, and VS. --- scripts/azure-pipelines/android/Dockerfile | 2 +- scripts/azure-pipelines/azure-pipelines.yml | 18 ++--------- .../azure-pipelines/linux/azure-pipelines.yml | 4 +-- .../windows/azure-pipelines.yml | 5 ++- .../azure-pipelines/windows/create-image.ps1 | 32 +++++++++++-------- .../azure-pipelines/windows/deploy-azcopy.ps1 | 4 --- .../azure-pipelines/windows/deploy-cuda.ps1 | 25 --------------- .../azure-pipelines/windows/deploy-cudnn.ps1 | 28 ++++++++++++++++ .../azure-pipelines/windows/deploy-pwsh.ps1 | 2 +- .../windows/deploy-visual-studio.ps1 | 2 +- .../azure-pipelines/windows/test-cudnn.ps1 | 5 +++ 11 files changed, 60 insertions(+), 67 deletions(-) delete mode 100644 scripts/azure-pipelines/windows/deploy-azcopy.ps1 create mode 100644 scripts/azure-pipelines/windows/deploy-cudnn.ps1 create mode 100644 scripts/azure-pipelines/windows/test-cudnn.ps1 diff --git a/scripts/azure-pipelines/android/Dockerfile b/scripts/azure-pipelines/android/Dockerfile index 3f284ffc47ebe7..826a532cc60b24 100644 --- a/scripts/azure-pipelines/android/Dockerfile +++ b/scripts/azure-pipelines/android/Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1.4 # DisableDockerDetector "Used to build the container deployed to Azure Container Registry" -FROM ubuntu:focal-20240216 +FROM ubuntu:focal-20240427 ADD https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb /packages-microsoft-prod.deb ADD https://dl.google.com/android/repository/android-ndk-r25c-linux.zip /android-ndk-r25c-linux.zip diff --git a/scripts/azure-pipelines/azure-pipelines.yml b/scripts/azure-pipelines/azure-pipelines.yml index cfca136a686cc7..0499445c790160 100644 --- a/scripts/azure-pipelines/azure-pipelines.yml +++ b/scripts/azure-pipelines/azure-pipelines.yml @@ -2,11 +2,7 @@ # SPDX-License-Identifier: MIT # variables: - linux-pool: 'PrLin-1ES-Pool' - windows-pool: 'PrWin-1ES' - osx-pool: 'PrOsx-2024-01-18' - osx-arm64-pool: 'PrOsx-2024-01-22-arm64' - linux-docker-image: 'vcpkgandroidwus3.azurecr.io/vcpkg-android:2024-04-11' + linux-docker-image: 'vcpkgandroidwus3.azurecr.io/vcpkg-android:2024-05-17' parameters: - name: vcpkgToolSha @@ -37,70 +33,62 @@ jobs: - template: windows/azure-pipelines.yml parameters: jobName: x86_windows - poolName: $(windows-pool) vcpkgToolSha: ${{ parameters.vcpkgToolSha }} tripletPattern: ${{ parameters.tripletPattern }} - template: windows/azure-pipelines.yml parameters: jobName: x64_windows - poolName: $(windows-pool) vcpkgToolSha: ${{ parameters.vcpkgToolSha }} tripletPattern: ${{ parameters.tripletPattern }} - template: windows/azure-pipelines.yml parameters: jobName: x64_windows_static - poolName: $(windows-pool) vcpkgToolSha: ${{ parameters.vcpkgToolSha }} tripletPattern: ${{ parameters.tripletPattern }} - template: windows/azure-pipelines.yml parameters: jobName: x64_windows_static_md - poolName: $(windows-pool) vcpkgToolSha: ${{ parameters.vcpkgToolSha }} tripletPattern: ${{ parameters.tripletPattern }} - template: windows/azure-pipelines.yml parameters: jobName: x64_uwp - poolName: $(windows-pool) vcpkgToolSha: ${{ parameters.vcpkgToolSha }} tripletPattern: ${{ parameters.tripletPattern }} - template: windows/azure-pipelines.yml parameters: jobName: arm64_windows - poolName: $(windows-pool) vcpkgToolSha: ${{ parameters.vcpkgToolSha }} tripletPattern: ${{ parameters.tripletPattern }} - template: windows/azure-pipelines.yml parameters: jobName: arm64_uwp - poolName: $(windows-pool) vcpkgToolSha: ${{ parameters.vcpkgToolSha }} tripletPattern: ${{ parameters.tripletPattern }} - template: osx/azure-pipelines.yml parameters: jobName: x64_osx - poolName: $(osx-pool) + poolName: 'PrOsx-2024-01-18' vcpkgToolSha: ${{ parameters.vcpkgToolSha }} tripletPattern: ${{ parameters.tripletPattern }} - template: osx/azure-pipelines.yml parameters: jobName: arm64_osx - poolName: $(osx-arm64-pool) + poolName: 'PrOsx-2024-01-22-arm64' vcpkgToolSha: ${{ parameters.vcpkgToolSha }} tripletPattern: ${{ parameters.tripletPattern }} - template: linux/azure-pipelines.yml parameters: jobName: x64_linux - poolName: $(linux-pool) vcpkgToolSha: ${{ parameters.vcpkgToolSha }} tripletPattern: ${{ parameters.tripletPattern }} diff --git a/scripts/azure-pipelines/linux/azure-pipelines.yml b/scripts/azure-pipelines/linux/azure-pipelines.yml index 7dcd3d53b6fdf4..dbd857ef72921a 100644 --- a/scripts/azure-pipelines/linux/azure-pipelines.yml +++ b/scripts/azure-pipelines/linux/azure-pipelines.yml @@ -10,8 +10,6 @@ parameters: - name: jobName type: string default: 'x64_linux' - - name: poolName - type: string - name: tripletPattern displayName: 'Enable the triplets which contain this substring' type: string @@ -21,7 +19,7 @@ jobs: - job: x64_linux condition: and(succeeded(), contains('^${{ replace(parameters.jobName, '_', '-') }}$', '${{ parameters.tripletPattern }}')) pool: - name: ${{ parameters.poolName }} + name: PrLin-1ES-Pool workspace: clean: resources timeoutInMinutes: 1440 # 1 day diff --git a/scripts/azure-pipelines/windows/azure-pipelines.yml b/scripts/azure-pipelines/windows/azure-pipelines.yml index 39a325c1f96869..ad3e5910e70342 100644 --- a/scripts/azure-pipelines/windows/azure-pipelines.yml +++ b/scripts/azure-pipelines/windows/azure-pipelines.yml @@ -10,8 +10,6 @@ parameters: - name: jobName type: string default: 'x86_windows' - - name: poolName - type: string - name: tripletPattern displayName: 'Enable the triplets which contain this substring' type: string @@ -21,7 +19,8 @@ jobs: - job: ${{ parameters.jobName }} condition: and(succeeded(), contains('^${{ replace(parameters.jobName, '_', '-') }}$', '${{ parameters.tripletPattern }}')) pool: - name: ${{ parameters.poolName }} + name: PrWin-1ES + demands: ImageVersionOverride -equals 2024.05.24 workspace: clean: resources timeoutInMinutes: 2880 # 2 days diff --git a/scripts/azure-pipelines/windows/create-image.ps1 b/scripts/azure-pipelines/windows/create-image.ps1 index 68f7afddbe7668..f8c038c2e6f04a 100644 --- a/scripts/azure-pipelines/windows/create-image.ps1 +++ b/scripts/azure-pipelines/windows/create-image.ps1 @@ -24,7 +24,7 @@ $WindowsServerSku = '2022-datacenter-azure-edition' $ErrorActionPreference = 'Stop' $ProgressActivity = 'Creating Windows Image' -$TotalProgress = 17 +$TotalProgress = 18 $CurrentProgress = 1 # Assigning this to another variable helps when running the commands in this script manually for @@ -87,11 +87,11 @@ Write-Progress ` $VcpkgImageMintingAccount = Get-AzStorageAccount -ResourceGroupName 'vcpkg-image-minting' -Name 'vcpkgimageminting' -# Grant 'Storage Blob Data Reader' (RoleDefinitionId 2a2b9908-6ea1-4ae2-8e65-a410df84e7d1) to the VM -New-AzRoleAssignment ` - -Scope $VcpkgImageMintingAccount.ID ` - -RoleDefinitionId '2a2b9908-6ea1-4ae2-8e65-a410df84e7d1' ` - -ObjectId $VMCreated.Identity.PrincipalId +$CudnnStorageContext = New-AzStorageContext -StorageAccountName 'vcpkgimageminting' -UseConnectedAccount +$StartTime = Get-Date +$ExpiryTime = $StartTime.AddDays(1) +$CudnnSas = New-AzStorageContainerSASToken -Name 'assets' -Permission r -StartTime $StartTime -ExpiryTime $ExpiryTime -Context $CudnnStorageContext +$CudnnUrl = "https://vcpkgimageminting.blob.core.windows.net/assets/cudnn-windows-x86_64-8.8.1.3_cuda12-archive.zip?$CudnnSas" #################################################################################################### Write-Progress ` @@ -127,7 +127,8 @@ Write-Host "deploy-psexec.ps1 output: $($DeployPsExecResult.value.Message)" function Invoke-ScriptWithPrefix { param( [string]$ScriptName, - [switch]$AddAdminPw + [switch]$AddAdminPw, + [string]$CudnnUrl = $null ) Write-Progress ` @@ -146,6 +147,10 @@ function Invoke-ScriptWithPrefix { } $script = $script.Replace('# REPLACE WITH UTILITY-PREFIX.ps1', $UtilityPrefixContent); + if (-not [string]::IsNullOrEmpty($CudnnUrl)) { + $script = $script.Replace('# REPLACE WITH CudnnUrl', "`$CudnnUrl = `"$CudnnUrl`"") + } + Set-Content -Path $tempScriptFilename -Value $script -Encoding utf8NoBOM $parameter = $null @@ -166,8 +171,6 @@ function Invoke-ScriptWithPrefix { } } -Invoke-ScriptWithPrefix -ScriptName 'deploy-azcopy.ps1' - #################################################################################################### Invoke-ScriptWithPrefix -ScriptName 'deploy-windows-sdks.ps1' -AddAdminPw @@ -180,6 +183,12 @@ Invoke-ScriptWithPrefix -ScriptName 'deploy-mpi.ps1' -AddAdminPw #################################################################################################### Invoke-ScriptWithPrefix -ScriptName 'deploy-cuda.ps1' -AddAdminPw +#################################################################################################### +Invoke-ScriptWithPrefix -ScriptName 'deploy-cudnn.ps1' -CudnnUrl $CudnnUrl + +#################################################################################################### +Invoke-ScriptWithPrefix -ScriptName 'test-cudnn.ps1' + #################################################################################################### Invoke-ScriptWithPrefix -ScriptName 'deploy-inteloneapi.ps1' -AddAdminPw @@ -256,11 +265,6 @@ Write-Progress ` -Status 'Deleting unused temporary resources' ` -PercentComplete (100 / $TotalProgress * $CurrentProgress++) -Remove-AzRoleAssignment ` - -Scope $VcpkgImageMintingAccount.ID ` - -RoleDefinitionId '2a2b9908-6ea1-4ae2-8e65-a410df84e7d1' ` - -ObjectId $VMCreated.Identity.PrincipalId - Remove-AzVM -Id $VMCreated.ID -Force Remove-AzDisk -ResourceGroupName 'vcpkg-image-minting' -Name $VMCreatedOsDisk -Force Remove-AzNetworkInterface -ResourceGroupName 'vcpkg-image-minting' -Name $NicName -Force diff --git a/scripts/azure-pipelines/windows/deploy-azcopy.ps1 b/scripts/azure-pipelines/windows/deploy-azcopy.ps1 deleted file mode 100644 index 4d1eb9b660d9d0..00000000000000 --- a/scripts/azure-pipelines/windows/deploy-azcopy.ps1 +++ /dev/null @@ -1,4 +0,0 @@ -$azcopyZipPath = "$PSScriptRoot\azcopyv10.zip" -& curl.exe -L -o $azcopyZipPath 'https://azcopyvnext.azureedge.net/releases/release-10.23.0-20240129/azcopy_windows_amd64_10.23.0.zip' -Expand-Archive -LiteralPath $azcopyZipPath -DestinationPath $env:PROGRAMFILES -Remove-Item -LiteralPath $azcopyZipPath -Force diff --git a/scripts/azure-pipelines/windows/deploy-cuda.ps1 b/scripts/azure-pipelines/windows/deploy-cuda.ps1 index a3249d06c9e0bc..2bb017ee9fd767 100644 --- a/scripts/azure-pipelines/windows/deploy-cuda.ps1 +++ b/scripts/azure-pipelines/windows/deploy-cuda.ps1 @@ -7,9 +7,6 @@ # If you are running this script outside of our Azure VMs, you will need to download cudnn from NVIDIA and place # it next to this script. -$CudnnUrl = 'https://vcpkgimageminting.blob.core.windows.net/assets/cudnn-windows-x86_64-8.8.1.3_cuda12-archive.zip' - -$CudnnLocalZipPath = "$PSScriptRoot\cudnn-windows-x86_64-8.8.1.3_cuda12-archive.zip" $CudaUrl = 'https://developer.download.nvidia.com/compute/cuda/12.1.0/network_installers/cuda_12.1.0_windows_network.exe' @@ -82,25 +79,3 @@ catch { Write-Error "Failed to install CUDA! $($_.Exception.Message)" throw } - -try { - if (Test-Path $CudnnLocalZipPath) { - $cudnnZipPath = $CudnnLocalZipPath - } else { - Write-Host 'Attempting to download cudnn. If this fails, you need to agree to NVidia''s EULA, download cudnn, and place it next to this script.' - $cudnnZipPath = Get-TempFilePath -Extension 'zip' - $env:AZCOPY_AUTO_LOGIN_TYPE = 'MSI' - & "$env:PROGRAMFILES\azcopy_windows_amd64_10.23.0\azcopy.exe" copy $CudnnUrl $cudnnZipPath - if ($LASTEXITCODE -ne 0) { - throw 'Failed to download cudnn!' - } - } - - Write-Host "Installing CUDNN to $destination..." - tar.exe -xvf "$cudnnZipPath" --strip 1 --directory "$destination" - Write-Host 'Installation successful!' -} -catch { - Write-Error "Failed to install CUDNN! $($_.Exception.Message)" - throw -} diff --git a/scripts/azure-pipelines/windows/deploy-cudnn.ps1 b/scripts/azure-pipelines/windows/deploy-cudnn.ps1 new file mode 100644 index 00000000000000..58e278973c1bec --- /dev/null +++ b/scripts/azure-pipelines/windows/deploy-cudnn.ps1 @@ -0,0 +1,28 @@ +# REPLACE WITH UTILITY-PREFIX.ps1 + +# REPLACE WITH CudnnUrl + +$destination = "$env:ProgramFiles\NVIDIA GPU Computing Toolkit\CUDA\v12.1" + +$CudnnLocalZipPath = "$PSScriptRoot\cudnn-windows-x86_64-8.8.1.3_cuda12-archive.zip" + +try { + if (Test-Path $CudnnLocalZipPath) { + $cudnnZipPath = $CudnnLocalZipPath + } else { + Write-Host 'Attempting to download cudnn. If this fails, you need to agree to NVidia''s EULA, download cudnn, and place it next to this script.' + $cudnnZipPath = Get-TempFilePath -Extension 'zip' + & curl.exe -L -o $cudnnZipPath $CudnnUrl + if ($LASTEXITCODE -ne 0) { + throw 'Failed to download cudnn!' + } + } + + Write-Host "Installing CUDNN to $destination..." + tar.exe -xvf "$cudnnZipPath" --strip 1 --directory "$destination" + Write-Host 'Installation successful!' +} +catch { + Write-Error "Failed to install CUDNN! $($_.Exception.Message)" + throw +} diff --git a/scripts/azure-pipelines/windows/deploy-pwsh.ps1 b/scripts/azure-pipelines/windows/deploy-pwsh.ps1 index 2bb6d25971a5f1..6f135b6fa5b06d 100644 --- a/scripts/azure-pipelines/windows/deploy-pwsh.ps1 +++ b/scripts/azure-pipelines/windows/deploy-pwsh.ps1 @@ -5,5 +5,5 @@ # REPLACE WITH UTILITY-PREFIX.ps1 -$PwshUrl = 'https://github.com/PowerShell/PowerShell/releases/download/v7.4.1/PowerShell-7.4.1-win-x64.msi' +$PwshUrl = 'https://github.com/PowerShell/PowerShell/releases/download/v7.4.2/PowerShell-7.4.2-win-x64.msi' InstallMSI -Url $PwshUrl -Name 'PowerShell Core' diff --git a/scripts/azure-pipelines/windows/deploy-visual-studio.ps1 b/scripts/azure-pipelines/windows/deploy-visual-studio.ps1 index 31cecbbd2a791d..07170ab727112e 100644 --- a/scripts/azure-pipelines/windows/deploy-visual-studio.ps1 +++ b/scripts/azure-pipelines/windows/deploy-visual-studio.ps1 @@ -6,7 +6,7 @@ # REPLACE WITH UTILITY-PREFIX.ps1 # See https://learn.microsoft.com/en-us/visualstudio/releases/2022/release-history -$VisualStudioBootstrapperUrl = 'https://download.visualstudio.microsoft.com/download/pr/ec7bd8ef-2c51-4e4f-a83f-9087ffbe8b76/f7fdf51d7c15b65c60f0f358776576f4423417da603ef46a213bd69ad463e9e2/vs_Enterprise.exe' +$VisualStudioBootstrapperUrl = 'https://download.visualstudio.microsoft.com/download/pr/a8a3940c-d415-4078-8df8-6af787f56dfa/ff486670bce61323e52b208ecbb71dc05b034c8bf156d0b7bfc0ad67b2611445/vs_Enterprise.exe' $Workloads = @( 'Microsoft.VisualStudio.Workload.NativeDesktop', 'Microsoft.VisualStudio.Workload.Universal', diff --git a/scripts/azure-pipelines/windows/test-cudnn.ps1 b/scripts/azure-pipelines/windows/test-cudnn.ps1 new file mode 100644 index 00000000000000..8ce2d7634b1eb0 --- /dev/null +++ b/scripts/azure-pipelines/windows/test-cudnn.ps1 @@ -0,0 +1,5 @@ +if (Test-Path "$env:ProgramFiles\NVIDIA GPU Computing Toolkit\CUDA\v12.1\include\cudnn.h") { + Write-Host 'cudnn appears correctly installed' +} else { + Write-Error 'cudnn appears broken!' +}