Add DisableExtensionOperations field

This option when set to true ensures no that no VMExtension are configured on the
machine by preventing users to add values to VMExtensions field in machine template,
Disabling the bootstrapping VMExtension and setting AllowExtensionOperations false
in the instance osProfile.

api/v1beta1/azuremachine_types.go

@@ -132,6 +132,12 @@ type AzureMachineSpec struct {
 	// +optional
 	DNSServers []string `json:"dnsServers,omitempty"`
 
+	// DisableExtensionOperations specifies whether extension operations should be disabled on the virtual machine.
+	// Use this setting only if VMExtensions are not supported by your image, as it disables CAPZ bootstrapping extension used for detecting Kubernetes bootstrap failure.
+	// This may only be set to True when no extensions are configured on the virtual machine.
+	// +optional
+	DisableExtensionOperations *bool `json:"disableExtensionOperations,omitempty"`
+
 	// VMExtensions specifies a list of extensions to be added to the virtual machine.
 	// +optional
 	VMExtensions []VMExtension `json:"vmExtensions,omitempty"`

api/v1beta1/azuremachine_validation.go

@@ -24,6 +24,7 @@ import (
 	"github.com/google/uuid"
 	"golang.org/x/crypto/ssh"
 	"k8s.io/apimachinery/pkg/util/validation/field"
+	"k8s.io/utils/ptr"
 	azureutil "sigs.k8s.io/cluster-api-provider-azure/util/azure"
 )
 
@@ -71,6 +72,10 @@ func ValidateAzureMachineSpec(spec AzureMachineSpec) field.ErrorList {
 		allErrs = append(allErrs, errs...)
 	}
 
+	if errs := ValidateVMExtensions(spec.DisableExtensionOperations, spec.VMExtensions, field.NewPath("vmExtensions")); len(errs) > 0 {
+		allErrs = append(allErrs, errs...)
+	}
+
 	return allErrs
 }
 
@@ -471,3 +476,14 @@ func ValidateCapacityReservationGroupID(capacityReservationGroupID *string, fldP
 
 	return allErrs
 }
+
+// ValidateVMExtensions validates the VMExtensions spec.
+func ValidateVMExtensions(disableExtensionOperations *bool, vmExtensions []VMExtension, fldPath *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+
+	if ptr.Deref(disableExtensionOperations, false) && len(vmExtensions) > 0 {
+		allErrs = append(allErrs, field.Forbidden(field.NewPath("AzureMachineTemplate", "spec", "template", "spec", "VMExtensions"), "VMExtensions must be empty when DisableExtensionOperations is true"))
+	}
+
+	return allErrs
+}

api/v1beta1/azuremachine_webhook.go

@@ -213,6 +213,13 @@ func (mw *azureMachineWebhook) ValidateUpdate(ctx context.Context, oldObj, newOb
 		allErrs = append(allErrs, err)
 	}
 
+	if err := webhookutils.ValidateImmutable(
+		field.NewPath("spec", "DisableExtensionOperations"),
+		old.Spec.DisableExtensionOperations,
+		m.Spec.DisableExtensionOperations); err != nil {
+		allErrs = append(allErrs, err)
+	}
+
 	if len(allErrs) == 0 {
 		return nil, nil
 	}

api/v1beta1/azuremachine_webhook_test.go

@@ -230,6 +230,16 @@ func TestAzureMachine_ValidateCreate(t *testing.T) {
 			machine: createMachineWithCapacityReservaionGroupID("invalid-capacity-group-id"),
 			wantErr: true,
 		},
+		{
+			name:    "azuremachine with DisableExtensionOperations true and without VMExtensions",
+			machine: createMachineWithDisableExtenionOperations(),
+			wantErr: false,
+		},
+		{
+			name:    "azuremachine with DisableExtensionOperations true and with VMExtension",
+			machine: createMachineWithDisableExtenionOperationsAndHasExtension(),
+			wantErr: true,
+		},
 	}
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
@@ -778,6 +788,34 @@ func TestAzureMachine_ValidateUpdate(t *testing.T) {
 			},
 			wantErr: true,
 		},
+		{
+			name: "invalidTest: azuremachine.spec.disableExtensionOperations is immutable",
+			oldMachine: &AzureMachine{
+				Spec: AzureMachineSpec{
+					DisableExtensionOperations: ptr.To(true),
+				},
+			},
+			newMachine: &AzureMachine{
+				Spec: AzureMachineSpec{
+					DisableExtensionOperations: ptr.To(false),
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "validTest: azuremachine.spec.disableExtensionOperations is immutable",
+			oldMachine: &AzureMachine{
+				Spec: AzureMachineSpec{
+					DisableExtensionOperations: ptr.To(true),
+				},
+			},
+			newMachine: &AzureMachine{
+				Spec: AzureMachineSpec{
+					DisableExtensionOperations: ptr.To(true),
+				},
+			},
+			wantErr: false,
+		},
 		{
 			name: "validTest: azuremachine.spec.networkInterfaces is immutable",
 			oldMachine: &AzureMachine{
@@ -1154,3 +1192,28 @@ func createMachineWithCapacityReservaionGroupID(capacityReservationGroupID strin
 		},
 	}
 }
+
+func createMachineWithDisableExtenionOperationsAndHasExtension() *AzureMachine {
+	return &AzureMachine{
+		Spec: AzureMachineSpec{
+			SSHPublicKey:               validSSHPublicKey,
+			OSDisk:                     validOSDisk,
+			DisableExtensionOperations: ptr.To(true),
+			VMExtensions: []VMExtension{{
+				Name:      "test-extension",
+				Publisher: "test-publiher",
+				Version:   "v0.0.1-test",
+			}},
+		},
+	}
+}
+
+func createMachineWithDisableExtenionOperations() *AzureMachine {
+	return &AzureMachine{
+		Spec: AzureMachineSpec{
+			SSHPublicKey:               validSSHPublicKey,
+			OSDisk:                     validOSDisk,
+			DisableExtensionOperations: ptr.To(true),
+		},
+	}
+}

api/v1beta1/azuremachinetemplate_webhook.go

@@ -24,6 +24,7 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/validation/field"
+	"k8s.io/utils/ptr"
 	"sigs.k8s.io/cluster-api/util/topology"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
@@ -85,6 +86,10 @@ func (r *AzureMachineTemplate) ValidateCreate(ctx context.Context, obj runtime.O
 		}
 	}
 
+	if ptr.Deref(r.Spec.Template.Spec.DisableExtensionOperations, false) && len(r.Spec.Template.Spec.VMExtensions) > 0 {
+		allErrs = append(allErrs, field.Forbidden(field.NewPath("AzureMachineTemplate", "spec", "template", "spec", "VMExtensions"), "VMExtensions must be empty when DisableExtensionOperations is true"))
+	}
+
 	if len(allErrs) == 0 {
 		return nil, nil
 	}

api/v1beta1/azuremachinetemplate_webhook_test.go

@@ -143,6 +143,16 @@ func TestAzureMachineTemplate_ValidateCreate(t *testing.T) {
 			machineTemplate: createAzureMachineTemplateFromMachine(createMachineWithRoleAssignmentName()),
 			wantErr:         true,
 		},
+		{
+			name:            "azuremachinetemplate with DisableExtensionOperations true and without VMExtensions",
+			machineTemplate: createAzureMachineTemplateFromMachine(createMachineWithDisableExtenionOperations()),
+			wantErr:         false,
+		},
+		{
+			name:            "azuremachinetempalte with DisableExtensionOperations true and with VMExtension",
+			machineTemplate: createAzureMachineTemplateFromMachine(createMachineWithDisableExtenionOperationsAndHasExtension()),
+			wantErr:         true,
+		},
 		{
 			name:            "azuremachinetemplate without RoleAssignmentName",
 			machineTemplate: createAzureMachineTemplateFromMachine(createMachineWithoutRoleAssignmentName()),

azure/scope/machine.go

@@ -178,6 +178,7 @@ func (m *MachineScope) VMSpec() azure.ResourceSpecGetter {
 		SpotVMOptions:              m.AzureMachine.Spec.SpotVMOptions,
 		SecurityProfile:            m.AzureMachine.Spec.SecurityProfile,
 		DiagnosticsProfile:         m.AzureMachine.Spec.Diagnostics,
+		DisableExtensionOperations: ptr.Deref(m.AzureMachine.Spec.DisableExtensionOperations, false),
 		AdditionalTags:             m.AdditionalTags(),
 		AdditionalCapabilities:     m.AzureMachine.Spec.AdditionalCapabilities,
 		CapacityReservationGroupID: m.GetCapacityReservationGroupID(),
@@ -374,6 +375,10 @@ func (m *MachineScope) HasSystemAssignedIdentity() bool {
 
 // VMExtensionSpecs returns the VM extension specs.
 func (m *MachineScope) VMExtensionSpecs() []azure.ResourceSpecGetter {
+	if ptr.Deref(m.AzureMachine.Spec.DisableExtensionOperations, false) {
+		return []azure.ResourceSpecGetter{}
+	}
+
 	var extensionSpecs = []azure.ResourceSpecGetter{}
 	for _, extension := range m.AzureMachine.Spec.VMExtensions {
 		extensionSpecs = append(extensionSpecs, &vmextensions.VMExtensionSpec{

azure/scope/machine_test.go

@@ -598,6 +598,44 @@ func TestMachineScope_VMExtensionSpecs(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "If OS type is Linux and cloud is AzurePublicCloud and DisableExtensionOperations is true, it returns empty",
+			machineScope: MachineScope{
+				Machine: &clusterv1.Machine{},
+				AzureMachine: &infrav1.AzureMachine{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "machine-name",
+					},
+					Spec: infrav1.AzureMachineSpec{
+						DisableExtensionOperations: ptr.To(true),
+						OSDisk: infrav1.OSDisk{
+							OSType: "Linux",
+						},
+					},
+				},
+				ClusterScoper: &ClusterScope{
+					AzureClients: AzureClients{
+						EnvironmentSettings: auth.EnvironmentSettings{
+							Environment: azureautorest.Environment{
+								Name: azureautorest.PublicCloud.Name,
+							},
+						},
+					},
+					AzureCluster: &infrav1.AzureCluster{
+						Spec: infrav1.AzureClusterSpec{
+							ResourceGroup: "my-rg",
+							AzureClusterClassSpec: infrav1.AzureClusterClassSpec{
+								Location: "westus",
+							},
+						},
+					},
+				},
+				cache: &MachineCache{
+					VMSKU: resourceskus.SKU{},
+				},
+			},
+			want: []azure.ResourceSpecGetter{},
+		},
 		{
 			name: "If OS type is Linux and cloud is not AzurePublicCloud, it returns empty",
 			machineScope: MachineScope{

azure/services/virtualmachines/spec.go

@@ -53,6 +53,7 @@ type VMSpec struct {
 	AdditionalTags             infrav1.Tags
 	AdditionalCapabilities     *infrav1.AdditionalCapabilities
 	DiagnosticsProfile         *infrav1.Diagnostics
+	DisableExtensionOperations bool
 	CapacityReservationGroupID string
 	SKU                        resourceskus.SKU
 	Image                      *infrav1.Image
@@ -263,9 +264,10 @@ func (s *VMSpec) generateOSProfile() (*armcompute.OSProfile, error) {
 	}
 
 	osProfile := &armcompute.OSProfile{
-		ComputerName:  ptr.To(s.Name),
-		AdminUsername: ptr.To(azure.DefaultUserName),
-		CustomData:    ptr.To(s.BootstrapData),
+		ComputerName:             ptr.To(s.Name),
+		AdminUsername:            ptr.To(azure.DefaultUserName),
+		CustomData:               ptr.To(s.BootstrapData),
+		AllowExtensionOperations: ptr.To(!s.DisableExtensionOperations),
 	}
 
 	switch s.OSDisk.OSType {

config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachines.yaml

@@ -238,6 +238,12 @@ spec:
                     - storageAccountType
                     type: object
                 type: object
+              disableExtensionOperations:
+                description: |-
+                  DisableExtensionOperations specifies whether extension operations should be disabled on the virtual machine.
+                  Use this setting only if VMExtensions are not supported by your image, as it disables CAPZ bootstrapping extension used for detecting Kubernetes bootstrap failure.
+                  This may only be set to True when no extensions are configured on the virtual machine.
+                type: boolean
               dnsServers:
                 description: DNSServers adds a list of DNS Server IP addresses to
                   the VM NICs.

config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachinetemplates.yaml

@@ -253,6 +253,12 @@ spec:
                             - storageAccountType
                             type: object
                         type: object
+                      disableExtensionOperations:
+                        description: |-
+                          DisableExtensionOperations specifies whether extension operations should be disabled on the virtual machine.
+                          Use this setting only if VMExtensions are not supported by your image, as it disables CAPZ bootstrapping extension used for detecting Kubernetes bootstrap failure.
+                          This may only be set to True when no extensions are configured on the virtual machine.
+                        type: boolean
                       dnsServers:
                         description: DNSServers adds a list of DNS Server IP addresses
                           to the VM NICs.