feat: BYO Subnet Per AKSNodeClass + Subnet Drift (#486)

* feat: Azure CNI Overlay support for specifying SubnetID in the AKSNodeClass

* fix: removing topology

* feat: mark nodeclaims as drifted if the subnet changes

* refactor: moving NodeResourceGroup into options for subnet drift

* test: refactoring test options to use NodeResourceGroup

* fix: modifying test suite

* test: drift e2e test

* test: updating expectations to include an extra validation of the nodeclaim status conditions

* refactor: rename VnetSubnetID to VNETSubnetID

* refactor: updating description of VNETSubnetID

* fix: removing checks for pod and node being removed

* refactor: removing pod annotation

* refactor: using primary ipConfig each time

* refactor: removing comment

* refactor: changing name of the function

* fix: adding stricter resource group validation and subsequent testing

* karpenter is a singular possessive noun so the apostrophe should be before the s

* refactor: renaming to crd validation since we both validate the nodeclass and nodepool in this file

* refactor: loosening validation on names

* fix: merge sadness

pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml

@@ -67,6 +67,12 @@ spec:
                   type: string
                 description: Tags to be applied on Azure resources like instances.
                 type: object
+              vnetSubnetID:
+                description: |-
+                  VNETSubnetID is the subnet used by nics provisioned with this nodeclass.
+                  If not specified, we will use the default --vnet-subnet-id specified in karpenter's options config
+                pattern: (?i)^\/subscriptions\/[^\/]+\/resourceGroups\/[a-zA-Z0-9_\-().]{0,89}[a-zA-Z0-9_\-()]\/providers\/Microsoft\.Network\/virtualNetworks\/[^\/]+\/subnets\/[^\/]+$
+                type: string
             type: object
           status:
             description: AKSNodeClassStatus contains the resolved state of the AKSNodeClass

pkg/apis/v1alpha2/aksnodeclass.go

@@ -27,6 +27,11 @@ import (
 // AKSNodeClassSpec is the top level specification for the AKS Karpenter Provider.
 // This will contain configuration necessary to launch instances in AKS.
 type AKSNodeClassSpec struct {
+	// VNETSubnetID is the subnet used by nics provisioned with this nodeclass.
+	// If not specified, we will use the default --vnet-subnet-id specified in karpenter's options config
+	// +kubebuilder:validation:Pattern=`(?i)^\/subscriptions\/[^\/]+\/resourceGroups\/[a-zA-Z0-9_\-().]{0,89}[a-zA-Z0-9_\-()]\/providers\/Microsoft\.Network\/virtualNetworks\/[^\/]+\/subnets\/[^\/]+$`
+	// +optional
+	VNETSubnetID *string `json:"vnetSubnetID,omitempty"`
 	// +kubebuilder:default=128
 	// +kubebuilder:validation:Minimum=100
 	// osDiskSizeGB is the size of the OS disk in GB.

pkg/apis/v1alpha2/nodepool_validation_cel_test.go → pkg/apis/v1alpha2/crd_validation_cel_test.go

@@ -59,6 +59,39 @@ var _ = Describe("CEL/Validation", func() {
 			},
 		}
 	})
+	Context("VnetSubnetID", func() {
+		DescribeTable("Should only accept valid VnetSubnetID", func(vnetSubnetID string, expected bool) {
+			nodeClass := &v1alpha2.AKSNodeClass{
+				ObjectMeta: metav1.ObjectMeta{Name: strings.ToLower(randomdata.SillyName())},
+				Spec: v1alpha2.AKSNodeClassSpec{
+					VNETSubnetID: &vnetSubnetID,
+				},
+			}
+			if expected {
+				Expect(env.Client.Create(ctx, nodeClass)).To(Succeed())
+			} else {
+				Expect(env.Client.Create(ctx, nodeClass)).ToNot(Succeed())
+			}
+		},
+			Entry("valid VnetSubnetID", "/subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/rgname/providers/Microsoft.Network/virtualNetworks/vnet/subnets/subnet", true),
+			Entry("should allow mixed casing in all the names", "/subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/rgName/providers/Microsoft.Network/virtualNetworks/vnetName/subnets/subnetName", true),
+			Entry("valid format with different subnet name", "/subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/rgname/providers/Microsoft.Network/virtualNetworks/vnet/subnets/anotherSubnet", true),
+			Entry("valid format with uppercase subnet name", "/subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/rgname/providers/Microsoft.Network/virtualNetworks/vnet/subnets/SUBNET", true),
+			Entry("valid format with mixed-case resource group and subnet name", "/subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/MyResourceGroup/providers/Microsoft.Network/virtualNetworks/MyVirtualNetwork/subnets/MySubnet", true),
+			Entry("missing resourceGroups in path", "/subscriptions/12345678-1234-1234-1234-123456789012/rgname/providers/Microsoft.Network/virtualNetworks/vnet/subnets/subnet", false),
+			Entry("invalid provider in path", "/subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/rgname/providers/Microsoft.Storage/virtualNetworks/vnet/subnets/subnet", false),
+			Entry("missing virtualNetworks in path", "/subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/rgname/providers/Microsoft.Network/subnets/subnet", false),
+			Entry("valid VnetSubnetID at max length", "/subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/"+strings.Repeat("a", 63)+"/providers/Microsoft.Network/virtualNetworks/"+strings.Repeat("b", 63)+"/subnets/"+strings.Repeat("c", 63), true),
+			Entry("valid resource group name 'my-resource_group'", "/subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/my-resource_group/providers/Microsoft.Network/virtualNetworks/vnet/subnets/subnet", true),
+			Entry("valid resource group name starting with dot '.starting.with.dot'", "/subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/.starting.with.dot/providers/Microsoft.Network/virtualNetworks/vnet/subnets/subnet", true),
+			Entry("valid resource group name ending with hyphen 'ends-with-hyphen-'", "/subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/ends-with-hyphen-/providers/Microsoft.Network/virtualNetworks/vnet/subnets/subnet", true),
+			Entry("valid resource group name with parentheses 'contains.(parentheses)'", "/subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/contains.(parentheses)/providers/Microsoft.Network/virtualNetworks/vnet/subnets/subnet", true),
+			Entry("valid resource group name 'valid.name-with-multiple.characters'", "/subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/valid.name-with-multiple.characters/providers/Microsoft.Network/virtualNetworks/vnet/subnets/subnet", true),
+			Entry("invalid resource group name ending with dot 'ends.with.dot.'", "/subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/ends.with.dot./providers/Microsoft.Network/virtualNetworks/vnet/subnets/subnet", false),
+			Entry("invalid resource group name with invalid character 'invalid#character'", "/subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/invalid#character/providers/Microsoft.Network/virtualNetworks/vnet/subnets/subnet", false),
+			Entry("invalid resource group name with unsupported chars 'name@with*unsupported&chars'", "/subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/name@with*unsupported&chars/providers/Microsoft.Network/virtualNetworks/vnet/subnets/subnet", false),
+		)
+	})
 	Context("Requirements", func() {
 		It("should allow restricted domains exceptions", func() {
 			oldNodePool := nodePool.DeepCopy()

pkg/cloudprovider/cloudprovider.go

@@ -206,6 +206,13 @@ func (c *CloudProvider) IsDrifted(ctx context.Context, nodeClaim *corev1beta1.No
 	if imageVersionDrifted != "" {
 		return imageVersionDrifted, nil
 	}
+	subnetDrifted, err := c.isSubnetDrifted(ctx, nodeClaim, nodeClass)
+	if err != nil {
+		return "", err
+	}
+	if subnetDrifted != "" {
+		return subnetDrifted, nil
+	}
 	return "", nil
 }
 

pkg/cloudprovider/drift.go

@@ -21,11 +21,15 @@ import (
 	"fmt"
 	"strings"
 
-	"knative.dev/pkg/logging"
-
+	sdkerrors "github.com/Azure/azure-sdk-for-go-extensions/pkg/errors"
+	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
 	"github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2"
+	"github.com/Azure/karpenter-provider-azure/pkg/operator/options"
 	"github.com/Azure/karpenter-provider-azure/pkg/providers/imagefamily"
+	"github.com/Azure/karpenter-provider-azure/pkg/providers/instance"
 	"github.com/Azure/karpenter-provider-azure/pkg/utils"
+	"github.com/samber/lo"
+	"knative.dev/pkg/logging"
 
 	v1 "k8s.io/api/core/v1"
 
@@ -38,6 +42,7 @@ import (
 const (
 	K8sVersionDrift   cloudprovider.DriftReason = "K8sVersionDrift"
 	ImageVersionDrift cloudprovider.DriftReason = "ImageVersionDrift"
+	SubnetDrift       cloudprovider.DriftReason = "SubnetDrift"
 )
 
 func (c *CloudProvider) isK8sVersionDrifted(ctx context.Context, nodeClaim *corev1beta1.NodeClaim) (cloudprovider.DriftReason, error) {
@@ -119,3 +124,35 @@ func (c *CloudProvider) isImageVersionDrifted(
 	}
 	return "", nil
 }
+
+// isSubnetDrifted returns drift if the nic for this nodeclaim does not match the expected subnet
+func (c *CloudProvider) isSubnetDrifted(ctx context.Context, nodeClaim *corev1beta1.NodeClaim, nodeClass *v1alpha2.AKSNodeClass) (cloudprovider.DriftReason, error) {
+	expectedSubnet := lo.Ternary(nodeClass.Spec.VNETSubnetID == nil, options.FromContext(ctx).SubnetID, lo.FromPtr(nodeClass.Spec.VNETSubnetID))
+	nicName := instance.GenerateResourceName(nodeClaim.Name)
+
+	// TODO: Refactor all of AzConfig to be part of options
+	nic, err := instance.GetNic(ctx, c.instanceProvider.AZClient.NetworkInterfacesClient, options.FromContext(ctx).NodeResourceGroup, nicName)
+	if err != nil {
+		if sdkerrors.IsNotFoundErr(err) {
+			return "", nil
+		}
+		return "", err
+	}
+	nicSubnet := getSubnetFromPrimaryIPConfig(nic)
+	if nicSubnet == "" {
+		return "", fmt.Errorf("no subnet found for nic: %s", nicName)
+	}
+	if nicSubnet != expectedSubnet {
+		return SubnetDrift, nil
+	}
+	return "", nil
+}
+
+func getSubnetFromPrimaryIPConfig(nic *armnetwork.Interface) string {
+	for _, ipConfig := range nic.Properties.IPConfigurations {
+		if ipConfig.Properties.Subnet != nil && lo.FromPtr(ipConfig.Properties.Primary) {
+			return lo.FromPtr(ipConfig.Properties.Subnet.ID)
+		}
+	}
+	return ""
+}

pkg/cloudprovider/suite_test.go

@@ -169,12 +169,11 @@ var _ = Describe("CloudProvider", func() {
 			})
 			ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod)
 			ExpectScheduled(ctx, env.Client, pod)
-
+			Expect(azureEnv.NetworkInterfacesAPI.NetworkInterfacesCreateOrUpdateBehavior.CalledWithInput.Len()).To(Equal(1))
 			Expect(azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Len()).To(Equal(1))
 			input := azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Pop()
 			rg := input.ResourceGroupName
 			vmName := input.VMName
-
 			// Corresponding NodeClaim
 			nodeClaim = coretest.NodeClaim(corev1beta1.NodeClaim{
 				Status: corev1beta1.NodeClaimStatus{

pkg/operator/operator.go

@@ -128,7 +128,6 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont
 		unavailableOfferingsCache,
 		azConfig.Location,
 		azConfig.NodeResourceGroup,
-		options.FromContext(ctx).SubnetID,
 		azConfig.SubscriptionID,
 	)
 

pkg/operator/options/options.go

@@ -73,6 +73,8 @@ type Options struct {
 
 	SubnetID string // => VnetSubnetID to use (for nodes in Azure CNI Overlay and Azure CNI + pod subnet; for for nodes and pods in Azure CNI), unless overridden via AKSNodeClass
 	setFlags map[string]bool
+
+	NodeResourceGroup string
 }
 
 func (o *Options) AddFlags(fs *coreoptions.FlagSet) {
@@ -87,6 +89,7 @@ func (o *Options) AddFlags(fs *coreoptions.FlagSet) {
 	fs.StringVar(&o.NetworkDataplane, "network-dataplane", env.WithDefaultString("NETWORK_DATAPLANE", "cilium"), "The network dataplane used by the cluster.")
 	fs.StringVar(&o.SubnetID, "vnet-subnet-id", env.WithDefaultString("VNET_SUBNET_ID", ""), "The default subnet ID to use for new nodes. This must be a valid ARM resource ID for subnet that does not overlap with the service CIDR or the pod CIDR")
 	fs.Var(newNodeIdentitiesValue(env.WithDefaultString("NODE_IDENTITIES", ""), &o.NodeIdentities), "node-identities", "User assigned identities for nodes.")
+	fs.StringVar(&o.NodeResourceGroup, "node-resource-group", env.WithDefaultString("AZURE_NODE_RESOURCE_GROUP", ""), "[REQUIRED] the resource group created and managed by AKS where the nodes live")
 }
 
 func (o Options) GetAPIServerName() string {

pkg/providers/instance/armutils.go

@@ -75,6 +75,14 @@ func createVirtualMachineExtension(ctx context.Context, client VirtualMachineExt
 	return &res.VirtualMachineExtension, nil
 }
 
+func GetNic(ctx context.Context, client NetworkInterfacesAPI, rg, nicName string) (*armnetwork.Interface, error) {
+	nicResponse, err := client.Get(ctx, rg, nicName, nil)
+	if err != nil {
+		return nil, err
+	}
+	return &nicResponse.Interface, nil
+}
+
 func createNic(ctx context.Context, client NetworkInterfacesAPI, rg, nicName string, nic armnetwork.Interface) (*armnetwork.Interface, error) {
 	poller, err := client.BeginCreateOrUpdate(ctx, rg, nicName, nic, nil)
 	if err != nil {

pkg/providers/instance/azure_client.go

@@ -61,7 +61,7 @@ type AZClient struct {
 	azureResourceGraphClient       AzureResourceGraphAPI
 	virtualMachinesClient          VirtualMachinesAPI
 	virtualMachinesExtensionClient VirtualMachineExtensionsAPI
-	networkInterfacesClient        NetworkInterfacesAPI
+	NetworkInterfacesClient        NetworkInterfacesAPI
 
 	ImageVersionsClient imagefamily.CommunityGalleryImageVersionsAPI
 	// SKU CLIENT is still using track 1 because skewer does not support the track 2 path. We need to refactor this once skewer supports track 2
@@ -82,7 +82,7 @@ func NewAZClientFromAPI(
 		virtualMachinesClient:          virtualMachinesClient,
 		azureResourceGraphClient:       azureResourceGraphClient,
 		virtualMachinesExtensionClient: virtualMachinesExtensionClient,
-		networkInterfacesClient:        interfacesClient,
+		NetworkInterfacesClient:        interfacesClient,
 		ImageVersionsClient:            imageVersionsClient,
 		SKUClient:                      skuClient,
 		LoadBalancersClient:            loadBalancersClient,

pkg/providers/instance/instance.go

@@ -81,36 +81,33 @@ type Resource = map[string]interface{}
 
 type Provider struct {
 	location               string
-	azClient               *AZClient
+	AZClient               *AZClient
 	instanceTypeProvider   *instancetype.Provider
 	launchTemplateProvider *launchtemplate.Provider
 	loadBalancerProvider   *loadbalancer.Provider
 	resourceGroup          string
-	subnetID               string
 	subscriptionID         string
 	unavailableOfferings   *cache.UnavailableOfferings
 }
 
 func NewProvider(
-	azClient *AZClient,
+	AZClient *AZClient,
 	instanceTypeProvider *instancetype.Provider,
 	launchTemplateProvider *launchtemplate.Provider,
 	loadBalancerProvider *loadbalancer.Provider,
 	offeringsCache *cache.UnavailableOfferings,
 	location string,
 	resourceGroup string,
-	subnetID string,
 	subscriptionID string,
 ) *Provider {
 	listQuery = GetListQueryBuilder(resourceGroup).String()
 	return &Provider{
-		azClient:               azClient,
+		AZClient:               AZClient,
 		instanceTypeProvider:   instanceTypeProvider,
 		launchTemplateProvider: launchTemplateProvider,
 		loadBalancerProvider:   loadBalancerProvider,
 		location:               location,
 		resourceGroup:          resourceGroup,
-		subnetID:               subnetID,
 		subscriptionID:         subscriptionID,
 		unavailableOfferings:   offeringsCache,
 	}
@@ -142,14 +139,14 @@ func (p *Provider) Create(ctx context.Context, nodeClass *v1alpha2.AKSNodeClass,
 }
 
 func (p *Provider) Update(ctx context.Context, vmName string, update armcompute.VirtualMachineUpdate) error {
-	return UpdateVirtualMachine(ctx, p.azClient.virtualMachinesClient, p.resourceGroup, vmName, update)
+	return UpdateVirtualMachine(ctx, p.AZClient.virtualMachinesClient, p.resourceGroup, vmName, update)
 }
 
 func (p *Provider) Get(ctx context.Context, vmName string) (*armcompute.VirtualMachine, error) {
 	var vm armcompute.VirtualMachinesClientGetResponse
 	var err error
 
-	if vm, err = p.azClient.virtualMachinesClient.Get(ctx, p.resourceGroup, vmName, nil); err != nil {
+	if vm, err = p.AZClient.virtualMachinesClient.Get(ctx, p.resourceGroup, vmName, nil); err != nil {
 		if sdkerrors.IsNotFoundErr(err) {
 			return nil, corecloudprovider.NewNodeClaimNotFoundError(err)
 		}
@@ -161,7 +158,7 @@ func (p *Provider) Get(ctx context.Context, vmName string) (*armcompute.VirtualM
 
 func (p *Provider) List(ctx context.Context) ([]*armcompute.VirtualMachine, error) {
 	req := NewQueryRequest(&(p.subscriptionID), listQuery)
-	client := p.azClient.azureResourceGraphClient
+	client := p.AZClient.azureResourceGraphClient
 	data, err := GetResourceData(ctx, client, *req)
 	if err != nil {
 		return nil, fmt.Errorf("querying azure resource graph, %w", err)
@@ -187,7 +184,7 @@ func (p *Provider) createAKSIdentifyingExtension(ctx context.Context, vmName str
 	vmExt := p.getAKSIdentifyingExtension()
 	vmExtName := *vmExt.Name
 	logging.FromContext(ctx).Debugf("Creating virtual machine AKS identifying extension for %s", vmName)
-	v, err := createVirtualMachineExtension(ctx, p.azClient.virtualMachinesExtensionClient, p.resourceGroup, vmName, vmExtName, *vmExt)
+	v, err := createVirtualMachineExtension(ctx, p.AZClient.virtualMachinesExtensionClient, p.resourceGroup, vmName, vmExtName, *vmExt)
 	if err != nil {
 		logging.FromContext(ctx).Errorf("Creating VM AKS identifying extension for VM %q failed, %w", vmName, err)
 		return fmt.Errorf("creating VM AKS identifying extension for VM %q, %w failed", vmName, err)
@@ -220,9 +217,7 @@ func (p *Provider) newNetworkInterfaceForVM(opts *createNICOptions) armnetwork.I
 					Properties: &armnetwork.InterfaceIPConfigurationPropertiesFormat{
 						Primary:                   lo.ToPtr(true),
 						PrivateIPAllocationMethod: lo.ToPtr(armnetwork.IPAllocationMethodDynamic),
-						Subnet: &armnetwork.Subnet{
-							ID: &p.subnetID,
-						},
+
 						LoadBalancerBackendAddressPools: ipv4BackendPools,
 					},
 				},
@@ -244,9 +239,6 @@ func (p *Provider) newNetworkInterfaceForVM(opts *createNICOptions) armnetwork.I
 					Properties: &armnetwork.InterfaceIPConfigurationPropertiesFormat{
 						Primary:                   lo.ToPtr(false),
 						PrivateIPAllocationMethod: lo.ToPtr(armnetwork.IPAllocationMethodDynamic),
-						Subnet: &armnetwork.Subnet{
-							ID: &p.subnetID,
-						},
 					},
 				},
 			)
@@ -272,7 +264,7 @@ func (p *Provider) createNetworkInterface(ctx context.Context, opts *createNICOp
 	nic := p.newNetworkInterfaceForVM(opts)
 	p.applyTemplateToNic(&nic, opts.LaunchTemplate)
 	logging.FromContext(ctx).Debugf("Creating network interface %s", opts.NICName)
-	res, err := createNic(ctx, p.azClient.networkInterfacesClient, p.resourceGroup, opts.NICName, nic)
+	res, err := createNic(ctx, p.AZClient.NetworkInterfacesClient, p.resourceGroup, opts.NICName, nic)
 	if err != nil {
 		return "", err
 	}
@@ -386,7 +378,7 @@ func setNodePoolNameTag(tags map[string]*string, nodeClaim *corev1beta1.NodeClai
 }
 
 func (p *Provider) createVirtualMachine(ctx context.Context, vm armcompute.VirtualMachine, vmName string) (*armcompute.VirtualMachine, error) {
-	result, err := CreateVirtualMachine(ctx, p.azClient.virtualMachinesClient, p.resourceGroup, vmName, vm)
+	result, err := CreateVirtualMachine(ctx, p.AZClient.virtualMachinesClient, p.resourceGroup, vmName, vm)
 	if err != nil {
 		logging.FromContext(ctx).Errorf("Creating virtual machine %q failed: %v", vmName, err)
 		return nil, fmt.Errorf("virtualMachine.BeginCreateOrUpdate for VM %q failed: %w", vmName, err)
@@ -533,8 +525,10 @@ func cpuLimitIsZero(err error) bool {
 }
 
 func (p *Provider) applyTemplateToNic(nic *armnetwork.Interface, template *launchtemplate.Template) {
-	// set tags
 	nic.Tags = template.Tags
+	for _, ipConfig := range nic.Properties.IPConfigurations {
+		ipConfig.Properties.Subnet = &armnetwork.Subnet{ID: &template.SubnetID}
+	}
 }
 
 func (p *Provider) getLaunchTemplate(ctx context.Context, nodeClass *v1alpha2.AKSNodeClass, nodeClaim *corev1beta1.NodeClaim,
@@ -593,14 +587,14 @@ func (p *Provider) pickSkuSizePriorityAndZone(ctx context.Context, nodeClaim *co
 }
 
 func (p *Provider) cleanupAzureResources(ctx context.Context, resourceName string) (err error) {
-	vmErr := deleteVirtualMachineIfExists(ctx, p.azClient.virtualMachinesClient, p.resourceGroup, resourceName)
+	vmErr := deleteVirtualMachineIfExists(ctx, p.AZClient.virtualMachinesClient, p.resourceGroup, resourceName)
 	if vmErr != nil {
 		logging.FromContext(ctx).Errorf("virtualMachine.Delete for %s failed: %v", resourceName, vmErr)
 	}
 	// The order here is intentional, if the VM was created successfully, then we attempt to delete the vm, the
 	// nic, disk and all associated resources will be removed. If the VM was not created successfully and a nic was found,
 	// then we attempt to delete the nic.
-	nicErr := deleteNicIfExists(ctx, p.azClient.networkInterfacesClient, p.resourceGroup, resourceName)
+	nicErr := deleteNicIfExists(ctx, p.AZClient.NetworkInterfacesClient, p.resourceGroup, resourceName)
 	if nicErr != nil {
 		logging.FromContext(ctx).Errorf("networkInterface.Delete for %s failed: %v", resourceName, nicErr)
 	}

pkg/providers/instance/instance_test.go

@@ -80,7 +80,6 @@ func TestGetPriorityCapacityAndInstanceType(t *testing.T) {
 	provider := NewProvider(nil, nil, nil, nil, cache.NewUnavailableOfferings(),
 		"westus-2",
 		"MC_xxxxx_yyyy-region",
-		"/subscriptions/0000000-0000-0000-0000-0000000000/resourceGroups/fake-resource-group-name/providers/Microsoft.Network/virtualNetworks/karpenter/subnets/nodesubnet",
 		"0000000-0000-0000-0000-0000000000",
 	)
 	for _, c := range cases {