fix: handle SkuNotAvailable provisioning error

pkg/providers/instance/instance.go

@@ -68,10 +68,15 @@ var (
 		string(compute.Regular): corev1beta1.CapacityTypeOnDemand,
 	}
 
-	SubscriptionQuotaReached = "SubscriptionQuotaReached"
-	ZonalAllocationFailure   = "ZonalAllocationFailure"
+	SubscriptionQuotaReachedReason = "SubscriptionQuotaReached"
+	ZonalAllocationFailureReason   = "ZonalAllocationFailure"
+	SKUNotAvailableReason          = "SKUNotAvailable"
 
-	DurationForNewQuotaRequest = 1 * time.Hour
+	SKUNotAvailableErrorCode = "SkuNotAvailable"
+
+	SubscriptionQuotaReachedTTL = 1 * time.Hour
+	SKUNotAvailableSpotTTL      = 1 * time.Hour
+	SKUNotAvailableOnDemandTTL  = 23 * time.Hour
 )
 
 type Resource = map[string]interface{}
@@ -440,10 +445,16 @@ func (p *Provider) launchInstance(
 	return resp, instanceType, nil
 }
 
+// isSKUNotAvailable - to be moved to azure-sdk-for-go-extensions
+func isSKUNotAvailable(err error) bool {
+	azErr := sdkerrors.IsResponseError(err)
+	return azErr != nil && azErr.ErrorCode == SKUNotAvailableErrorCode
+}
+
 func (p *Provider) handleResponseErrors(ctx context.Context, instanceType *corecloudprovider.InstanceType, zone, capacityType string, err error) error {
 	if sdkerrors.SubscriptionQuotaHasBeenReached(err) {
 		// Subscription quota reached, mark the instance type as unavailable in all zones available to the offering
-		// THis will also update the TTL for an existing offering in the cache that is already unavailable
+		// This will also update the TTL for an existing offering in the cache that is already unavailable
 		for _, offering := range instanceType.Offerings {
 			if offering.CapacityType != capacityType {
 				continue
@@ -452,14 +463,32 @@ func (p *Provider) handleResponseErrors(ctx context.Context, instanceType *corec
 			// If we have a quota limit of 0 vcpus, we mark the offerings unavailable for an hour.
 			// CPU limits of 0 are usually due to a subscription having no allocated quota for that instance type at all on the subscription.
 			if cpuLimitIsZero(err) {
-				p.unavailableOfferings.MarkUnavailableWithTTL(ctx, SubscriptionQuotaReached, instanceType.Name, offering.Zone, capacityType, DurationForNewQuotaRequest)
+				p.unavailableOfferings.MarkUnavailableWithTTL(ctx, SubscriptionQuotaReachedReason, instanceType.Name, offering.Zone, capacityType, SubscriptionQuotaReachedTTL)
 			} else {
-				p.unavailableOfferings.MarkUnavailable(ctx, SubscriptionQuotaReached, instanceType.Name, offering.Zone, capacityType)
+				p.unavailableOfferings.MarkUnavailable(ctx, SubscriptionQuotaReachedReason, instanceType.Name, offering.Zone, capacityType)
 			}
 		}
 	} else if sdkerrors.ZonalAllocationFailureOccurred(err) {
-		p.unavailableOfferings.MarkUnavailable(ctx, ZonalAllocationFailure, instanceType.Name, zone, corev1beta1.CapacityTypeOnDemand)
-		p.unavailableOfferings.MarkUnavailable(ctx, ZonalAllocationFailure, instanceType.Name, zone, corev1beta1.CapacityTypeSpot)
+		p.unavailableOfferings.MarkUnavailable(ctx, ZonalAllocationFailureReason, instanceType.Name, zone, corev1beta1.CapacityTypeOnDemand)
+		p.unavailableOfferings.MarkUnavailable(ctx, ZonalAllocationFailureReason, instanceType.Name, zone, corev1beta1.CapacityTypeSpot)
+	} else if isSKUNotAvailable(err) {
+		// https://aka.ms/azureskunotavailable: either not available for a location or zone, or out of capacity for Spot.
+		// We only expect to observe the Spot case, not location or zone restrictions, because:
+		// - SKUs with location restriction are already filtered out via sku.HasLocationRestriction
+		// - zonal restrictions are filtered out internally by sku.AvailabilityZones, and don't get offerings
+		skuNotAvailableTTL := SKUNotAvailableSpotTTL
+		err = fmt.Errorf("out of spot capacity for %s: %w", instanceType.Name, err)
+		if capacityType == corev1beta1.CapacityTypeOnDemand { // should not happen, defensive check
+			err = fmt.Errorf("unexpected SkuNotAvailable error for %s (on-demand): %w", instanceType.Name, err)
+			skuNotAvailableTTL = SKUNotAvailableOnDemandTTL // still mark all offerings as unavailable, but with a longer TTL
+		}
+		// mark the instance type as unavailable for all offerings/zones for the capacity type
+		for _, offering := range instanceType.Offerings {
+			if offering.CapacityType != capacityType {
+				continue
+			}
+			p.unavailableOfferings.MarkUnavailableWithTTL(ctx, SKUNotAvailableReason, instanceType.Name, offering.Zone, capacityType, skuNotAvailableTTL)
+		}
 	}
 	return err
 }

pkg/providers/instancetype/suite_test.go

@@ -48,15 +48,18 @@ import (
 	coretest "github.com/aws/karpenter-core/pkg/test"
 	. "github.com/aws/karpenter-core/pkg/test/expectations"
 
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute"
 	"github.com/Azure/karpenter/pkg/apis"
 	"github.com/Azure/karpenter/pkg/apis/settings"
 	"github.com/Azure/karpenter/pkg/apis/v1alpha2"
 	"github.com/Azure/karpenter/pkg/cloudprovider"
 	"github.com/Azure/karpenter/pkg/fake"
+	"github.com/Azure/karpenter/pkg/providers/instance"
 	"github.com/Azure/karpenter/pkg/providers/instancetype"
 	"github.com/Azure/karpenter/pkg/providers/loadbalancer"
 	"github.com/Azure/karpenter/pkg/test"
+	. "github.com/Azure/karpenter/pkg/test/expectations"
 	"github.com/Azure/karpenter/pkg/utils"
 )
 
@@ -374,8 +377,7 @@ var _ = Describe("InstanceType Provider", func() {
 				Key:      v1.LabelInstanceTypeStable,
 				Operator: v1.NodeSelectorOpIn,
 				Values:   []string{"Standard_D2_v2"},
-			},
-			)
+			})
 
 			ExpectApplied(ctx, env.Client, nodePool, nodeClass)
 			// Try this 100 times to make sure we don't get a node in eastus-1,
@@ -489,6 +491,34 @@ var _ = Describe("InstanceType Provider", func() {
 			node := ExpectScheduled(ctx, env.Client, pod)
 			Expect(node.Labels).To(HaveKeyWithValue(v1.LabelInstanceTypeStable, "Standard_D2_v2"))
 		})
+
+		Context("on SkuNotUnavailable, should cache SKU as unavailable in all zones", func() {
+			AssertUnavailable := func(sku string, capacityType string) {
+				// fake a SKU not available error
+				azureEnv.VirtualMachinesAPI.VirtualMachinesBehavior.VirtualMachineCreateOrUpdateBehavior.Error.Set(
+					&azcore.ResponseError{ErrorCode: instance.SKUNotAvailableErrorCode},
+				)
+				coretest.ReplaceRequirements(nodePool,
+					v1.NodeSelectorRequirement{Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{sku}},
+					v1.NodeSelectorRequirement{Key: corev1beta1.CapacityTypeLabelKey, Operator: v1.NodeSelectorOpIn, Values: []string{capacityType}},
+				)
+				ExpectApplied(ctx, env.Client, nodeClass, nodePool)
+				pod := coretest.UnschedulablePod()
+				ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod)
+				ExpectNotScheduled(ctx, env.Client, pod)
+				for _, zone := range []string{"1", "2", "3"} {
+					ExpectUnavailable(azureEnv, sku, zone, capacityType)
+				}
+			}
+
+			It("should mark SKU as unavailable in all zones for Spot", func() {
+				AssertUnavailable("Standard_D2_v2", corev1beta1.CapacityTypeSpot)
+			})
+
+			It("should mark SKU as unavailable in all zones for OnDemand", func() {
+				AssertUnavailable("Standard_D2_v2", corev1beta1.CapacityTypeOnDemand)
+			})
+		})
 	})
 	Context("Provider List", func() {
 		var instanceTypes corecloudprovider.InstanceTypes

pkg/test/expectations/expectations.go

@@ -0,0 +1,29 @@
+/*
+Portions Copyright (c) Microsoft Corporation.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package expectations
+
+import (
+	"fmt"
+
+	"github.com/Azure/karpenter/pkg/fake"
+	"github.com/Azure/karpenter/pkg/test"
+	. "github.com/onsi/gomega"
+)
+
+func ExpectUnavailable(env *test.Environment, instanceType string, zone string, capacityType string) {
+	Expect(env.UnavailableOfferingsCache.IsUnavailable(instanceType, fmt.Sprintf("%s-%s", fake.Region, zone), capacityType)).To(BeTrue())
+}