From 3a9cbc0d7bf64e4637fb6803bbd26b4c248cacca Mon Sep 17 00:00:00 2001 From: Bryce Soghigian <49734722+Bryce-Soghigian@users.noreply.github.com> Date: Sat, 21 Dec 2024 04:18:44 -0800 Subject: [PATCH] feat: ListNodeImageVersions + shared image gallery support (#526) * feat: support shared image galleries inside of karpenter * chore: populating image stubs for shared image galleries * fix: progress * fix: PopulateResourceStub accessing the wrong index * test: properly testing ListNodeImageVersions * refactor: rename symbol for SIG Subscription id * test: conditional use of sig dependent on the managed karpenter flag * refactor: removing panics used in testing * test: adding RBAC and helm values to the template for SIG Gallery logic * fix: bug in azure linux sig image resolution * chore: update cleanupenv to handle inflate too ratehr than just job pods * test: fix randomized test order flake * ci: shadow declaration * refactor: comment wording * test: validate all image ids are resolved correctly * fix: adding filtering for duplicate sku + os combinations and filtering out unsupported galleries * refactor: renaming var * refactor: rename the managedKarpenter reference to UseSIG * refactor: spelling * fix: v1 migration for test * ci: lint * fix: lint * ci: fix * ci: license * ci: fix * fix: test pollution * fix: resetting options before each test run * fix: accounting for versions of the shape 'yy.mm.dd' * test: removing unused cleanup funcs * fix: making the key for shared image gallery smaller * refactor: removing windows types leaving them to be added back later * refactor: reducing key even more * fix: extending key * fix: removing log line * refactor: not nesting options as deep, leaving refactor of USESIG to provider for later * fix: comment about pulling out the variable * refactor: decoupling the cache reads and cache writes from the image gallery id retrivial functions * test: adding some coverage to FilteredNodeImages * refactor: removing community galleries fake method for GET since this implementation doesn't use it * test: test that we drift nodes when switching from community gallery to shared image gallery * test: SIG_SUBSCRIPTION_ID * refactor: using sig from context * fix: remove extra space --------- Co-authored-by: Alex Leites <18728999+tallaxes@users.noreply.github.com> --- Makefile-az.mk | 9 +- karpenter-values-template.yaml | 6 + pkg/cloudprovider/drift.go | 18 +- pkg/cloudprovider/suite_test.go | 10 + pkg/fake/nodeimageversionsapi.go | 381 ++++++++++++++++++ pkg/fake/nodeimageversionsapi_test.go | 101 +++++ pkg/operator/operator.go | 3 + pkg/operator/options/options.go | 9 +- pkg/providers/imagefamily/azlinux.go | 24 +- pkg/providers/imagefamily/image.go | 107 +++-- pkg/providers/imagefamily/image_test.go | 57 --- .../imagefamily/nodeimageversionsclient.go | 132 ++++++ .../nodeimageversionsclient_test.go | 49 +++ pkg/providers/imagefamily/types.go | 51 ++- pkg/providers/imagefamily/ubuntu_2204.go | 24 +- pkg/providers/instance/azure_client.go | 14 +- pkg/providers/instance/instance.go | 29 +- pkg/providers/instancetype/instancetypes.go | 4 +- pkg/providers/instancetype/suite_test.go | 85 +++- pkg/test/environment.go | 9 +- pkg/test/options.go | 6 + 21 files changed, 967 insertions(+), 161 deletions(-) create mode 100644 pkg/fake/nodeimageversionsapi.go create mode 100644 pkg/fake/nodeimageversionsapi_test.go delete mode 100644 pkg/providers/imagefamily/image_test.go create mode 100644 pkg/providers/imagefamily/nodeimageversionsclient.go create mode 100644 pkg/providers/imagefamily/nodeimageversionsclient_test.go diff --git a/Makefile-az.mk b/Makefile-az.mk index d5ecf67e3..9533caad5 100755 --- a/Makefile-az.mk +++ b/Makefile-az.mk @@ -8,6 +8,7 @@ else AZURE_ACR_NAME ?= $(COMMON_NAME) endif +AZURE_SIG_SUBSCRIPTION_ID ?= $(AZURE_SUBSCRIPTION_ID) AZURE_CLUSTER_NAME ?= $(COMMON_NAME) AZURE_RESOURCE_GROUP_MC = MC_$(AZURE_RESOURCE_GROUP)_$(AZURE_CLUSTER_NAME)_$(AZURE_LOCATION) @@ -46,7 +47,8 @@ az-mkacr: az-mkrg ## Create test ACR az-acrimport: ## Imports an image to an acr registry az acr import --name $(AZURE_ACR_NAME) --source "mcr.microsoft.com/oss/kubernetes/pause:3.6" --image "pause:3.6" -az-cleanenv: az-rmnodeclaims-fin ## Deletes a few common karpenter testing resources(pods, nodepools, nodeclaims, aksnodeclasses) +az-cleanenv: az-rmnodeclaims-fin ## Deletes a few common karpenter testing resources(pods, nodepools, nodeclaims, aksnodeclasses) + kubectl delete deployments -n default --all kubectl delete pods -n default --all kubectl delete nodeclaims --all kubectl delete nodepools --all @@ -136,6 +138,11 @@ az-perm: ## Create role assignments to let Karpenter manage VMs and Network az role assignment create --assignee $(KARPENTER_USER_ASSIGNED_CLIENT_ID) --scope /subscriptions/$(AZURE_SUBSCRIPTION_ID)/resourceGroups/$(AZURE_RESOURCE_GROUP) --role "Network Contributor" # in some case we create vnet here @echo Consider "make az-configure-values"! +az-perm-sig: ## Create role assignments when testing with SIG images + $(eval KARPENTER_USER_ASSIGNED_CLIENT_ID=$(shell az identity show --resource-group "${AZURE_RESOURCE_GROUP}" --name "${AZURE_KARPENTER_USER_ASSIGNED_IDENTITY_NAME}" --query 'principalId' -otsv)) + az role assignment create --assignee $(KARPENTER_USER_ASSIGNED_CLIENT_ID) --role "Reader" --scope /subscriptions/$(AZURE_SIG_SUBSCRIPTION_ID)/resourceGroups/AKS-Ubuntu/providers/Microsoft.Compute/galleries/AKSUbuntu + az role assignment create --assignee $(KARPENTER_USER_ASSIGNED_CLIENT_ID) --role "Reader" --scope /subscriptions/$(AZURE_SIG_SUBSCRIPTION_ID)/resourceGroups/AKS-AzureLinux/providers/Microsoft.Compute/galleries/AKSAzureLinux + az-perm-subnet-custom: az-perm ## Create role assignments to let Karpenter manage VMs and Network (custom VNet) $(eval VNET_SUBNET_ID=$(shell az aks show --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) | jq -r ".agentPoolProfiles[0].vnetSubnetId")) $(eval KARPENTER_USER_ASSIGNED_CLIENT_ID=$(shell az identity show --resource-group "${AZURE_RESOURCE_GROUP}" --name "${AZURE_KARPENTER_USER_ASSIGNED_IDENTITY_NAME}" --query 'principalId' -otsv)) diff --git a/karpenter-values-template.yaml b/karpenter-values-template.yaml index 8b8fa5e12..dcbbeddfc 100644 --- a/karpenter-values-template.yaml +++ b/karpenter-values-template.yaml @@ -37,6 +37,12 @@ controller: value: "" - name: AZURE_NODE_RESOURCE_GROUP value: ${AZURE_RESOURCE_GROUP_MC} + + # managed karpenter settings + - name: USE_SIG + value: "false" + - name: SIG_SUBSCRIPTION_ID + value: "" serviceAccount: name: ${KARPENTER_SERVICE_ACCOUNT_NAME} annotations: diff --git a/pkg/cloudprovider/drift.go b/pkg/cloudprovider/drift.go index 98c1114bd..b453403a3 100644 --- a/pkg/cloudprovider/drift.go +++ b/pkg/cloudprovider/drift.go @@ -145,21 +145,17 @@ func (c *CloudProvider) isImageVersionDrifted( if vm.Properties == nil || vm.Properties.StorageProfile == nil || - vm.Properties.StorageProfile.ImageReference == nil || - vm.Properties.StorageProfile.ImageReference.CommunityGalleryImageID == nil || - *vm.Properties.StorageProfile.ImageReference.CommunityGalleryImageID == "" { - logger.Debug("not using a CommunityGalleryImageID for nodeClaim %s", nodeClaim.Name) + vm.Properties.StorageProfile.ImageReference == nil { return "", nil } + CIGID := lo.FromPtr(vm.Properties.StorageProfile.ImageReference.CommunityGalleryImageID) + SIGID := lo.FromPtr(vm.Properties.StorageProfile.ImageReference.ID) + vmImageID := lo.Ternary(SIGID != "", SIGID, CIGID) - vmImageID := *vm.Properties.StorageProfile.ImageReference.CommunityGalleryImageID + var imageStub imagefamily.DefaultImageOutput + imageStub.PopulateImageTraitsFromID(vmImageID) - publicGalleryURL, communityImageName, _, err := imagefamily.ParseCommunityImageIDInfo(vmImageID) - if err != nil { - return "", err - } - - expectedImageID, err := c.imageProvider.GetImageID(ctx, communityImageName, publicGalleryURL) + expectedImageID, err := c.imageProvider.GetLatestImageID(ctx, imageStub) if err != nil { return "", err } diff --git a/pkg/cloudprovider/suite_test.go b/pkg/cloudprovider/suite_test.go index b4cde4dee..0a9c86c0f 100644 --- a/pkg/cloudprovider/suite_test.go +++ b/pkg/cloudprovider/suite_test.go @@ -27,6 +27,7 @@ import ( "github.com/awslabs/operatorpkg/object" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/samber/lo" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -223,5 +224,14 @@ var _ = Describe("CloudProvider", func() { Expect(err).To(HaveOccurred()) Expect(drifted).To(BeEmpty()) }) + It("should trigger drift when the image gallery changes to SIG", func() { + options := test.Options(test.OptionsFields{ + UseSIG: lo.ToPtr(true), + }) + ctx = options.ToContext(ctx) + drifted, err := cloudProvider.IsDrifted(ctx, nodeClaim) + Expect(err).ToNot(HaveOccurred()) + Expect(string(drifted)).To(Equal("ImageVersionDrift")) + }) }) }) diff --git a/pkg/fake/nodeimageversionsapi.go b/pkg/fake/nodeimageversionsapi.go new file mode 100644 index 000000000..5d449361b --- /dev/null +++ b/pkg/fake/nodeimageversionsapi.go @@ -0,0 +1,381 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fake + +import ( + "context" + + "github.com/Azure/karpenter-provider-azure/pkg/providers/imagefamily" +) + +type NodeImageVersionsAPI struct { +} + +var _ imagefamily.NodeImageVersionsAPI = &NodeImageVersionsAPI{} + +var ( + NodeImageVersions = []imagefamily.NodeImageVersion{ + { + FullName: "AKSUbuntu-1804gpucontainerd-202410.09.0", + OS: "AKSUbuntu", + SKU: "1804gpucontainerd", + Version: "202410.09.0", + }, + { + FullName: "AKSWindows-2019-17763.2019.221114", + OS: "AKSWindows", + SKU: "windows-2019", + Version: "17763.2019.221114", + }, + { + FullName: "AKSAzureLinux-V3-202409.23.0", + OS: "AKSAzureLinux", + SKU: "V3", + Version: "202409.23.0", + }, + { + FullName: "AKSUbuntu-2204gen2containerd-202410.09.0", + OS: "AKSUbuntu", + SKU: "2204gen2containerd", + Version: "202410.09.0", + }, + { + FullName: "AKSUbuntu-1804gpu-2022.08.29", + OS: "AKSUbuntu", + SKU: "1804gpu", + Version: "2022.08.29", + }, + { + FullName: "AKSWindows-2022-containerd-gen2-20348.2762.241009", + OS: "AKSWindows", + SKU: "windows-2022-containerd-gen2", + Version: "20348.2762.241009", + }, + { + FullName: "AKSCBLMariner-V2katagen2TL-2022.12.15", + OS: "AKSCBLMariner", + SKU: "V2katagen2TL", + Version: "2022.12.15", + }, + { + FullName: "AKSUbuntu-2004gen2fipscontainerd-202410.09.0", + OS: "AKSUbuntu", + SKU: "2004gen2fipscontainerd", + Version: "202410.09.0", + }, + { + FullName: "AKSAzureLinux-V2fips-202410.09.0", + OS: "AKSAzureLinux", + SKU: "V2fips", + Version: "202410.09.0", + }, + { + FullName: "AKSAzureLinux-V2gen2fips-202410.09.0", + OS: "AKSAzureLinux", + SKU: "V2gen2fips", + Version: "202410.09.0", + }, + { + FullName: "AKSUbuntuEdgeZone-1804gen2containerd-202410.09.0", + OS: "AKSUbuntuEdgeZone", + SKU: "1804gen2containerd", + Version: "202410.09.0", + }, + { + FullName: "AKSAzureLinux-V2katagen2-202410.09.0", + OS: "AKSAzureLinux", + SKU: "V2katagen2", + Version: "202410.09.0", + }, + { + FullName: "AKSAzureLinux-V3gen2-202409.23.0", + OS: "AKSAzureLinux", + SKU: "V3gen2", + Version: "202409.23.0", + }, + { + FullName: "AKSUbuntu-2204gen2arm64containerd-202410.09.0", + OS: "AKSUbuntu", + SKU: "2204gen2arm64containerd", + Version: "202410.09.0", + }, + { + FullName: "AKSUbuntu-1804gen2gpucontainerd-202410.09.0", + OS: "AKSUbuntu", + SKU: "1804gen2gpucontainerd", + Version: "202410.09.0", + }, + { + FullName: "AKSAzureLinux-V2gen2TL-202410.09.0", + OS: "AKSAzureLinux", + SKU: "V2gen2TL", + Version: "202410.09.0", + }, + { + FullName: "AKSCBLMariner-V2-202410.09.0", + OS: "AKSCBLMariner", + SKU: "V2", + Version: "202410.09.0", + }, + { + FullName: "AKSCBLMariner-V2fips-202410.09.0", + OS: "AKSCBLMariner", + SKU: "V2fips", + Version: "202410.09.0", + }, + { + FullName: "AKSUbuntu-1804containerd-202410.09.0", + OS: "AKSUbuntu", + SKU: "1804containerd", + Version: "202410.09.0", + }, + { + FullName: "AKSUbuntuEdgeZone-1804containerd-202410.09.0", + OS: "AKSUbuntuEdgeZone", + SKU: "1804containerd", + Version: "202410.09.0", + }, + { + FullName: "AKSUbuntuEdgeZone-2204gen2containerd-202410.09.0", + OS: "AKSUbuntuEdgeZone", + SKU: "2204gen2containerd", + Version: "202410.09.0", + }, + { + FullName: "AKSUbuntu-1804fipscontainerd-202410.09.0", + OS: "AKSUbuntu", + SKU: "1804fipscontainerd", + Version: "202410.09.0", + }, + { + FullName: "AKSAzureLinux-V2gen2arm64-202410.09.0", + OS: "AKSAzureLinux", + SKU: "V2gen2arm64", + Version: "202410.09.0", + }, + { + FullName: "AKSAzureLinux-V2gen2-202410.09.0", + OS: "AKSAzureLinux", + SKU: "V2gen2", + Version: "202410.09.0", + }, + { + FullName: "AKSUbuntu-2204gen2fipscontainerd-202404.09.0", + OS: "AKSUbuntu", + SKU: "2204gen2fipscontainerd", + Version: "202404.09.0", + }, + { + FullName: "AKSWindows-2019-containerd-17763.6414.241010", + OS: "AKSWindows", + SKU: "windows-2019-containerd", + Version: "17763.6414.241010", + }, + { + FullName: "AKSUbuntu-2204gen2TLcontainerd-202410.09.0", + OS: "AKSUbuntu", + SKU: "2204gen2TLcontainerd", + Version: "202410.09.0", + }, + { + FullName: "AKSUbuntu-1804-2022.08.29", + OS: "AKSUbuntu", + SKU: "1804", + Version: "2022.08.29", + }, + { + FullName: "AKSUbuntuEdgeZone-2204containerd-202410.09.0", + OS: "AKSUbuntuEdgeZone", + SKU: "2204containerd", + Version: "202410.09.0", + }, + { + FullName: "AKSCBLMariner-V2gen2-202410.09.0", + OS: "AKSCBLMariner", + SKU: "V2gen2", + Version: "202410.09.0", + }, + { + FullName: "AKSCBLMariner-V2gen2fips-202410.09.0", + OS: "AKSCBLMariner", + SKU: "V2gen2fips", + Version: "202410.09.0", + }, + { + FullName: "AKSCBLMariner-V2gen2arm64-202410.09.0", + OS: "AKSCBLMariner", + SKU: "V2gen2arm64", + Version: "202410.09.0", + }, + { + FullName: "AKSCBLMariner-V2gen2TL-202410.09.0", + OS: "AKSCBLMariner", + SKU: "V2gen2TL", + Version: "202410.09.0", + }, + { + FullName: "AKSUbuntu-2404gen2arm64containerd-202405.20.0", + OS: "AKSUbuntu", + SKU: "2404gen2arm64containerd", + Version: "202405.20.0", + }, + { + FullName: "AKSAzureLinux-V3gen2arm64-202409.23.0", + OS: "AKSAzureLinux", + SKU: "V3gen2arm64", + Version: "202409.23.0", + }, + { + FullName: "AKSAzureLinux-V3fips-202409.23.0", + OS: "AKSAzureLinux", + SKU: "V3fips", + Version: "202409.23.0", + }, + { + FullName: "AKSUbuntu-2004gen2CVMcontainerd-202410.09.0", + OS: "AKSUbuntu", + SKU: "2004gen2CVMcontainerd", + Version: "202410.09.0", + }, + { + FullName: "AKSCBLMariner-V1-202308.28.0", + OS: "AKSCBLMariner", + SKU: "V1", + Version: "202308.28.0", + }, + { + FullName: "AKSUbuntu-1804gen2containerd-202410.09.0", + OS: "AKSUbuntu", + SKU: "1804gen2containerd", + Version: "202410.09.0", + }, + { + FullName: "AKSUbuntu-1804gen2gpu-2022.08.29", + OS: "AKSUbuntu", + SKU: "1804gen2gpu", + Version: "2022.08.29", + }, + { + FullName: "AKSUbuntu-2204gen2minimalcontainerd-202401.12.0", + OS: "AKSUbuntu", + SKU: "2204gen2minimalcontainerd", + Version: "202401.12.0", + }, + { + FullName: "AKSWindows-23H2-gen2-25398.1189.241009", + OS: "AKSWindows", + SKU: "windows-23H2-gen2", + Version: "25398.1189.241009", + }, + { + FullName: "AKSCBLMariner-V2katagen2-202410.09.0", + OS: "AKSCBLMariner", + SKU: "V2katagen2", + Version: "202410.09.0", + }, + { + FullName: "AKSUbuntu-1604-2021.11.06", + OS: "AKSUbuntu", + SKU: "1604", + Version: "2021.11.06", + }, + { + FullName: "AKSUbuntu-2204fipscontainerd-202404.09.0", + OS: "AKSUbuntu", + SKU: "2204fipscontainerd", + Version: "202404.09.0", + }, + { + FullName: "AKSUbuntu-2204minimalcontainerd-202401.12.0", + OS: "AKSUbuntu", + SKU: "2204minimalcontainerd", + Version: "202401.12.0", + }, + { + FullName: "AKSWindows-2022-containerd-20348.2762.241009", + OS: "AKSWindows", + SKU: "windows-2022-containerd", + Version: "20348.2762.241009", + }, + { + FullName: "AKSUbuntu-2204containerd-202410.09.0", + OS: "AKSUbuntu", + SKU: "2204containerd", + Version: "202410.09.0", + }, + { + FullName: "AKSUbuntu-2004fipscontainerd-202410.09.0", + OS: "AKSUbuntu", + SKU: "2004fipscontainerd", + Version: "202410.09.0", + }, + { + FullName: "AKSUbuntu-1804gen2fipscontainerd-202410.09.0", + OS: "AKSUbuntu", + SKU: "1804gen2fipscontainerd", + Version: "202410.09.0", + }, + { + FullName: "AKSWindows-23H2-25398.1189.241009", + OS: "AKSWindows", + SKU: "windows-23H2", + Version: "25398.1189.241009", + }, + { + FullName: "AKSAzureLinux-V2-202410.09.0", + OS: "AKSAzureLinux", + SKU: "V2", + Version: "202410.09.0", + }, + { + FullName: "AKSAzureLinux-V3gen2fips-202409.23.0", + OS: "AKSAzureLinux", + SKU: "V3gen2fips", + Version: "202409.23.0", + }, + { + FullName: "AKSUbuntu-2404gen2containerd-202405.20.0", + OS: "AKSUbuntu", + SKU: "2404gen2containerd", + Version: "202405.20.0", + }, + { + FullName: "AKSUbuntu-2204gen2containerd-2022.10.03", + OS: "AKSUbuntu", + SKU: "2204gen2containerd", + Version: "2022.10.03", + }, + { + FullName: "AKSUbuntu-2404containerd-202405.20.0", + OS: "AKSUbuntu", + SKU: "2404containerd", + Version: "202405.20.0", + }, + { + FullName: "AKSUbuntu-1804gen2-2022.08.29", + OS: "AKSUbuntu", + SKU: "1804gen2", + Version: "2022.08.29", + }, + } +) + +func (n NodeImageVersionsAPI) List(_ context.Context, _, _ string) (imagefamily.NodeImageVersionsResponse, error) { + return imagefamily.NodeImageVersionsResponse{ + Values: imagefamily.FilteredNodeImages(NodeImageVersions), + }, nil +} diff --git a/pkg/fake/nodeimageversionsapi_test.go b/pkg/fake/nodeimageversionsapi_test.go new file mode 100644 index 000000000..cd5d69e3c --- /dev/null +++ b/pkg/fake/nodeimageversionsapi_test.go @@ -0,0 +1,101 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fake + +import ( + "context" + "testing" + + "github.com/Azure/karpenter-provider-azure/pkg/providers/imagefamily" + "github.com/stretchr/testify/assert" +) + +func TestFilteredNodeImagesGalleryFilter(t *testing.T) { + nodeImageVersionAPI := NodeImageVersionsAPI{} + nodeImageVersions, _ := nodeImageVersionAPI.List(context.TODO(), "", "") + filteredNodeImages := imagefamily.FilteredNodeImages(nodeImageVersions.Values) + for _, val := range filteredNodeImages { + assert.NotEqual(t, val.OS, "AKSWindows") + assert.NotEqual(t, val.OS, "AKSUbuntuEdgeZone") + } +} + +// The reasoning behind the test is the following set of output +// az rest --method get --url "/subscriptions//providers/Microsoft.ContainerService/locations/westus2/nodeImageVersions?api-version=2024-04-02-preview" | jq '.values[] | select(.sku == "2204gen2containerd")' +// +// { +// "fullName": "AKSUbuntuEdgeZone-2204gen2containerd-202411.12.0", +// "os": "AKSUbuntuEdgeZone", +// "sku": "2204gen2containerd", +// "version": "202411.12.0" +// } +// { +// "fullName": "AKSUbuntu-2204gen2containerd-2022.10.03", +// "os": "AKSUbuntu", +// "sku": "2204gen2containerd", +// "version": "2022.10.03" +// } +// { +// "fullName": "AKSUbuntu-2204gen2containerd-202411.12.0", +// "os": "AKSUbuntu", +// "sku": "2204gen2containerd", +// "version": "202411.12.0" +// } +// +// In some cases, due to a different distro implementation of the same os + sku pairing, we can get +// duplicate entries for os + sku matchings. +// This test validates we simply ignore the legacy distros and take in the latest Version. +func TestFilteredNodeImagesMinimalUbuntuEdgeCase(t *testing.T) { + filteredNodeImages := imagefamily.FilteredNodeImages(NodeImageVersions) + + expectedVersion := "202410.09.0" + found := false + + for _, val := range filteredNodeImages { + if val.SKU == "2204gen2containerd" && val.Version == expectedVersion { + found = true + break + } + } + + if !found { + t.Errorf("Expected to find node image with version %s but it was not present in the filtered results", expectedVersion) + } +} + +// This function tests that the node image versions that come from the fake are already filtered with the right values +// similar to the behavior we would see if someone is using the node image versions api call. +// the fake imports the same clientside filtering so we need to assert that behavior is the same +func TestFilteredNodeImageVersionsFromProviderList(t *testing.T) { + nodeImageVersionsAPI := NodeImageVersionsAPI{} + filteredNodeImages, err := nodeImageVersionsAPI.List(context.TODO(), "", "") + assert.Nil(t, err) + + expectedVersion := "202410.09.0" + found := false + + for _, val := range filteredNodeImages.Values { + if val.SKU == "2204gen2containerd" && val.Version == expectedVersion { + found = true + break + } + } + + if !found { + t.Errorf("Expected to find node image with version %s but it was not present in the filtered results", expectedVersion) + } +} diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 0cf48ff27..c3adfa6b6 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -98,12 +98,15 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont azConfig.Location, operator.Elected(), ) + imageProvider := imagefamily.NewProvider( operator.KubernetesInterface, cache.New(azurecache.KubernetesVersionTTL, azurecache.DefaultCleanupInterval), azClient.ImageVersionsClient, azConfig.Location, + azConfig.SubscriptionID, + azClient.NodeImageVersionsClient, ) imageResolver := imagefamily.New( operator.GetClient(), diff --git a/pkg/operator/options/options.go b/pkg/operator/options/options.go index 3e71012bd..60cc81cfd 100644 --- a/pkg/operator/options/options.go +++ b/pkg/operator/options/options.go @@ -75,9 +75,12 @@ type Options struct { SubnetID string // => VnetSubnetID to use (for nodes in Azure CNI Overlay and Azure CNI + pod subnet; for for nodes and pods in Azure CNI), unless overridden via AKSNodeClass setFlags map[string]bool - NodeResourceGroup string ProvisionMode string NodeBootstrappingServerURL string + UseSIG bool // => UseSIG is true if Karpenter is managed by AKS, false if it is a self-hosted karpenter installation + + SIGSubscriptionID string + NodeResourceGroup string } func (o *Options) AddFlags(fs *coreoptions.FlagSet) { @@ -92,9 +95,11 @@ func (o *Options) AddFlags(fs *coreoptions.FlagSet) { fs.StringVar(&o.NetworkDataplane, "network-dataplane", env.WithDefaultString("NETWORK_DATAPLANE", "cilium"), "The network dataplane used by the cluster.") fs.StringVar(&o.SubnetID, "vnet-subnet-id", env.WithDefaultString("VNET_SUBNET_ID", ""), "The default subnet ID to use for new nodes. This must be a valid ARM resource ID for subnet that does not overlap with the service CIDR or the pod CIDR.") fs.Var(newNodeIdentitiesValue(env.WithDefaultString("NODE_IDENTITIES", ""), &o.NodeIdentities), "node-identities", "User assigned identities for nodes.") - fs.StringVar(&o.NodeResourceGroup, "node-resource-group", env.WithDefaultString("AZURE_NODE_RESOURCE_GROUP", ""), "[REQUIRED] the resource group created and managed by AKS where the nodes live.") fs.StringVar(&o.ProvisionMode, "provision-mode", env.WithDefaultString("PROVISION_MODE", consts.ProvisionModeAKSScriptless), "[UNSUPPORTED] The provision mode for the cluster.") fs.StringVar(&o.NodeBootstrappingServerURL, "nodebootstrapping-server-url", env.WithDefaultString("NODEBOOTSTRAPPING_SERVER_URL", ""), "[UNSUPPORTED] The url for the node bootstrapping provider server.") + fs.StringVar(&o.NodeResourceGroup, "node-resource-group", env.WithDefaultString("AZURE_NODE_RESOURCE_GROUP", ""), "[REQUIRED] the resource group created and managed by AKS where the nodes live") + fs.BoolVar(&o.UseSIG, "use-sig", env.WithDefaultBool("USE_SIG", false), "If set to true karpenter will use the AKS managed shared image galleries and the node image versions api. If set to false karpenter will use community image galleries. Only a subset of image features will be available in the community image galleries and this flag is only for the managed node provisioning addon.") + fs.StringVar(&o.SIGSubscriptionID, "sig-subscription-id", env.WithDefaultString("SIG_SUBSCRIPTION_ID", ""), "The subscription ID of the shared image gallery.") } func (o Options) GetAPIServerName() string { diff --git a/pkg/providers/imagefamily/azlinux.go b/pkg/providers/imagefamily/azlinux.go index 6f4979f8d..67931aa50 100644 --- a/pkg/providers/imagefamily/azlinux.go +++ b/pkg/providers/imagefamily/azlinux.go @@ -30,9 +30,9 @@ import ( ) const ( - AzureLinuxGen2CommunityImage = "V2gen2" - AzureLinuxGen1CommunityImage = "V2" - AzureLinuxGen2ArmCommunityImage = "V2gen2arm64" + AzureLinuxGen2ImageDefinition = "V2gen2" + AzureLinuxGen1ImageDefinition = "V2" + AzureLinuxGen2ArmImageDefinition = "V2gen2arm64" ) type AzureLinux struct { @@ -47,8 +47,10 @@ func (u AzureLinux) DefaultImages() []DefaultImageOutput { // image provider will select these images in order, first match wins. This is why we chose to put Gen2 first in the defaultImages, as we prefer gen2 over gen1 return []DefaultImageOutput{ { - CommunityImage: AzureLinuxGen2CommunityImage, - PublicGalleryURL: AKSAzureLinuxPublicGalleryURL, + PublicGalleryURL: AKSAzureLinuxPublicGalleryURL, + GalleryResourceGroup: AKSAzureLinuxResourceGroup, + GalleryName: AKSAzureLinuxGalleryName, + ImageDefinition: AzureLinuxGen2ImageDefinition, Requirements: scheduling.NewRequirements( scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, karpv1.ArchitectureAmd64), scheduling.NewRequirement(v1alpha2.LabelSKUHyperVGeneration, v1.NodeSelectorOpIn, v1alpha2.HyperVGenerationV2), @@ -56,8 +58,10 @@ func (u AzureLinux) DefaultImages() []DefaultImageOutput { Distro: "aks-azurelinux-v2-gen2", }, { - CommunityImage: AzureLinuxGen1CommunityImage, - PublicGalleryURL: AKSAzureLinuxPublicGalleryURL, + PublicGalleryURL: AKSAzureLinuxPublicGalleryURL, + GalleryResourceGroup: AKSAzureLinuxResourceGroup, + GalleryName: AKSAzureLinuxGalleryName, + ImageDefinition: AzureLinuxGen1ImageDefinition, Requirements: scheduling.NewRequirements( scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, karpv1.ArchitectureAmd64), scheduling.NewRequirement(v1alpha2.LabelSKUHyperVGeneration, v1.NodeSelectorOpIn, v1alpha2.HyperVGenerationV1), @@ -65,8 +69,10 @@ func (u AzureLinux) DefaultImages() []DefaultImageOutput { Distro: "aks-azurelinux-v2", }, { - CommunityImage: AzureLinuxGen2ArmCommunityImage, - PublicGalleryURL: AKSAzureLinuxPublicGalleryURL, + PublicGalleryURL: AKSAzureLinuxPublicGalleryURL, + GalleryResourceGroup: AKSAzureLinuxResourceGroup, + GalleryName: AKSAzureLinuxGalleryName, + ImageDefinition: AzureLinuxGen2ArmImageDefinition, Requirements: scheduling.NewRequirements( scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, karpv1.ArchitectureArm64), scheduling.NewRequirement(v1alpha2.LabelSKUHyperVGeneration, v1.NodeSelectorOpIn, v1alpha2.HyperVGenerationV2), diff --git a/pkg/providers/imagefamily/image.go b/pkg/providers/imagefamily/image.go index b073db7e8..b80d74abb 100644 --- a/pkg/providers/imagefamily/image.go +++ b/pkg/providers/imagefamily/image.go @@ -19,12 +19,12 @@ package imagefamily import ( "context" "fmt" - "regexp" "strings" "time" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5" "github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2" + "github.com/Azure/karpenter-provider-azure/pkg/operator/options" "github.com/patrickmn/go-cache" "github.com/samber/lo" "k8s.io/client-go/kubernetes" @@ -40,6 +40,8 @@ type Provider struct { kubernetesInterface kubernetes.Interface imageCache *cache.Cache imageVersionsClient CommunityGalleryImageVersionsAPI + subscription string + NodeImageVersions NodeImageVersionsAPI } const ( @@ -48,10 +50,13 @@ const ( imageExpirationInterval = time.Hour * 24 * 3 imageCacheCleaningInterval = time.Hour * 1 - imageIDFormat = "/CommunityGalleries/%s/images/%s/versions/%s" + sharedImageKey = "%s/%s" // imageGallery + imageDefinition + sharedImageGalleryImageIDFormat = "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/galleries/%s/images/%s/versions/%s" + communityImageKey = "%s/%s" // PublicGalleryURL + communityImageName + communityImageIDFormat = "/CommunityGalleries/%s/images/%s/versions/%s" ) -func NewProvider(kubernetesInterface kubernetes.Interface, kubernetesVersionCache *cache.Cache, versionsClient CommunityGalleryImageVersionsAPI, location string) *Provider { +func NewProvider(kubernetesInterface kubernetes.Interface, kubernetesVersionCache *cache.Cache, versionsClient CommunityGalleryImageVersionsAPI, location, subscription string, nodeImageVersionsClient NodeImageVersionsAPI) *Provider { return &Provider{ kubernetesVersionCache: kubernetesVersionCache, imageCache: cache.New(imageExpirationInterval, imageCacheCleaningInterval), @@ -59,6 +64,8 @@ func NewProvider(kubernetesInterface kubernetes.Interface, kubernetesVersionCach imageVersionsClient: versionsClient, cm: pretty.NewChangeMonitor(), kubernetesInterface: kubernetesInterface, + subscription: subscription, + NodeImageVersions: nodeImageVersionsClient, } } @@ -67,18 +74,35 @@ func (p *Provider) Get(ctx context.Context, nodeClass *v1alpha2.AKSNodeClass, in defaultImages := imageFamily.DefaultImages() for _, defaultImage := range defaultImages { if err := instanceType.Requirements.Compatible(defaultImage.Requirements, v1alpha2.AllowUndefinedWellKnownAndRestrictedLabels); err == nil { - communityImageName, publicGalleryURL := defaultImage.CommunityImage, defaultImage.PublicGalleryURL - imageID, err := p.GetImageID(ctx, communityImageName, publicGalleryURL) - if err != nil { - return "", "", err - } - return defaultImage.Distro, imageID, nil + imageID, imageRetrievalErr := p.GetLatestImageID(ctx, defaultImage) + return defaultImage.Distro, imageID, imageRetrievalErr } } return "", "", fmt.Errorf("no compatible images found for instance type %s", instanceType.Name) } +func (p *Provider) GetLatestImageID(ctx context.Context, defaultImage DefaultImageOutput) (string, error) { + // Note: one could argue that we could narrow the key one level further to ImageDefinition since no two AKS ImageDefinitions that are supported + // by karpenter have the same name, but for EdgeZone support this is not the case. + key := lo.Ternary(options.FromContext(ctx).UseSIG, + fmt.Sprintf(sharedImageKey, defaultImage.GalleryName, defaultImage.ImageDefinition), + fmt.Sprintf(communityImageKey, defaultImage.PublicGalleryURL, defaultImage.ImageDefinition), + ) + if imageID, ok := p.imageCache.Get(key); ok { + return imageID.(string), nil + } + + // retrieve ARM Resource ID for the image and write it to the cache + imageID, err := p.resolveImageID(ctx, defaultImage, options.FromContext(ctx).UseSIG) + if err != nil { + return "", err + } + p.imageCache.Set(key, imageID, imageExpirationInterval) + logging.FromContext(ctx).With("image-id", imageID).Info("discovered new image id") + return imageID, nil +} + func (p *Provider) KubeServerVersion(ctx context.Context) (string, error) { if version, ok := p.kubernetesVersionCache.Get(kubernetesVersionCacheKey); ok { return version.(string), nil @@ -95,14 +119,36 @@ func (p *Provider) KubeServerVersion(ctx context.Context) (string, error) { return version, nil } -// Input versionName == "" to get the latest version -func (p *Provider) GetImageID(ctx context.Context, communityImageName, publicGalleryURL string) (string, error) { - key := fmt.Sprintf("%s/%s", publicGalleryURL, communityImageName) - imageID, found := p.imageCache.Get(key) - if found { - return imageID.(string), nil +func (p *Provider) resolveImageID(ctx context.Context, defaultImage DefaultImageOutput, useSIG bool) (string, error) { + if useSIG { + return p.getSIGImageID(ctx, defaultImage) + } + return p.getCIGImageID(defaultImage.PublicGalleryURL, defaultImage.ImageDefinition) +} + +func (p *Provider) getSIGImageID(ctx context.Context, imgStub DefaultImageOutput) (string, error) { + versions, err := p.NodeImageVersions.List(ctx, p.location, p.subscription) + if err != nil { + return "", err } + for _, version := range versions.Values { + if imgStub.ImageDefinition == version.SKU { + imageID := fmt.Sprintf(sharedImageGalleryImageIDFormat, options.FromContext(ctx).SIGSubscriptionID, imgStub.GalleryResourceGroup, imgStub.GalleryName, imgStub.ImageDefinition, version.Version) + return imageID, nil + } + } + return "", fmt.Errorf("failed to get the latest version of the image %s", imgStub.ImageDefinition) +} + +func (p *Provider) getCIGImageID(publicGalleryURL, communityImageName string) (string, error) { + imageVersion, err := p.latestNodeImageVersionCommunity(publicGalleryURL, communityImageName) + if err != nil { + return "", err + } + return BuildImageIDCIG(publicGalleryURL, communityImageName, imageVersion), nil +} +func (p *Provider) latestNodeImageVersionCommunity(publicGalleryURL, communityImageName string) (string, error) { pager := p.imageVersionsClient.NewListPager(p.location, publicGalleryURL, communityImageName, nil) topImageVersionCandidate := armcompute.CommunityGalleryImageVersion{} for pager.More() { @@ -116,34 +162,9 @@ func (p *Provider) GetImageID(ctx context.Context, communityImageName, publicGal } } } - versionName := lo.FromPtr(topImageVersionCandidate.Name) - - selectedImageID := BuildImageID(publicGalleryURL, communityImageName, versionName) - if p.cm.HasChanged(key, selectedImageID) { - logging.FromContext(ctx).With("image-id", selectedImageID).Info("discovered new image id") - } - p.imageCache.Set(key, selectedImageID, imageExpirationInterval) - return selectedImageID, nil -} - -func BuildImageID(publicGalleryURL, communityImageName, imageVersion string) string { - return fmt.Sprintf(imageIDFormat, publicGalleryURL, communityImageName, imageVersion) + return lo.FromPtr(topImageVersionCandidate.Name), nil } -// ParseImageIDInfo parses the publicGalleryURL, communityImageName, and imageVersion out of an imageID -func ParseCommunityImageIDInfo(imageID string) (string, string, string, error) { - // TODO (charliedmcb): assess if doing validation on splitting the string and validating the results is better? Mostly is regex too expensive? - regexStr := fmt.Sprintf(imageIDFormat, "(?P.*)", "(?P.*)", "(?P.*)") - if imageID == "" { - return "", "", "", fmt.Errorf("can not parse empty string. Expect it of the form \"%s\"", regexStr) - } - r := regexp.MustCompile(regexStr) - matches := r.FindStringSubmatch(imageID) - if matches == nil { - return "", "", "", fmt.Errorf("no matches while parsing image id %s", imageID) - } - if r.SubexpIndex("publicGalleryURL") == -1 || r.SubexpIndex("communityImageName") == -1 || r.SubexpIndex("imageVersion") == -1 { - return "", "", "", fmt.Errorf("failed to find sub expressions in %s, for imageID: %s", regexStr, imageID) - } - return matches[r.SubexpIndex("publicGalleryURL")], matches[r.SubexpIndex("communityImageName")], matches[r.SubexpIndex("imageVersion")], nil +func BuildImageIDCIG(publicGalleryURL, communityImageName, imageVersion string) string { + return fmt.Sprintf(communityImageIDFormat, publicGalleryURL, communityImageName, imageVersion) } diff --git a/pkg/providers/imagefamily/image_test.go b/pkg/providers/imagefamily/image_test.go deleted file mode 100644 index 0da43af9e..000000000 --- a/pkg/providers/imagefamily/image_test.go +++ /dev/null @@ -1,57 +0,0 @@ -/* -Portions Copyright (c) Microsoft Corporation. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package imagefamily_test - -import ( - "fmt" - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/Azure/karpenter-provider-azure/pkg/providers/imagefamily" -) - -func TestAzure(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Providers/ImageProvider/Azure") -} - -const ( - testImageID = "/CommunityGalleries/previewaks-1a06572d-8508-419c-a0d1-baffcbcb2f3b/images/2204gen2containerd/Versions/1.1685741267.25933" - olderImageVersion = "1.1686127203.20214" - latestImageVersion = "1.1686127203.20217" -) - -var _ = Describe("Image ID Parsing", func() { - DescribeTable("Parse Image ID", - func(imageID string, expectedPublicGalleryURL, expectedCommunityImageName, expectedImageVersion string, expectError bool) { - publicGalleryURL, communityImageName, imageVersion, err := imagefamily.ParseCommunityImageIDInfo(imageID) - if expectError { - Expect(err).To(HaveOccurred()) - return - } - Expect(err).To(BeNil()) - Expect(publicGalleryURL).To(Equal(expectedPublicGalleryURL)) - Expect(communityImageName).To(Equal(expectedCommunityImageName)) - Expect(imageVersion).To(Equal(expectedImageVersion)) - }, - Entry("Valid image id should parse", fmt.Sprintf("/CommunityGalleries/%s/images/%s/versions/%s", imagefamily.AKSUbuntuPublicGalleryURL, imagefamily.Ubuntu2204Gen2CommunityImage, olderImageVersion), imagefamily.AKSUbuntuPublicGalleryURL, imagefamily.Ubuntu2204Gen2CommunityImage, olderImageVersion, nil), - Entry("invalid image id should not parse", "badimageid", "", "", "", true), - Entry("empty image id should not parse", "badimageid", "", "", "", true), - ) -}) diff --git a/pkg/providers/imagefamily/nodeimageversionsclient.go b/pkg/providers/imagefamily/nodeimageversionsclient.go new file mode 100644 index 000000000..de9305a20 --- /dev/null +++ b/pkg/providers/imagefamily/nodeimageversionsclient.go @@ -0,0 +1,132 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package imagefamily + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strconv" + "strings" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" +) + +type NodeImageVersionsClient struct { + cred azcore.TokenCredential +} + +func NewNodeImageVersionsClient(cred azcore.TokenCredential) *NodeImageVersionsClient { + return &NodeImageVersionsClient{ + cred: cred, + } +} + +func (l *NodeImageVersionsClient) List(ctx context.Context, location, subscription string) (NodeImageVersionsResponse, error) { + resourceURL := fmt.Sprintf( + "https://management.azure.com/subscriptions/%s/providers/Microsoft.ContainerService/locations/%s/nodeImageVersions?api-version=%s", + subscription, location, "2024-04-02-preview", + ) + + token, err := l.cred.GetToken(ctx, policy.TokenRequestOptions{ + Scopes: []string{"https://management.azure.com/.default"}, + }) + if err != nil { + return NodeImageVersionsResponse{}, err + } + + req, err := http.NewRequestWithContext(context.Background(), "GET", resourceURL, nil) + if err != nil { + return NodeImageVersionsResponse{}, err + } + + req.Header.Set("Authorization", "Bearer "+token.Token) + req.Header.Set("Content-Type", "application/json") + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return NodeImageVersionsResponse{}, err + } + defer resp.Body.Close() + + var response NodeImageVersionsResponse + decoder := json.NewDecoder(resp.Body) + err = decoder.Decode(&response) + if err != nil { + return NodeImageVersionsResponse{}, err + } + + response.Values = FilteredNodeImages(response.Values) + return response, nil +} + +// FilteredNodeImages filters on two conditions +// 1. The image is the latest version for the given OS and SKU +// 2. the image belongs to a supported gallery(AKS Ubuntu or Azure Linux) +func FilteredNodeImages(nodeImageVersions []NodeImageVersion) []NodeImageVersion { + latestImages := make(map[string]NodeImageVersion) + + for _, image := range nodeImageVersions { + // Skip the galleries that Karpenter does not support + if image.OS != AKSUbuntuGalleryName && image.OS != AKSAzureLinuxGalleryName { + continue + } + + key := image.OS + "-" + image.SKU + + currentLatest, exists := latestImages[key] + if !exists || isNewerVersion(image.Version, currentLatest.Version) { + latestImages[key] = image + } + } + + var filteredImages []NodeImageVersion + for _, image := range latestImages { + filteredImages = append(filteredImages, image) + } + return filteredImages +} + +// isNewerVersion will return if version1 is greater than version2, note the new versioning scheme is yearmm.dd.build, previously it was yy.mm.dd without the build id. +func isNewerVersion(version1, version2 string) bool { + // Split by dots and compare each segment as an integer getting the largest vhd version + v1Segments := strings.Split(version1, ".") + v2Segments := strings.Split(version2, ".") + + for i := 0; i < len(v1Segments) && i < len(v2Segments); i++ { + v1Segment, err1 := strconv.Atoi(v1Segments[i]) + v2Segment, err2 := strconv.Atoi(v2Segments[i]) + + if err1 != nil || err2 != nil { + return false + } + + if v1Segment > v2Segment { + return true + } else if v1Segment < v2Segment { + return false + } + } + + // If all segments are equal up to the length of the shorter version, + // the longer version is considered newer if it has additional segments + // the legacy linux versions use "yy.mm.dd" whereas new linux versions use "yymm.dd.build" + return len(v1Segments) > len(v2Segments) +} diff --git a/pkg/providers/imagefamily/nodeimageversionsclient_test.go b/pkg/providers/imagefamily/nodeimageversionsclient_test.go new file mode 100644 index 000000000..078b08fc7 --- /dev/null +++ b/pkg/providers/imagefamily/nodeimageversionsclient_test.go @@ -0,0 +1,49 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package imagefamily + +import ( + "testing" +) + +func TestIsNewerVersion(t *testing.T) { + testCases := []struct { + version1 string + version2 string + expected bool + }{ + {"202308.28.0", "202411.12.0", false}, + {"202411.12.0", "202308.28.0", true}, + {"202202.08.29", "202405.20.0", false}, + {"202404.09.0", "202411.12.0", false}, + {"202405.20.0", "202404.09.0", true}, + {"2022.10.03", "2022.12.15", false}, + {"202411.12.0", "2022.12.15", true}, + {"2022.12.15", "2022.10.03", true}, + {"202411.12.0", "202411.12.0", false}, + {"2o2411.12.0", "202411.12.0", false}, // invalid version strings should be ignored and return false + } + + for _, tc := range testCases { + t.Run(tc.version1+"_"+tc.version2, func(t *testing.T) { + result := isNewerVersion(tc.version1, tc.version2) + if result != tc.expected { + t.Errorf("isNewerVersion(%q, %q) = %v; want %v", tc.version1, tc.version2, result, tc.expected) + } + }) + } +} diff --git a/pkg/providers/imagefamily/types.go b/pkg/providers/imagefamily/types.go index b3c3510aa..009131069 100644 --- a/pkg/providers/imagefamily/types.go +++ b/pkg/providers/imagefamily/types.go @@ -17,6 +17,9 @@ limitations under the License. package imagefamily import ( + "context" + "strings" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" armcomputev5 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5" "sigs.k8s.io/karpenter/pkg/scheduling" @@ -25,17 +28,57 @@ import ( const ( AKSUbuntuPublicGalleryURL = "AKSUbuntu-38d80f77-467a-481f-a8d4-09b6d4220bd2" AKSAzureLinuxPublicGalleryURL = "AKSAzureLinux-f7c7cda5-1c9a-4bdc-a222-9614c968580b" + + AKSUbuntuResourceGroup = "AKS-Ubuntu" + AKSAzureLinuxResourceGroup = "AKS-AzureLinux" + + AKSUbuntuGalleryName = "AKSUbuntu" + AKSAzureLinuxGalleryName = "AKSAzureLinux" ) -// DefaultImageOutput is the Stub of an Image we return from an ImageFamily De +// DefaultImageOutput is a stub describing our desired image with an image's name and requirements to run that image type DefaultImageOutput struct { - CommunityImage string + // Community Image Gallery PublicGalleryURL string - Requirements scheduling.Requirements - Distro string + // Shared Image Gallery + GalleryResourceGroup string + GalleryName string + // Common + ImageDefinition string + Distro string + Requirements scheduling.Requirements +} + +func (d *DefaultImageOutput) PopulateImageTraitsFromID(imageID string) { + // We want to take a community image gallery image id or a shared image gallery id and populate the contents of DefaultImageOutput + imageIDParts := strings.Split(imageID, "/") + if imageIDParts[1] == "subscriptions" { // Shared Image Gallery + d.GalleryResourceGroup = imageIDParts[4] + d.GalleryName = imageIDParts[8] + d.ImageDefinition = imageIDParts[10] + } + if imageIDParts[1] == "CommunityGalleries" { // Community Image Gallery + d.PublicGalleryURL = imageIDParts[2] + d.ImageDefinition = imageIDParts[4] + } } // CommunityGalleryImageVersionsAPI is used for listing community gallery image versions. type CommunityGalleryImageVersionsAPI interface { NewListPager(location string, publicGalleryName string, galleryImageName string, options *armcomputev5.CommunityGalleryImageVersionsClientListOptions) *runtime.Pager[armcomputev5.CommunityGalleryImageVersionsClientListResponse] } + +type NodeImageVersion struct { + FullName string `json:"fullName"` + OS string `json:"os"` + SKU string `json:"sku"` + Version string `json:"version"` +} + +type NodeImageVersionsResponse struct { + Values []NodeImageVersion `json:"values"` +} + +type NodeImageVersionsAPI interface { + List(ctx context.Context, location, subscription string) (NodeImageVersionsResponse, error) +} diff --git a/pkg/providers/imagefamily/ubuntu_2204.go b/pkg/providers/imagefamily/ubuntu_2204.go index 54806cba2..df034db8a 100644 --- a/pkg/providers/imagefamily/ubuntu_2204.go +++ b/pkg/providers/imagefamily/ubuntu_2204.go @@ -30,9 +30,9 @@ import ( ) const ( - Ubuntu2204Gen2CommunityImage = "2204gen2containerd" - Ubuntu2204Gen1CommunityImage = "2204containerd" - Ubuntu2204Gen2ArmCommunityImage = "2204gen2arm64containerd" + Ubuntu2204Gen2ImageDefinition = "2204gen2containerd" + Ubuntu2204Gen1ImageDefinition = "2204containerd" + Ubuntu2204Gen2ArmImageDefinition = "2204gen2arm64containerd" ) type Ubuntu2204 struct { @@ -47,8 +47,10 @@ func (u Ubuntu2204) DefaultImages() []DefaultImageOutput { // image provider will select these images in order, first match wins. This is why we chose to put Ubuntu2204Gen2containerd first in the defaultImages return []DefaultImageOutput{ { - CommunityImage: Ubuntu2204Gen2CommunityImage, - PublicGalleryURL: AKSUbuntuPublicGalleryURL, + PublicGalleryURL: AKSUbuntuPublicGalleryURL, + GalleryResourceGroup: AKSUbuntuResourceGroup, + GalleryName: AKSUbuntuGalleryName, + ImageDefinition: Ubuntu2204Gen2ImageDefinition, Requirements: scheduling.NewRequirements( scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, karpv1.ArchitectureAmd64), scheduling.NewRequirement(v1alpha2.LabelSKUHyperVGeneration, v1.NodeSelectorOpIn, v1alpha2.HyperVGenerationV2), @@ -56,8 +58,10 @@ func (u Ubuntu2204) DefaultImages() []DefaultImageOutput { Distro: "aks-ubuntu-containerd-22.04-gen2", }, { - CommunityImage: Ubuntu2204Gen1CommunityImage, - PublicGalleryURL: AKSUbuntuPublicGalleryURL, + PublicGalleryURL: AKSUbuntuPublicGalleryURL, + GalleryResourceGroup: AKSUbuntuResourceGroup, + GalleryName: AKSUbuntuGalleryName, + ImageDefinition: Ubuntu2204Gen1ImageDefinition, Requirements: scheduling.NewRequirements( scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, karpv1.ArchitectureAmd64), scheduling.NewRequirement(v1alpha2.LabelSKUHyperVGeneration, v1.NodeSelectorOpIn, v1alpha2.HyperVGenerationV1), @@ -65,8 +69,10 @@ func (u Ubuntu2204) DefaultImages() []DefaultImageOutput { Distro: "aks-ubuntu-containerd-22.04", }, { - CommunityImage: Ubuntu2204Gen2ArmCommunityImage, - PublicGalleryURL: AKSUbuntuPublicGalleryURL, + PublicGalleryURL: AKSUbuntuPublicGalleryURL, + GalleryResourceGroup: AKSUbuntuResourceGroup, + GalleryName: AKSUbuntuGalleryName, + ImageDefinition: Ubuntu2204Gen2ArmImageDefinition, Requirements: scheduling.NewRequirements( scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, karpv1.ArchitectureArm64), scheduling.NewRequirement(v1alpha2.LabelSKUHyperVGeneration, v1.NodeSelectorOpIn, v1alpha2.HyperVGenerationV2), diff --git a/pkg/providers/instance/azure_client.go b/pkg/providers/instance/azure_client.go index 62a41f99b..ff797d4d2 100644 --- a/pkg/providers/instance/azure_client.go +++ b/pkg/providers/instance/azure_client.go @@ -63,7 +63,8 @@ type AZClient struct { virtualMachinesExtensionClient VirtualMachineExtensionsAPI networkInterfacesClient NetworkInterfacesAPI - ImageVersionsClient imagefamily.CommunityGalleryImageVersionsAPI + NodeImageVersionsClient imagefamily.NodeImageVersionsAPI + ImageVersionsClient imagefamily.CommunityGalleryImageVersionsAPI // SKU CLIENT is still using track 1 because skewer does not support the track 2 path. We need to refactor this once skewer supports track 2 SKUClient skuclient.SkuClient LoadBalancersClient loadbalancer.LoadBalancersAPI @@ -76,6 +77,7 @@ func NewAZClientFromAPI( interfacesClient NetworkInterfacesAPI, loadBalancersClient loadbalancer.LoadBalancersAPI, imageVersionsClient imagefamily.CommunityGalleryImageVersionsAPI, + nodeImageVersionsClient imagefamily.NodeImageVersionsAPI, skuClient skuclient.SkuClient, ) *AZClient { return &AZClient{ @@ -84,6 +86,7 @@ func NewAZClientFromAPI( virtualMachinesExtensionClient: virtualMachinesExtensionClient, networkInterfacesClient: interfacesClient, ImageVersionsClient: imageVersionsClient, + NodeImageVersionsClient: nodeImageVersionsClient, SKUClient: skuClient, LoadBalancersClient: loadBalancersClient, } @@ -137,11 +140,13 @@ func NewAZClient(ctx context.Context, cfg *auth.Config, env *azure.Environment) } klog.V(5).Infof("Created azure resource graph client %v, using a token credential", azureResourceGraphClient) - imageVersionsClient, err := armcomputev5.NewCommunityGalleryImageVersionsClient(cfg.SubscriptionID, cred, opts) + communityImageVersionsClient, err := armcomputev5.NewCommunityGalleryImageVersionsClient(cfg.SubscriptionID, cred, opts) if err != nil { return nil, err } - klog.V(5).Infof("Created image versions client %v, using a token credential", imageVersionsClient) + klog.V(5).Infof("Created image versions client %v, using a token credential", communityImageVersionsClient) + + nodeImageVersionsClient := imagefamily.NewNodeImageVersionsClient(cred) loadBalancersClient, err := armnetwork.NewLoadBalancersClient(cfg.SubscriptionID, cred, opts) if err != nil { @@ -158,6 +163,7 @@ func NewAZClient(ctx context.Context, cfg *auth.Config, env *azure.Environment) extensionsClient, interfacesClient, loadBalancersClient, - imageVersionsClient, + communityImageVersionsClient, + nodeImageVersionsClient, skuClient), nil } diff --git a/pkg/providers/instance/instance.go b/pkg/providers/instance/instance.go index 89939746e..0149c875e 100644 --- a/pkg/providers/instance/instance.go +++ b/pkg/providers/instance/instance.go @@ -314,19 +314,14 @@ func newVMObject( vmName, nicReference, zone, - capacityType string, - location string, + capacityType, + location, sshPublicKey string, nodeIdentities []string, nodeClass *v1alpha2.AKSNodeClass, launchTemplate *launchtemplate.Template, instanceType *corecloudprovider.InstanceType, - provisionMode string) armcompute.VirtualMachine { - // Build the image reference from template - imageReference := armcompute.ImageReference{ - CommunityGalleryImageID: &launchTemplate.ImageID, - } - + provisionMode string, useSIG bool) armcompute.VirtualMachine { if launchTemplate.IsWindows { return armcompute.VirtualMachine{} // TODO(Windows) } @@ -346,7 +341,6 @@ func newVMObject( CreateOption: lo.ToPtr(armcompute.DiskCreateOptionTypesFromImage), DeleteOption: lo.ToPtr(armcompute.DiskDeleteOptionTypesDelete), }, - ImageReference: &imageReference, }, NetworkProfile: &armcompute.NetworkProfile{ @@ -384,6 +378,7 @@ func newVMObject( Tags: launchTemplate.Tags, } setVMPropertiesOSDiskType(vm.Properties, launchTemplate.StorageProfile) + setImageReference(vm.Properties, launchTemplate.ImageID, useSIG) setVMPropertiesBillingProfile(vm.Properties, capacityType) if provisionMode == consts.ProvisionModeBootstrappingClient { @@ -406,6 +401,19 @@ func setVMPropertiesOSDiskType(vmProperties *armcompute.VirtualMachineProperties } } +// setImageReference sets the image reference for the VM based on if we are using self hosted karpenter or the node auto provisioning addon +func setImageReference(vmProperties *armcompute.VirtualMachineProperties, imageID string, useSIG bool) { + if useSIG { + vmProperties.StorageProfile.ImageReference = &armcompute.ImageReference{ + ID: lo.ToPtr(imageID), + } + return + } + vmProperties.StorageProfile.ImageReference = &armcompute.ImageReference{ + CommunityGalleryImageID: lo.ToPtr(imageID), + } +} + // setVMPropertiesBillingProfile sets a default MaxPrice of -1 for Spot func setVMPropertiesBillingProfile(vmProperties *armcompute.VirtualMachineProperties, capacityType string) { if capacityType == karpv1.CapacityTypeSpot { @@ -471,7 +479,8 @@ func (p *DefaultProvider) launchInstance( sshPublicKey := options.FromContext(ctx).SSHPublicKey nodeIdentityIDs := options.FromContext(ctx).NodeIdentities - vm := newVMObject(resourceName, nicReference, zone, capacityType, p.location, sshPublicKey, nodeIdentityIDs, nodeClass, launchTemplate, instanceType, p.provisionMode) + useSIG := options.FromContext(ctx).UseSIG + vm := newVMObject(resourceName, nicReference, zone, capacityType, p.location, sshPublicKey, nodeIdentityIDs, nodeClass, launchTemplate, instanceType, p.provisionMode, useSIG) logging.FromContext(ctx).Debugf("Creating virtual machine %s (%s)", resourceName, instanceType.Name) // Uses AZ Client to create a new virtual machine using the vm object we prepared earlier diff --git a/pkg/providers/instancetype/instancetypes.go b/pkg/providers/instancetype/instancetypes.go index f5b400f68..946db59e8 100644 --- a/pkg/providers/instancetype/instancetypes.go +++ b/pkg/providers/instancetype/instancetypes.go @@ -36,6 +36,7 @@ import ( "github.com/Azure/go-autorest/autorest/to" "github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2" kcache "github.com/Azure/karpenter-provider-azure/pkg/cache" + "github.com/Azure/karpenter-provider-azure/pkg/operator/options" "github.com/Azure/karpenter-provider-azure/pkg/utils" "github.com/patrickmn/go-cache" "k8s.io/apimachinery/pkg/util/sets" @@ -274,8 +275,7 @@ func (p *DefaultProvider) getInstanceTypes(ctx context.Context) (map[string]*ske logging.FromContext(ctx).Errorf("parsing VM size %s, %v", *skus[i].Size, err) continue } - - useSIG := false // replace with options.FromContext(ctx).UseSIG when available + useSIG := options.FromContext(ctx).UseSIG if !skus[i].HasLocationRestriction(p.region) && p.isSupported(&skus[i], vmsize, useSIG) { instanceTypes[skus[i].GetName()] = &skus[i] } diff --git a/pkg/providers/instancetype/suite_test.go b/pkg/providers/instancetype/suite_test.go index ce27f1290..ae1f0d515 100644 --- a/pkg/providers/instancetype/suite_test.go +++ b/pkg/providers/instancetype/suite_test.go @@ -128,6 +128,7 @@ var _ = Describe("InstanceType Provider", func() { }, }) + ctx = options.ToContext(ctx, test.Options()) cluster.Reset() clusterNonZonal.Reset() azureEnv.Reset() @@ -885,7 +886,79 @@ var _ = Describe("InstanceType Provider", func() { }) }) + Context("ImageReference", func() { + It("should use shared image gallery images when options are set to UseSIG", func() { + options := test.Options(test.OptionsFields{ + UseSIG: lo.ToPtr(true), + }) + ctx = options.ToContext(ctx) + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + pod := coretest.UnschedulablePod(coretest.PodOptions{}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod) + ExpectScheduled(ctx, env.Client, pod) + + // Expect virtual machine to have a shared image gallery id set on it + Expect(azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Len()).To(Equal(1)) + vm := azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Pop().VM + Expect(vm.Properties.StorageProfile.ImageReference).ToNot(BeNil()) + Expect(vm.Properties.StorageProfile.ImageReference.ID).ShouldNot(BeNil()) + Expect(vm.Properties.StorageProfile.ImageReference.CommunityGalleryImageID).Should(BeNil()) + + Expect(*vm.Properties.StorageProfile.ImageReference.ID).To(ContainSubstring(options.SIGSubscriptionID)) + Expect(*vm.Properties.StorageProfile.ImageReference.ID).To(ContainSubstring("AKSUbuntu")) + }) + It("should use Community Images when options are set to UseSIG=false", func() { + options := test.Options(test.OptionsFields{ + UseSIG: lo.ToPtr(false), + }) + ctx = options.ToContext(ctx) + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + pod := coretest.UnschedulablePod(coretest.PodOptions{}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod) + ExpectScheduled(ctx, env.Client, pod) + + Expect(azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Len()).To(Equal(1)) + vm := azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Pop().VM + Expect(vm.Properties.StorageProfile.ImageReference.CommunityGalleryImageID).Should(Not(BeNil())) + + }) + + }) Context("ImageProvider + Image Family", func() { + DescribeTable("should select the right Shared Image Gallery image for a given instance type", func(instanceType string, imageFamily string, expectedImageDefinition string, expectedGalleryRG string, expectedGalleryURL string) { + options := test.Options(test.OptionsFields{ + UseSIG: lo.ToPtr(true), + }) + ctx = options.ToContext(ctx) + nodeClass.Spec.ImageFamily = lo.ToPtr(imageFamily) + coretest.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelInstanceTypeStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{instanceType}, + }}) + nodePool.Spec.Template.Spec.NodeClassRef = &karpv1.NodeClassReference{Name: nodeClass.Name} + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + pod := coretest.UnschedulablePod(coretest.PodOptions{}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod) + ExpectScheduled(ctx, env.Client, pod) + + Expect(azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Len()).To(Equal(1)) + vm := azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Pop().VM + Expect(vm.Properties.StorageProfile.ImageReference).ToNot(BeNil()) + + expectedPrefix := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/galleries/%s/images/%s", options.SIGSubscriptionID, expectedGalleryRG, expectedGalleryURL, expectedImageDefinition) + Expect(*vm.Properties.StorageProfile.ImageReference.ID).To(ContainSubstring(expectedPrefix)) + + }, + + Entry("Gen2, Gen1 instance type with AKSUbuntu image family", "Standard_D2_v5", v1alpha2.Ubuntu2204ImageFamily, imagefamily.Ubuntu2204Gen2ImageDefinition, imagefamily.AKSUbuntuResourceGroup, imagefamily.AKSUbuntuGalleryName), + Entry("Gen1 instance type with AKSUbuntu image family", "Standard_D2_v3", v1alpha2.Ubuntu2204ImageFamily, imagefamily.Ubuntu2204Gen1ImageDefinition, imagefamily.AKSUbuntuResourceGroup, imagefamily.AKSUbuntuGalleryName), + Entry("ARM instance type with AKSUbuntu image family", "Standard_D16plds_v5", v1alpha2.Ubuntu2204ImageFamily, imagefamily.Ubuntu2204Gen2ArmImageDefinition, imagefamily.AKSUbuntuResourceGroup, imagefamily.AKSUbuntuGalleryName), + Entry("Gen2 instance type with AzureLinux image family", "Standard_D2_v5", v1alpha2.AzureLinuxImageFamily, imagefamily.AzureLinuxGen2ImageDefinition, imagefamily.AKSAzureLinuxResourceGroup, imagefamily.AKSAzureLinuxGalleryName), + Entry("Gen1 instance type with AzureLinux image family", "Standard_D2_v3", v1alpha2.AzureLinuxImageFamily, imagefamily.AzureLinuxGen1ImageDefinition, imagefamily.AKSAzureLinuxResourceGroup, imagefamily.AKSAzureLinuxGalleryName), + Entry("ARM instance type with AzureLinux image family", "Standard_D16plds_v5", v1alpha2.AzureLinuxImageFamily, imagefamily.AzureLinuxGen2ArmImageDefinition, imagefamily.AKSAzureLinuxResourceGroup, imagefamily.AKSAzureLinuxGalleryName), + ) DescribeTable("should select the right image for a given instance type", func(instanceType string, imageFamily string, expectedImageDefinition string, expectedGalleryURL string) { nodeClass.Spec.ImageFamily = lo.ToPtr(imageFamily) @@ -914,17 +987,17 @@ var _ = Describe("InstanceType Provider", func() { azureEnv.Reset() }, Entry("Gen2, Gen1 instance type with AKSUbuntu image family", - "Standard_D2_v5", v1alpha2.Ubuntu2204ImageFamily, imagefamily.Ubuntu2204Gen2CommunityImage, imagefamily.AKSUbuntuPublicGalleryURL), + "Standard_D2_v5", v1alpha2.Ubuntu2204ImageFamily, imagefamily.Ubuntu2204Gen2ImageDefinition, imagefamily.AKSUbuntuPublicGalleryURL), Entry("Gen1 instance type with AKSUbuntu image family", - "Standard_D2_v3", v1alpha2.Ubuntu2204ImageFamily, imagefamily.Ubuntu2204Gen1CommunityImage, imagefamily.AKSUbuntuPublicGalleryURL), + "Standard_D2_v3", v1alpha2.Ubuntu2204ImageFamily, imagefamily.Ubuntu2204Gen1ImageDefinition, imagefamily.AKSUbuntuPublicGalleryURL), Entry("ARM instance type with AKSUbuntu image family", - "Standard_D16plds_v5", v1alpha2.Ubuntu2204ImageFamily, imagefamily.Ubuntu2204Gen2ArmCommunityImage, imagefamily.AKSUbuntuPublicGalleryURL), + "Standard_D16plds_v5", v1alpha2.Ubuntu2204ImageFamily, imagefamily.Ubuntu2204Gen2ArmImageDefinition, imagefamily.AKSUbuntuPublicGalleryURL), Entry("Gen2 instance type with AzureLinux image family", - "Standard_D2_v5", v1alpha2.AzureLinuxImageFamily, imagefamily.AzureLinuxGen2CommunityImage, imagefamily.AKSAzureLinuxPublicGalleryURL), + "Standard_D2_v5", v1alpha2.AzureLinuxImageFamily, imagefamily.AzureLinuxGen2ImageDefinition, imagefamily.AKSAzureLinuxPublicGalleryURL), Entry("Gen1 instance type with AzureLinux image family", - "Standard_D2_v3", v1alpha2.AzureLinuxImageFamily, imagefamily.AzureLinuxGen1CommunityImage, imagefamily.AKSAzureLinuxPublicGalleryURL), + "Standard_D2_v3", v1alpha2.AzureLinuxImageFamily, imagefamily.AzureLinuxGen1ImageDefinition, imagefamily.AKSAzureLinuxPublicGalleryURL), Entry("ARM instance type with AzureLinux image family", - "Standard_D16plds_v5", v1alpha2.AzureLinuxImageFamily, imagefamily.AzureLinuxGen2ArmCommunityImage, imagefamily.AKSAzureLinuxPublicGalleryURL), + "Standard_D16plds_v5", v1alpha2.AzureLinuxImageFamily, imagefamily.AzureLinuxGen2ArmImageDefinition, imagefamily.AKSAzureLinuxPublicGalleryURL), ) }) Context("Instance Types", func() { diff --git a/pkg/test/environment.go b/pkg/test/environment.go index 30b6777bf..11d3faf0e 100644 --- a/pkg/test/environment.go +++ b/pkg/test/environment.go @@ -46,6 +46,7 @@ func init() { var ( resourceGroup = "test-resourceGroup" + subscription = "12345678-1234-1234-1234-123456789012" ) type Environment struct { @@ -98,6 +99,7 @@ func NewRegionalEnvironment(ctx context.Context, env *coretest.Environment, regi skuClientSingleton := &fake.MockSkuClientSingleton{SKUClient: &fake.ResourceSKUsAPI{Location: region}} communityImageVersionsAPI := &fake.CommunityGalleryImageVersionsAPI{} loadBalancersAPI := &fake.LoadBalancersAPI{} + nodeImageVersionsAPI := &fake.NodeImageVersionsAPI{} // Cache kubernetesVersionCache := cache.New(azurecache.KubernetesVersionTTL, azurecache.DefaultCleanupInterval) @@ -107,7 +109,7 @@ func NewRegionalEnvironment(ctx context.Context, env *coretest.Environment, regi // Providers pricingProvider := pricing.NewProvider(ctx, pricingAPI, region, make(chan struct{})) - imageFamilyProvider := imagefamily.NewProvider(env.KubernetesInterface, kubernetesVersionCache, communityImageVersionsAPI, region) + imageFamilyProvider := imagefamily.NewProvider(env.KubernetesInterface, kubernetesVersionCache, communityImageVersionsAPI, region, subscription, nodeImageVersionsAPI) imageFamilyResolver := imagefamily.New(env.Client, imageFamilyProvider) instanceTypesProvider := instancetype.NewDefaultProvider(region, instanceTypeCache, skuClientSingleton, pricingProvider, unavailableOfferingsCache) launchTemplateProvider := launchtemplate.NewProvider( @@ -117,7 +119,7 @@ func NewRegionalEnvironment(ctx context.Context, env *coretest.Environment, regi ptr.String("ca-bundle"), testOptions.ClusterEndpoint, "test-tenant", - "test-subscription", + subscription, "test-cluster-resource-group", "test-kubelet-identity-client-id", testOptions.NodeResourceGroup, @@ -137,6 +139,7 @@ func NewRegionalEnvironment(ctx context.Context, env *coretest.Environment, regi networkInterfacesAPI, loadBalancersAPI, communityImageVersionsAPI, + nodeImageVersionsAPI, skuClientSingleton, ) instanceProvider := instance.NewDefaultProvider( @@ -147,7 +150,7 @@ func NewRegionalEnvironment(ctx context.Context, env *coretest.Environment, regi unavailableOfferingsCache, region, testOptions.NodeResourceGroup, - "", // subscriptionID + subscription, testOptions.ProvisionMode, ) diff --git a/pkg/test/options.go b/pkg/test/options.go index 32e2b2734..4cdbbb3a4 100644 --- a/pkg/test/options.go +++ b/pkg/test/options.go @@ -41,6 +41,10 @@ type OptionsFields struct { NodeResourceGroup *string ProvisionMode *string NodeBootstrappingServerURL *string + + // UseSIG Flags not required by the self hosted offering + UseSIG *bool + SIGSubscriptionID *string } func Options(overrides ...OptionsFields) *azoptions.Options { @@ -65,5 +69,7 @@ func Options(overrides ...OptionsFields) *azoptions.Options { SubnetID: lo.FromPtrOr(options.SubnetID, "/subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/sillygeese/providers/Microsoft.Network/virtualNetworks/karpentervnet/subnets/karpentersub"), NodeResourceGroup: lo.FromPtrOr(options.NodeResourceGroup, "test-resourceGroup"), ProvisionMode: lo.FromPtrOr(options.ProvisionMode, "aksscriptless"), + UseSIG: lo.FromPtrOr(options.UseSIG, false), + SIGSubscriptionID: lo.FromPtrOr(options.SIGSubscriptionID, "10945678-1234-1234-1234-123456789012"), } }