From a33da0aa7f6aa853359689954df8878035a94968 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 17 Dec 2024 13:06:27 -0800 Subject: [PATCH 1/6] chore(deps): bump golang.org/x/crypto from 0.27.0 to 0.31.0 (#626) Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.27.0 to 0.31.0. - [Commits](https://github.com/golang/crypto/compare/v0.27.0...v0.31.0) --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Alex Leites <18728999+tallaxes@users.noreply.github.com> --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index c8fc14502..f92355a4d 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( github.com/stretchr/testify v1.9.0 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.0 - golang.org/x/sync v0.8.0 + golang.org/x/sync v0.10.0 gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.30.3 k8s.io/apiextensions-apiserver v0.30.3 @@ -142,13 +142,13 @@ require ( go.opentelemetry.io/otel/trace v1.24.0 // indirect go.uber.org/automaxprocs v1.5.3 // indirect go.uber.org/mock v0.4.0 // indirect - golang.org/x/crypto v0.27.0 // indirect + golang.org/x/crypto v0.31.0 // indirect golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa // indirect golang.org/x/net v0.29.0 // indirect golang.org/x/oauth2 v0.22.0 // indirect - golang.org/x/sys v0.25.0 // indirect - golang.org/x/term v0.24.0 // indirect - golang.org/x/text v0.18.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/term v0.27.0 // indirect + golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.6.0 // indirect golang.org/x/tools v0.24.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 92808e710..65ead3fb2 100644 --- a/go.sum +++ b/go.sum @@ -506,8 +506,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= -golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -607,8 +607,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= -golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -655,15 +655,15 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= -golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= -golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= -golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -674,8 +674,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= -golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From 746ec9ac2ed812761aaad09b7aca06d9e82fd60f Mon Sep 17 00:00:00 2001 From: Bryce Soghigian <49734722+Bryce-Soghigian@users.noreply.github.com> Date: Fri, 20 Dec 2024 13:47:51 -0800 Subject: [PATCH 2/6] test: adding manual step for deploying goldpinger (#629) --- Makefile-az.mk | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile-az.mk b/Makefile-az.mk index 6702658b0..d5ecf67e3 100755 --- a/Makefile-az.mk +++ b/Makefile-az.mk @@ -201,6 +201,10 @@ az-debug-bootstrap: ## Debug bootstrap (target first privateIP of the first NIC az-cleanup: ## Delete the deployment skaffold delete || true +az-deploy-goldpinger: ## Deploy goldpinger for testing networking + kubectl apply -f https://gist.githubusercontent.com/paulgmiller/084bd4605f1661a329e5ab891a826ae0/raw/94a32d259e137bb300ac8af3ef71caa471463f23/goldpinger-daemon.yaml + kubectl apply -f https://gist.githubusercontent.com/paulgmiller/7bca68cd08cccb4e9bc72b0a08485edf/raw/d6a103fb79a65083f6555e4d822554ed64f510f8/goldpinger-deploy.yaml + az-mon-deploy: ## Deploy monitoring stack (w/o node-exporter) helm repo add grafana-charts https://grafana.github.io/helm-charts helm repo add prometheus-community https://prometheus-community.github.io/helm-charts From 3a9cbc0d7bf64e4637fb6803bbd26b4c248cacca Mon Sep 17 00:00:00 2001 From: Bryce Soghigian <49734722+Bryce-Soghigian@users.noreply.github.com> Date: Sat, 21 Dec 2024 04:18:44 -0800 Subject: [PATCH 3/6] feat: ListNodeImageVersions + shared image gallery support (#526) * feat: support shared image galleries inside of karpenter * chore: populating image stubs for shared image galleries * fix: progress * fix: PopulateResourceStub accessing the wrong index * test: properly testing ListNodeImageVersions * refactor: rename symbol for SIG Subscription id * test: conditional use of sig dependent on the managed karpenter flag * refactor: removing panics used in testing * test: adding RBAC and helm values to the template for SIG Gallery logic * fix: bug in azure linux sig image resolution * chore: update cleanupenv to handle inflate too ratehr than just job pods * test: fix randomized test order flake * ci: shadow declaration * refactor: comment wording * test: validate all image ids are resolved correctly * fix: adding filtering for duplicate sku + os combinations and filtering out unsupported galleries * refactor: renaming var * refactor: rename the managedKarpenter reference to UseSIG * refactor: spelling * fix: v1 migration for test * ci: lint * fix: lint * ci: fix * ci: license * ci: fix * fix: test pollution * fix: resetting options before each test run * fix: accounting for versions of the shape 'yy.mm.dd' * test: removing unused cleanup funcs * fix: making the key for shared image gallery smaller * refactor: removing windows types leaving them to be added back later * refactor: reducing key even more * fix: extending key * fix: removing log line * refactor: not nesting options as deep, leaving refactor of USESIG to provider for later * fix: comment about pulling out the variable * refactor: decoupling the cache reads and cache writes from the image gallery id retrivial functions * test: adding some coverage to FilteredNodeImages * refactor: removing community galleries fake method for GET since this implementation doesn't use it * test: test that we drift nodes when switching from community gallery to shared image gallery * test: SIG_SUBSCRIPTION_ID * refactor: using sig from context * fix: remove extra space --------- Co-authored-by: Alex Leites <18728999+tallaxes@users.noreply.github.com> --- Makefile-az.mk | 9 +- karpenter-values-template.yaml | 6 + pkg/cloudprovider/drift.go | 18 +- pkg/cloudprovider/suite_test.go | 10 + pkg/fake/nodeimageversionsapi.go | 381 ++++++++++++++++++ pkg/fake/nodeimageversionsapi_test.go | 101 +++++ pkg/operator/operator.go | 3 + pkg/operator/options/options.go | 9 +- pkg/providers/imagefamily/azlinux.go | 24 +- pkg/providers/imagefamily/image.go | 107 +++-- pkg/providers/imagefamily/image_test.go | 57 --- .../imagefamily/nodeimageversionsclient.go | 132 ++++++ .../nodeimageversionsclient_test.go | 49 +++ pkg/providers/imagefamily/types.go | 51 ++- pkg/providers/imagefamily/ubuntu_2204.go | 24 +- pkg/providers/instance/azure_client.go | 14 +- pkg/providers/instance/instance.go | 29 +- pkg/providers/instancetype/instancetypes.go | 4 +- pkg/providers/instancetype/suite_test.go | 85 +++- pkg/test/environment.go | 9 +- pkg/test/options.go | 6 + 21 files changed, 967 insertions(+), 161 deletions(-) create mode 100644 pkg/fake/nodeimageversionsapi.go create mode 100644 pkg/fake/nodeimageversionsapi_test.go delete mode 100644 pkg/providers/imagefamily/image_test.go create mode 100644 pkg/providers/imagefamily/nodeimageversionsclient.go create mode 100644 pkg/providers/imagefamily/nodeimageversionsclient_test.go diff --git a/Makefile-az.mk b/Makefile-az.mk index d5ecf67e3..9533caad5 100755 --- a/Makefile-az.mk +++ b/Makefile-az.mk @@ -8,6 +8,7 @@ else AZURE_ACR_NAME ?= $(COMMON_NAME) endif +AZURE_SIG_SUBSCRIPTION_ID ?= $(AZURE_SUBSCRIPTION_ID) AZURE_CLUSTER_NAME ?= $(COMMON_NAME) AZURE_RESOURCE_GROUP_MC = MC_$(AZURE_RESOURCE_GROUP)_$(AZURE_CLUSTER_NAME)_$(AZURE_LOCATION) @@ -46,7 +47,8 @@ az-mkacr: az-mkrg ## Create test ACR az-acrimport: ## Imports an image to an acr registry az acr import --name $(AZURE_ACR_NAME) --source "mcr.microsoft.com/oss/kubernetes/pause:3.6" --image "pause:3.6" -az-cleanenv: az-rmnodeclaims-fin ## Deletes a few common karpenter testing resources(pods, nodepools, nodeclaims, aksnodeclasses) +az-cleanenv: az-rmnodeclaims-fin ## Deletes a few common karpenter testing resources(pods, nodepools, nodeclaims, aksnodeclasses) + kubectl delete deployments -n default --all kubectl delete pods -n default --all kubectl delete nodeclaims --all kubectl delete nodepools --all @@ -136,6 +138,11 @@ az-perm: ## Create role assignments to let Karpenter manage VMs and Network az role assignment create --assignee $(KARPENTER_USER_ASSIGNED_CLIENT_ID) --scope /subscriptions/$(AZURE_SUBSCRIPTION_ID)/resourceGroups/$(AZURE_RESOURCE_GROUP) --role "Network Contributor" # in some case we create vnet here @echo Consider "make az-configure-values"! +az-perm-sig: ## Create role assignments when testing with SIG images + $(eval KARPENTER_USER_ASSIGNED_CLIENT_ID=$(shell az identity show --resource-group "${AZURE_RESOURCE_GROUP}" --name "${AZURE_KARPENTER_USER_ASSIGNED_IDENTITY_NAME}" --query 'principalId' -otsv)) + az role assignment create --assignee $(KARPENTER_USER_ASSIGNED_CLIENT_ID) --role "Reader" --scope /subscriptions/$(AZURE_SIG_SUBSCRIPTION_ID)/resourceGroups/AKS-Ubuntu/providers/Microsoft.Compute/galleries/AKSUbuntu + az role assignment create --assignee $(KARPENTER_USER_ASSIGNED_CLIENT_ID) --role "Reader" --scope /subscriptions/$(AZURE_SIG_SUBSCRIPTION_ID)/resourceGroups/AKS-AzureLinux/providers/Microsoft.Compute/galleries/AKSAzureLinux + az-perm-subnet-custom: az-perm ## Create role assignments to let Karpenter manage VMs and Network (custom VNet) $(eval VNET_SUBNET_ID=$(shell az aks show --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) | jq -r ".agentPoolProfiles[0].vnetSubnetId")) $(eval KARPENTER_USER_ASSIGNED_CLIENT_ID=$(shell az identity show --resource-group "${AZURE_RESOURCE_GROUP}" --name "${AZURE_KARPENTER_USER_ASSIGNED_IDENTITY_NAME}" --query 'principalId' -otsv)) diff --git a/karpenter-values-template.yaml b/karpenter-values-template.yaml index 8b8fa5e12..dcbbeddfc 100644 --- a/karpenter-values-template.yaml +++ b/karpenter-values-template.yaml @@ -37,6 +37,12 @@ controller: value: "" - name: AZURE_NODE_RESOURCE_GROUP value: ${AZURE_RESOURCE_GROUP_MC} + + # managed karpenter settings + - name: USE_SIG + value: "false" + - name: SIG_SUBSCRIPTION_ID + value: "" serviceAccount: name: ${KARPENTER_SERVICE_ACCOUNT_NAME} annotations: diff --git a/pkg/cloudprovider/drift.go b/pkg/cloudprovider/drift.go index 98c1114bd..b453403a3 100644 --- a/pkg/cloudprovider/drift.go +++ b/pkg/cloudprovider/drift.go @@ -145,21 +145,17 @@ func (c *CloudProvider) isImageVersionDrifted( if vm.Properties == nil || vm.Properties.StorageProfile == nil || - vm.Properties.StorageProfile.ImageReference == nil || - vm.Properties.StorageProfile.ImageReference.CommunityGalleryImageID == nil || - *vm.Properties.StorageProfile.ImageReference.CommunityGalleryImageID == "" { - logger.Debug("not using a CommunityGalleryImageID for nodeClaim %s", nodeClaim.Name) + vm.Properties.StorageProfile.ImageReference == nil { return "", nil } + CIGID := lo.FromPtr(vm.Properties.StorageProfile.ImageReference.CommunityGalleryImageID) + SIGID := lo.FromPtr(vm.Properties.StorageProfile.ImageReference.ID) + vmImageID := lo.Ternary(SIGID != "", SIGID, CIGID) - vmImageID := *vm.Properties.StorageProfile.ImageReference.CommunityGalleryImageID + var imageStub imagefamily.DefaultImageOutput + imageStub.PopulateImageTraitsFromID(vmImageID) - publicGalleryURL, communityImageName, _, err := imagefamily.ParseCommunityImageIDInfo(vmImageID) - if err != nil { - return "", err - } - - expectedImageID, err := c.imageProvider.GetImageID(ctx, communityImageName, publicGalleryURL) + expectedImageID, err := c.imageProvider.GetLatestImageID(ctx, imageStub) if err != nil { return "", err } diff --git a/pkg/cloudprovider/suite_test.go b/pkg/cloudprovider/suite_test.go index b4cde4dee..0a9c86c0f 100644 --- a/pkg/cloudprovider/suite_test.go +++ b/pkg/cloudprovider/suite_test.go @@ -27,6 +27,7 @@ import ( "github.com/awslabs/operatorpkg/object" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/samber/lo" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -223,5 +224,14 @@ var _ = Describe("CloudProvider", func() { Expect(err).To(HaveOccurred()) Expect(drifted).To(BeEmpty()) }) + It("should trigger drift when the image gallery changes to SIG", func() { + options := test.Options(test.OptionsFields{ + UseSIG: lo.ToPtr(true), + }) + ctx = options.ToContext(ctx) + drifted, err := cloudProvider.IsDrifted(ctx, nodeClaim) + Expect(err).ToNot(HaveOccurred()) + Expect(string(drifted)).To(Equal("ImageVersionDrift")) + }) }) }) diff --git a/pkg/fake/nodeimageversionsapi.go b/pkg/fake/nodeimageversionsapi.go new file mode 100644 index 000000000..5d449361b --- /dev/null +++ b/pkg/fake/nodeimageversionsapi.go @@ -0,0 +1,381 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fake + +import ( + "context" + + "github.com/Azure/karpenter-provider-azure/pkg/providers/imagefamily" +) + +type NodeImageVersionsAPI struct { +} + +var _ imagefamily.NodeImageVersionsAPI = &NodeImageVersionsAPI{} + +var ( + NodeImageVersions = []imagefamily.NodeImageVersion{ + { + FullName: "AKSUbuntu-1804gpucontainerd-202410.09.0", + OS: "AKSUbuntu", + SKU: "1804gpucontainerd", + Version: "202410.09.0", + }, + { + FullName: "AKSWindows-2019-17763.2019.221114", + OS: "AKSWindows", + SKU: "windows-2019", + Version: "17763.2019.221114", + }, + { + FullName: "AKSAzureLinux-V3-202409.23.0", + OS: "AKSAzureLinux", + SKU: "V3", + Version: "202409.23.0", + }, + { + FullName: "AKSUbuntu-2204gen2containerd-202410.09.0", + OS: "AKSUbuntu", + SKU: "2204gen2containerd", + Version: "202410.09.0", + }, + { + FullName: "AKSUbuntu-1804gpu-2022.08.29", + OS: "AKSUbuntu", + SKU: "1804gpu", + Version: "2022.08.29", + }, + { + FullName: "AKSWindows-2022-containerd-gen2-20348.2762.241009", + OS: "AKSWindows", + SKU: "windows-2022-containerd-gen2", + Version: "20348.2762.241009", + }, + { + FullName: "AKSCBLMariner-V2katagen2TL-2022.12.15", + OS: "AKSCBLMariner", + SKU: "V2katagen2TL", + Version: "2022.12.15", + }, + { + FullName: "AKSUbuntu-2004gen2fipscontainerd-202410.09.0", + OS: "AKSUbuntu", + SKU: "2004gen2fipscontainerd", + Version: "202410.09.0", + }, + { + FullName: "AKSAzureLinux-V2fips-202410.09.0", + OS: "AKSAzureLinux", + SKU: "V2fips", + Version: "202410.09.0", + }, + { + FullName: "AKSAzureLinux-V2gen2fips-202410.09.0", + OS: "AKSAzureLinux", + SKU: "V2gen2fips", + Version: "202410.09.0", + }, + { + FullName: "AKSUbuntuEdgeZone-1804gen2containerd-202410.09.0", + OS: "AKSUbuntuEdgeZone", + SKU: "1804gen2containerd", + Version: "202410.09.0", + }, + { + FullName: "AKSAzureLinux-V2katagen2-202410.09.0", + OS: "AKSAzureLinux", + SKU: "V2katagen2", + Version: "202410.09.0", + }, + { + FullName: "AKSAzureLinux-V3gen2-202409.23.0", + OS: "AKSAzureLinux", + SKU: "V3gen2", + Version: "202409.23.0", + }, + { + FullName: "AKSUbuntu-2204gen2arm64containerd-202410.09.0", + OS: "AKSUbuntu", + SKU: "2204gen2arm64containerd", + Version: "202410.09.0", + }, + { + FullName: "AKSUbuntu-1804gen2gpucontainerd-202410.09.0", + OS: "AKSUbuntu", + SKU: "1804gen2gpucontainerd", + Version: "202410.09.0", + }, + { + FullName: "AKSAzureLinux-V2gen2TL-202410.09.0", + OS: "AKSAzureLinux", + SKU: "V2gen2TL", + Version: "202410.09.0", + }, + { + FullName: "AKSCBLMariner-V2-202410.09.0", + OS: "AKSCBLMariner", + SKU: "V2", + Version: "202410.09.0", + }, + { + FullName: "AKSCBLMariner-V2fips-202410.09.0", + OS: "AKSCBLMariner", + SKU: "V2fips", + Version: "202410.09.0", + }, + { + FullName: "AKSUbuntu-1804containerd-202410.09.0", + OS: "AKSUbuntu", + SKU: "1804containerd", + Version: "202410.09.0", + }, + { + FullName: "AKSUbuntuEdgeZone-1804containerd-202410.09.0", + OS: "AKSUbuntuEdgeZone", + SKU: "1804containerd", + Version: "202410.09.0", + }, + { + FullName: "AKSUbuntuEdgeZone-2204gen2containerd-202410.09.0", + OS: "AKSUbuntuEdgeZone", + SKU: "2204gen2containerd", + Version: "202410.09.0", + }, + { + FullName: "AKSUbuntu-1804fipscontainerd-202410.09.0", + OS: "AKSUbuntu", + SKU: "1804fipscontainerd", + Version: "202410.09.0", + }, + { + FullName: "AKSAzureLinux-V2gen2arm64-202410.09.0", + OS: "AKSAzureLinux", + SKU: "V2gen2arm64", + Version: "202410.09.0", + }, + { + FullName: "AKSAzureLinux-V2gen2-202410.09.0", + OS: "AKSAzureLinux", + SKU: "V2gen2", + Version: "202410.09.0", + }, + { + FullName: "AKSUbuntu-2204gen2fipscontainerd-202404.09.0", + OS: "AKSUbuntu", + SKU: "2204gen2fipscontainerd", + Version: "202404.09.0", + }, + { + FullName: "AKSWindows-2019-containerd-17763.6414.241010", + OS: "AKSWindows", + SKU: "windows-2019-containerd", + Version: "17763.6414.241010", + }, + { + FullName: "AKSUbuntu-2204gen2TLcontainerd-202410.09.0", + OS: "AKSUbuntu", + SKU: "2204gen2TLcontainerd", + Version: "202410.09.0", + }, + { + FullName: "AKSUbuntu-1804-2022.08.29", + OS: "AKSUbuntu", + SKU: "1804", + Version: "2022.08.29", + }, + { + FullName: "AKSUbuntuEdgeZone-2204containerd-202410.09.0", + OS: "AKSUbuntuEdgeZone", + SKU: "2204containerd", + Version: "202410.09.0", + }, + { + FullName: "AKSCBLMariner-V2gen2-202410.09.0", + OS: "AKSCBLMariner", + SKU: "V2gen2", + Version: "202410.09.0", + }, + { + FullName: "AKSCBLMariner-V2gen2fips-202410.09.0", + OS: "AKSCBLMariner", + SKU: "V2gen2fips", + Version: "202410.09.0", + }, + { + FullName: "AKSCBLMariner-V2gen2arm64-202410.09.0", + OS: "AKSCBLMariner", + SKU: "V2gen2arm64", + Version: "202410.09.0", + }, + { + FullName: "AKSCBLMariner-V2gen2TL-202410.09.0", + OS: "AKSCBLMariner", + SKU: "V2gen2TL", + Version: "202410.09.0", + }, + { + FullName: "AKSUbuntu-2404gen2arm64containerd-202405.20.0", + OS: "AKSUbuntu", + SKU: "2404gen2arm64containerd", + Version: "202405.20.0", + }, + { + FullName: "AKSAzureLinux-V3gen2arm64-202409.23.0", + OS: "AKSAzureLinux", + SKU: "V3gen2arm64", + Version: "202409.23.0", + }, + { + FullName: "AKSAzureLinux-V3fips-202409.23.0", + OS: "AKSAzureLinux", + SKU: "V3fips", + Version: "202409.23.0", + }, + { + FullName: "AKSUbuntu-2004gen2CVMcontainerd-202410.09.0", + OS: "AKSUbuntu", + SKU: "2004gen2CVMcontainerd", + Version: "202410.09.0", + }, + { + FullName: "AKSCBLMariner-V1-202308.28.0", + OS: "AKSCBLMariner", + SKU: "V1", + Version: "202308.28.0", + }, + { + FullName: "AKSUbuntu-1804gen2containerd-202410.09.0", + OS: "AKSUbuntu", + SKU: "1804gen2containerd", + Version: "202410.09.0", + }, + { + FullName: "AKSUbuntu-1804gen2gpu-2022.08.29", + OS: "AKSUbuntu", + SKU: "1804gen2gpu", + Version: "2022.08.29", + }, + { + FullName: "AKSUbuntu-2204gen2minimalcontainerd-202401.12.0", + OS: "AKSUbuntu", + SKU: "2204gen2minimalcontainerd", + Version: "202401.12.0", + }, + { + FullName: "AKSWindows-23H2-gen2-25398.1189.241009", + OS: "AKSWindows", + SKU: "windows-23H2-gen2", + Version: "25398.1189.241009", + }, + { + FullName: "AKSCBLMariner-V2katagen2-202410.09.0", + OS: "AKSCBLMariner", + SKU: "V2katagen2", + Version: "202410.09.0", + }, + { + FullName: "AKSUbuntu-1604-2021.11.06", + OS: "AKSUbuntu", + SKU: "1604", + Version: "2021.11.06", + }, + { + FullName: "AKSUbuntu-2204fipscontainerd-202404.09.0", + OS: "AKSUbuntu", + SKU: "2204fipscontainerd", + Version: "202404.09.0", + }, + { + FullName: "AKSUbuntu-2204minimalcontainerd-202401.12.0", + OS: "AKSUbuntu", + SKU: "2204minimalcontainerd", + Version: "202401.12.0", + }, + { + FullName: "AKSWindows-2022-containerd-20348.2762.241009", + OS: "AKSWindows", + SKU: "windows-2022-containerd", + Version: "20348.2762.241009", + }, + { + FullName: "AKSUbuntu-2204containerd-202410.09.0", + OS: "AKSUbuntu", + SKU: "2204containerd", + Version: "202410.09.0", + }, + { + FullName: "AKSUbuntu-2004fipscontainerd-202410.09.0", + OS: "AKSUbuntu", + SKU: "2004fipscontainerd", + Version: "202410.09.0", + }, + { + FullName: "AKSUbuntu-1804gen2fipscontainerd-202410.09.0", + OS: "AKSUbuntu", + SKU: "1804gen2fipscontainerd", + Version: "202410.09.0", + }, + { + FullName: "AKSWindows-23H2-25398.1189.241009", + OS: "AKSWindows", + SKU: "windows-23H2", + Version: "25398.1189.241009", + }, + { + FullName: "AKSAzureLinux-V2-202410.09.0", + OS: "AKSAzureLinux", + SKU: "V2", + Version: "202410.09.0", + }, + { + FullName: "AKSAzureLinux-V3gen2fips-202409.23.0", + OS: "AKSAzureLinux", + SKU: "V3gen2fips", + Version: "202409.23.0", + }, + { + FullName: "AKSUbuntu-2404gen2containerd-202405.20.0", + OS: "AKSUbuntu", + SKU: "2404gen2containerd", + Version: "202405.20.0", + }, + { + FullName: "AKSUbuntu-2204gen2containerd-2022.10.03", + OS: "AKSUbuntu", + SKU: "2204gen2containerd", + Version: "2022.10.03", + }, + { + FullName: "AKSUbuntu-2404containerd-202405.20.0", + OS: "AKSUbuntu", + SKU: "2404containerd", + Version: "202405.20.0", + }, + { + FullName: "AKSUbuntu-1804gen2-2022.08.29", + OS: "AKSUbuntu", + SKU: "1804gen2", + Version: "2022.08.29", + }, + } +) + +func (n NodeImageVersionsAPI) List(_ context.Context, _, _ string) (imagefamily.NodeImageVersionsResponse, error) { + return imagefamily.NodeImageVersionsResponse{ + Values: imagefamily.FilteredNodeImages(NodeImageVersions), + }, nil +} diff --git a/pkg/fake/nodeimageversionsapi_test.go b/pkg/fake/nodeimageversionsapi_test.go new file mode 100644 index 000000000..cd5d69e3c --- /dev/null +++ b/pkg/fake/nodeimageversionsapi_test.go @@ -0,0 +1,101 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fake + +import ( + "context" + "testing" + + "github.com/Azure/karpenter-provider-azure/pkg/providers/imagefamily" + "github.com/stretchr/testify/assert" +) + +func TestFilteredNodeImagesGalleryFilter(t *testing.T) { + nodeImageVersionAPI := NodeImageVersionsAPI{} + nodeImageVersions, _ := nodeImageVersionAPI.List(context.TODO(), "", "") + filteredNodeImages := imagefamily.FilteredNodeImages(nodeImageVersions.Values) + for _, val := range filteredNodeImages { + assert.NotEqual(t, val.OS, "AKSWindows") + assert.NotEqual(t, val.OS, "AKSUbuntuEdgeZone") + } +} + +// The reasoning behind the test is the following set of output +// az rest --method get --url "/subscriptions//providers/Microsoft.ContainerService/locations/westus2/nodeImageVersions?api-version=2024-04-02-preview" | jq '.values[] | select(.sku == "2204gen2containerd")' +// +// { +// "fullName": "AKSUbuntuEdgeZone-2204gen2containerd-202411.12.0", +// "os": "AKSUbuntuEdgeZone", +// "sku": "2204gen2containerd", +// "version": "202411.12.0" +// } +// { +// "fullName": "AKSUbuntu-2204gen2containerd-2022.10.03", +// "os": "AKSUbuntu", +// "sku": "2204gen2containerd", +// "version": "2022.10.03" +// } +// { +// "fullName": "AKSUbuntu-2204gen2containerd-202411.12.0", +// "os": "AKSUbuntu", +// "sku": "2204gen2containerd", +// "version": "202411.12.0" +// } +// +// In some cases, due to a different distro implementation of the same os + sku pairing, we can get +// duplicate entries for os + sku matchings. +// This test validates we simply ignore the legacy distros and take in the latest Version. +func TestFilteredNodeImagesMinimalUbuntuEdgeCase(t *testing.T) { + filteredNodeImages := imagefamily.FilteredNodeImages(NodeImageVersions) + + expectedVersion := "202410.09.0" + found := false + + for _, val := range filteredNodeImages { + if val.SKU == "2204gen2containerd" && val.Version == expectedVersion { + found = true + break + } + } + + if !found { + t.Errorf("Expected to find node image with version %s but it was not present in the filtered results", expectedVersion) + } +} + +// This function tests that the node image versions that come from the fake are already filtered with the right values +// similar to the behavior we would see if someone is using the node image versions api call. +// the fake imports the same clientside filtering so we need to assert that behavior is the same +func TestFilteredNodeImageVersionsFromProviderList(t *testing.T) { + nodeImageVersionsAPI := NodeImageVersionsAPI{} + filteredNodeImages, err := nodeImageVersionsAPI.List(context.TODO(), "", "") + assert.Nil(t, err) + + expectedVersion := "202410.09.0" + found := false + + for _, val := range filteredNodeImages.Values { + if val.SKU == "2204gen2containerd" && val.Version == expectedVersion { + found = true + break + } + } + + if !found { + t.Errorf("Expected to find node image with version %s but it was not present in the filtered results", expectedVersion) + } +} diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 0cf48ff27..c3adfa6b6 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -98,12 +98,15 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont azConfig.Location, operator.Elected(), ) + imageProvider := imagefamily.NewProvider( operator.KubernetesInterface, cache.New(azurecache.KubernetesVersionTTL, azurecache.DefaultCleanupInterval), azClient.ImageVersionsClient, azConfig.Location, + azConfig.SubscriptionID, + azClient.NodeImageVersionsClient, ) imageResolver := imagefamily.New( operator.GetClient(), diff --git a/pkg/operator/options/options.go b/pkg/operator/options/options.go index 3e71012bd..60cc81cfd 100644 --- a/pkg/operator/options/options.go +++ b/pkg/operator/options/options.go @@ -75,9 +75,12 @@ type Options struct { SubnetID string // => VnetSubnetID to use (for nodes in Azure CNI Overlay and Azure CNI + pod subnet; for for nodes and pods in Azure CNI), unless overridden via AKSNodeClass setFlags map[string]bool - NodeResourceGroup string ProvisionMode string NodeBootstrappingServerURL string + UseSIG bool // => UseSIG is true if Karpenter is managed by AKS, false if it is a self-hosted karpenter installation + + SIGSubscriptionID string + NodeResourceGroup string } func (o *Options) AddFlags(fs *coreoptions.FlagSet) { @@ -92,9 +95,11 @@ func (o *Options) AddFlags(fs *coreoptions.FlagSet) { fs.StringVar(&o.NetworkDataplane, "network-dataplane", env.WithDefaultString("NETWORK_DATAPLANE", "cilium"), "The network dataplane used by the cluster.") fs.StringVar(&o.SubnetID, "vnet-subnet-id", env.WithDefaultString("VNET_SUBNET_ID", ""), "The default subnet ID to use for new nodes. This must be a valid ARM resource ID for subnet that does not overlap with the service CIDR or the pod CIDR.") fs.Var(newNodeIdentitiesValue(env.WithDefaultString("NODE_IDENTITIES", ""), &o.NodeIdentities), "node-identities", "User assigned identities for nodes.") - fs.StringVar(&o.NodeResourceGroup, "node-resource-group", env.WithDefaultString("AZURE_NODE_RESOURCE_GROUP", ""), "[REQUIRED] the resource group created and managed by AKS where the nodes live.") fs.StringVar(&o.ProvisionMode, "provision-mode", env.WithDefaultString("PROVISION_MODE", consts.ProvisionModeAKSScriptless), "[UNSUPPORTED] The provision mode for the cluster.") fs.StringVar(&o.NodeBootstrappingServerURL, "nodebootstrapping-server-url", env.WithDefaultString("NODEBOOTSTRAPPING_SERVER_URL", ""), "[UNSUPPORTED] The url for the node bootstrapping provider server.") + fs.StringVar(&o.NodeResourceGroup, "node-resource-group", env.WithDefaultString("AZURE_NODE_RESOURCE_GROUP", ""), "[REQUIRED] the resource group created and managed by AKS where the nodes live") + fs.BoolVar(&o.UseSIG, "use-sig", env.WithDefaultBool("USE_SIG", false), "If set to true karpenter will use the AKS managed shared image galleries and the node image versions api. If set to false karpenter will use community image galleries. Only a subset of image features will be available in the community image galleries and this flag is only for the managed node provisioning addon.") + fs.StringVar(&o.SIGSubscriptionID, "sig-subscription-id", env.WithDefaultString("SIG_SUBSCRIPTION_ID", ""), "The subscription ID of the shared image gallery.") } func (o Options) GetAPIServerName() string { diff --git a/pkg/providers/imagefamily/azlinux.go b/pkg/providers/imagefamily/azlinux.go index 6f4979f8d..67931aa50 100644 --- a/pkg/providers/imagefamily/azlinux.go +++ b/pkg/providers/imagefamily/azlinux.go @@ -30,9 +30,9 @@ import ( ) const ( - AzureLinuxGen2CommunityImage = "V2gen2" - AzureLinuxGen1CommunityImage = "V2" - AzureLinuxGen2ArmCommunityImage = "V2gen2arm64" + AzureLinuxGen2ImageDefinition = "V2gen2" + AzureLinuxGen1ImageDefinition = "V2" + AzureLinuxGen2ArmImageDefinition = "V2gen2arm64" ) type AzureLinux struct { @@ -47,8 +47,10 @@ func (u AzureLinux) DefaultImages() []DefaultImageOutput { // image provider will select these images in order, first match wins. This is why we chose to put Gen2 first in the defaultImages, as we prefer gen2 over gen1 return []DefaultImageOutput{ { - CommunityImage: AzureLinuxGen2CommunityImage, - PublicGalleryURL: AKSAzureLinuxPublicGalleryURL, + PublicGalleryURL: AKSAzureLinuxPublicGalleryURL, + GalleryResourceGroup: AKSAzureLinuxResourceGroup, + GalleryName: AKSAzureLinuxGalleryName, + ImageDefinition: AzureLinuxGen2ImageDefinition, Requirements: scheduling.NewRequirements( scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, karpv1.ArchitectureAmd64), scheduling.NewRequirement(v1alpha2.LabelSKUHyperVGeneration, v1.NodeSelectorOpIn, v1alpha2.HyperVGenerationV2), @@ -56,8 +58,10 @@ func (u AzureLinux) DefaultImages() []DefaultImageOutput { Distro: "aks-azurelinux-v2-gen2", }, { - CommunityImage: AzureLinuxGen1CommunityImage, - PublicGalleryURL: AKSAzureLinuxPublicGalleryURL, + PublicGalleryURL: AKSAzureLinuxPublicGalleryURL, + GalleryResourceGroup: AKSAzureLinuxResourceGroup, + GalleryName: AKSAzureLinuxGalleryName, + ImageDefinition: AzureLinuxGen1ImageDefinition, Requirements: scheduling.NewRequirements( scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, karpv1.ArchitectureAmd64), scheduling.NewRequirement(v1alpha2.LabelSKUHyperVGeneration, v1.NodeSelectorOpIn, v1alpha2.HyperVGenerationV1), @@ -65,8 +69,10 @@ func (u AzureLinux) DefaultImages() []DefaultImageOutput { Distro: "aks-azurelinux-v2", }, { - CommunityImage: AzureLinuxGen2ArmCommunityImage, - PublicGalleryURL: AKSAzureLinuxPublicGalleryURL, + PublicGalleryURL: AKSAzureLinuxPublicGalleryURL, + GalleryResourceGroup: AKSAzureLinuxResourceGroup, + GalleryName: AKSAzureLinuxGalleryName, + ImageDefinition: AzureLinuxGen2ArmImageDefinition, Requirements: scheduling.NewRequirements( scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, karpv1.ArchitectureArm64), scheduling.NewRequirement(v1alpha2.LabelSKUHyperVGeneration, v1.NodeSelectorOpIn, v1alpha2.HyperVGenerationV2), diff --git a/pkg/providers/imagefamily/image.go b/pkg/providers/imagefamily/image.go index b073db7e8..b80d74abb 100644 --- a/pkg/providers/imagefamily/image.go +++ b/pkg/providers/imagefamily/image.go @@ -19,12 +19,12 @@ package imagefamily import ( "context" "fmt" - "regexp" "strings" "time" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5" "github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2" + "github.com/Azure/karpenter-provider-azure/pkg/operator/options" "github.com/patrickmn/go-cache" "github.com/samber/lo" "k8s.io/client-go/kubernetes" @@ -40,6 +40,8 @@ type Provider struct { kubernetesInterface kubernetes.Interface imageCache *cache.Cache imageVersionsClient CommunityGalleryImageVersionsAPI + subscription string + NodeImageVersions NodeImageVersionsAPI } const ( @@ -48,10 +50,13 @@ const ( imageExpirationInterval = time.Hour * 24 * 3 imageCacheCleaningInterval = time.Hour * 1 - imageIDFormat = "/CommunityGalleries/%s/images/%s/versions/%s" + sharedImageKey = "%s/%s" // imageGallery + imageDefinition + sharedImageGalleryImageIDFormat = "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/galleries/%s/images/%s/versions/%s" + communityImageKey = "%s/%s" // PublicGalleryURL + communityImageName + communityImageIDFormat = "/CommunityGalleries/%s/images/%s/versions/%s" ) -func NewProvider(kubernetesInterface kubernetes.Interface, kubernetesVersionCache *cache.Cache, versionsClient CommunityGalleryImageVersionsAPI, location string) *Provider { +func NewProvider(kubernetesInterface kubernetes.Interface, kubernetesVersionCache *cache.Cache, versionsClient CommunityGalleryImageVersionsAPI, location, subscription string, nodeImageVersionsClient NodeImageVersionsAPI) *Provider { return &Provider{ kubernetesVersionCache: kubernetesVersionCache, imageCache: cache.New(imageExpirationInterval, imageCacheCleaningInterval), @@ -59,6 +64,8 @@ func NewProvider(kubernetesInterface kubernetes.Interface, kubernetesVersionCach imageVersionsClient: versionsClient, cm: pretty.NewChangeMonitor(), kubernetesInterface: kubernetesInterface, + subscription: subscription, + NodeImageVersions: nodeImageVersionsClient, } } @@ -67,18 +74,35 @@ func (p *Provider) Get(ctx context.Context, nodeClass *v1alpha2.AKSNodeClass, in defaultImages := imageFamily.DefaultImages() for _, defaultImage := range defaultImages { if err := instanceType.Requirements.Compatible(defaultImage.Requirements, v1alpha2.AllowUndefinedWellKnownAndRestrictedLabels); err == nil { - communityImageName, publicGalleryURL := defaultImage.CommunityImage, defaultImage.PublicGalleryURL - imageID, err := p.GetImageID(ctx, communityImageName, publicGalleryURL) - if err != nil { - return "", "", err - } - return defaultImage.Distro, imageID, nil + imageID, imageRetrievalErr := p.GetLatestImageID(ctx, defaultImage) + return defaultImage.Distro, imageID, imageRetrievalErr } } return "", "", fmt.Errorf("no compatible images found for instance type %s", instanceType.Name) } +func (p *Provider) GetLatestImageID(ctx context.Context, defaultImage DefaultImageOutput) (string, error) { + // Note: one could argue that we could narrow the key one level further to ImageDefinition since no two AKS ImageDefinitions that are supported + // by karpenter have the same name, but for EdgeZone support this is not the case. + key := lo.Ternary(options.FromContext(ctx).UseSIG, + fmt.Sprintf(sharedImageKey, defaultImage.GalleryName, defaultImage.ImageDefinition), + fmt.Sprintf(communityImageKey, defaultImage.PublicGalleryURL, defaultImage.ImageDefinition), + ) + if imageID, ok := p.imageCache.Get(key); ok { + return imageID.(string), nil + } + + // retrieve ARM Resource ID for the image and write it to the cache + imageID, err := p.resolveImageID(ctx, defaultImage, options.FromContext(ctx).UseSIG) + if err != nil { + return "", err + } + p.imageCache.Set(key, imageID, imageExpirationInterval) + logging.FromContext(ctx).With("image-id", imageID).Info("discovered new image id") + return imageID, nil +} + func (p *Provider) KubeServerVersion(ctx context.Context) (string, error) { if version, ok := p.kubernetesVersionCache.Get(kubernetesVersionCacheKey); ok { return version.(string), nil @@ -95,14 +119,36 @@ func (p *Provider) KubeServerVersion(ctx context.Context) (string, error) { return version, nil } -// Input versionName == "" to get the latest version -func (p *Provider) GetImageID(ctx context.Context, communityImageName, publicGalleryURL string) (string, error) { - key := fmt.Sprintf("%s/%s", publicGalleryURL, communityImageName) - imageID, found := p.imageCache.Get(key) - if found { - return imageID.(string), nil +func (p *Provider) resolveImageID(ctx context.Context, defaultImage DefaultImageOutput, useSIG bool) (string, error) { + if useSIG { + return p.getSIGImageID(ctx, defaultImage) + } + return p.getCIGImageID(defaultImage.PublicGalleryURL, defaultImage.ImageDefinition) +} + +func (p *Provider) getSIGImageID(ctx context.Context, imgStub DefaultImageOutput) (string, error) { + versions, err := p.NodeImageVersions.List(ctx, p.location, p.subscription) + if err != nil { + return "", err } + for _, version := range versions.Values { + if imgStub.ImageDefinition == version.SKU { + imageID := fmt.Sprintf(sharedImageGalleryImageIDFormat, options.FromContext(ctx).SIGSubscriptionID, imgStub.GalleryResourceGroup, imgStub.GalleryName, imgStub.ImageDefinition, version.Version) + return imageID, nil + } + } + return "", fmt.Errorf("failed to get the latest version of the image %s", imgStub.ImageDefinition) +} + +func (p *Provider) getCIGImageID(publicGalleryURL, communityImageName string) (string, error) { + imageVersion, err := p.latestNodeImageVersionCommunity(publicGalleryURL, communityImageName) + if err != nil { + return "", err + } + return BuildImageIDCIG(publicGalleryURL, communityImageName, imageVersion), nil +} +func (p *Provider) latestNodeImageVersionCommunity(publicGalleryURL, communityImageName string) (string, error) { pager := p.imageVersionsClient.NewListPager(p.location, publicGalleryURL, communityImageName, nil) topImageVersionCandidate := armcompute.CommunityGalleryImageVersion{} for pager.More() { @@ -116,34 +162,9 @@ func (p *Provider) GetImageID(ctx context.Context, communityImageName, publicGal } } } - versionName := lo.FromPtr(topImageVersionCandidate.Name) - - selectedImageID := BuildImageID(publicGalleryURL, communityImageName, versionName) - if p.cm.HasChanged(key, selectedImageID) { - logging.FromContext(ctx).With("image-id", selectedImageID).Info("discovered new image id") - } - p.imageCache.Set(key, selectedImageID, imageExpirationInterval) - return selectedImageID, nil -} - -func BuildImageID(publicGalleryURL, communityImageName, imageVersion string) string { - return fmt.Sprintf(imageIDFormat, publicGalleryURL, communityImageName, imageVersion) + return lo.FromPtr(topImageVersionCandidate.Name), nil } -// ParseImageIDInfo parses the publicGalleryURL, communityImageName, and imageVersion out of an imageID -func ParseCommunityImageIDInfo(imageID string) (string, string, string, error) { - // TODO (charliedmcb): assess if doing validation on splitting the string and validating the results is better? Mostly is regex too expensive? - regexStr := fmt.Sprintf(imageIDFormat, "(?P.*)", "(?P.*)", "(?P.*)") - if imageID == "" { - return "", "", "", fmt.Errorf("can not parse empty string. Expect it of the form \"%s\"", regexStr) - } - r := regexp.MustCompile(regexStr) - matches := r.FindStringSubmatch(imageID) - if matches == nil { - return "", "", "", fmt.Errorf("no matches while parsing image id %s", imageID) - } - if r.SubexpIndex("publicGalleryURL") == -1 || r.SubexpIndex("communityImageName") == -1 || r.SubexpIndex("imageVersion") == -1 { - return "", "", "", fmt.Errorf("failed to find sub expressions in %s, for imageID: %s", regexStr, imageID) - } - return matches[r.SubexpIndex("publicGalleryURL")], matches[r.SubexpIndex("communityImageName")], matches[r.SubexpIndex("imageVersion")], nil +func BuildImageIDCIG(publicGalleryURL, communityImageName, imageVersion string) string { + return fmt.Sprintf(communityImageIDFormat, publicGalleryURL, communityImageName, imageVersion) } diff --git a/pkg/providers/imagefamily/image_test.go b/pkg/providers/imagefamily/image_test.go deleted file mode 100644 index 0da43af9e..000000000 --- a/pkg/providers/imagefamily/image_test.go +++ /dev/null @@ -1,57 +0,0 @@ -/* -Portions Copyright (c) Microsoft Corporation. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package imagefamily_test - -import ( - "fmt" - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/Azure/karpenter-provider-azure/pkg/providers/imagefamily" -) - -func TestAzure(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Providers/ImageProvider/Azure") -} - -const ( - testImageID = "/CommunityGalleries/previewaks-1a06572d-8508-419c-a0d1-baffcbcb2f3b/images/2204gen2containerd/Versions/1.1685741267.25933" - olderImageVersion = "1.1686127203.20214" - latestImageVersion = "1.1686127203.20217" -) - -var _ = Describe("Image ID Parsing", func() { - DescribeTable("Parse Image ID", - func(imageID string, expectedPublicGalleryURL, expectedCommunityImageName, expectedImageVersion string, expectError bool) { - publicGalleryURL, communityImageName, imageVersion, err := imagefamily.ParseCommunityImageIDInfo(imageID) - if expectError { - Expect(err).To(HaveOccurred()) - return - } - Expect(err).To(BeNil()) - Expect(publicGalleryURL).To(Equal(expectedPublicGalleryURL)) - Expect(communityImageName).To(Equal(expectedCommunityImageName)) - Expect(imageVersion).To(Equal(expectedImageVersion)) - }, - Entry("Valid image id should parse", fmt.Sprintf("/CommunityGalleries/%s/images/%s/versions/%s", imagefamily.AKSUbuntuPublicGalleryURL, imagefamily.Ubuntu2204Gen2CommunityImage, olderImageVersion), imagefamily.AKSUbuntuPublicGalleryURL, imagefamily.Ubuntu2204Gen2CommunityImage, olderImageVersion, nil), - Entry("invalid image id should not parse", "badimageid", "", "", "", true), - Entry("empty image id should not parse", "badimageid", "", "", "", true), - ) -}) diff --git a/pkg/providers/imagefamily/nodeimageversionsclient.go b/pkg/providers/imagefamily/nodeimageversionsclient.go new file mode 100644 index 000000000..de9305a20 --- /dev/null +++ b/pkg/providers/imagefamily/nodeimageversionsclient.go @@ -0,0 +1,132 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package imagefamily + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strconv" + "strings" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" +) + +type NodeImageVersionsClient struct { + cred azcore.TokenCredential +} + +func NewNodeImageVersionsClient(cred azcore.TokenCredential) *NodeImageVersionsClient { + return &NodeImageVersionsClient{ + cred: cred, + } +} + +func (l *NodeImageVersionsClient) List(ctx context.Context, location, subscription string) (NodeImageVersionsResponse, error) { + resourceURL := fmt.Sprintf( + "https://management.azure.com/subscriptions/%s/providers/Microsoft.ContainerService/locations/%s/nodeImageVersions?api-version=%s", + subscription, location, "2024-04-02-preview", + ) + + token, err := l.cred.GetToken(ctx, policy.TokenRequestOptions{ + Scopes: []string{"https://management.azure.com/.default"}, + }) + if err != nil { + return NodeImageVersionsResponse{}, err + } + + req, err := http.NewRequestWithContext(context.Background(), "GET", resourceURL, nil) + if err != nil { + return NodeImageVersionsResponse{}, err + } + + req.Header.Set("Authorization", "Bearer "+token.Token) + req.Header.Set("Content-Type", "application/json") + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return NodeImageVersionsResponse{}, err + } + defer resp.Body.Close() + + var response NodeImageVersionsResponse + decoder := json.NewDecoder(resp.Body) + err = decoder.Decode(&response) + if err != nil { + return NodeImageVersionsResponse{}, err + } + + response.Values = FilteredNodeImages(response.Values) + return response, nil +} + +// FilteredNodeImages filters on two conditions +// 1. The image is the latest version for the given OS and SKU +// 2. the image belongs to a supported gallery(AKS Ubuntu or Azure Linux) +func FilteredNodeImages(nodeImageVersions []NodeImageVersion) []NodeImageVersion { + latestImages := make(map[string]NodeImageVersion) + + for _, image := range nodeImageVersions { + // Skip the galleries that Karpenter does not support + if image.OS != AKSUbuntuGalleryName && image.OS != AKSAzureLinuxGalleryName { + continue + } + + key := image.OS + "-" + image.SKU + + currentLatest, exists := latestImages[key] + if !exists || isNewerVersion(image.Version, currentLatest.Version) { + latestImages[key] = image + } + } + + var filteredImages []NodeImageVersion + for _, image := range latestImages { + filteredImages = append(filteredImages, image) + } + return filteredImages +} + +// isNewerVersion will return if version1 is greater than version2, note the new versioning scheme is yearmm.dd.build, previously it was yy.mm.dd without the build id. +func isNewerVersion(version1, version2 string) bool { + // Split by dots and compare each segment as an integer getting the largest vhd version + v1Segments := strings.Split(version1, ".") + v2Segments := strings.Split(version2, ".") + + for i := 0; i < len(v1Segments) && i < len(v2Segments); i++ { + v1Segment, err1 := strconv.Atoi(v1Segments[i]) + v2Segment, err2 := strconv.Atoi(v2Segments[i]) + + if err1 != nil || err2 != nil { + return false + } + + if v1Segment > v2Segment { + return true + } else if v1Segment < v2Segment { + return false + } + } + + // If all segments are equal up to the length of the shorter version, + // the longer version is considered newer if it has additional segments + // the legacy linux versions use "yy.mm.dd" whereas new linux versions use "yymm.dd.build" + return len(v1Segments) > len(v2Segments) +} diff --git a/pkg/providers/imagefamily/nodeimageversionsclient_test.go b/pkg/providers/imagefamily/nodeimageversionsclient_test.go new file mode 100644 index 000000000..078b08fc7 --- /dev/null +++ b/pkg/providers/imagefamily/nodeimageversionsclient_test.go @@ -0,0 +1,49 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package imagefamily + +import ( + "testing" +) + +func TestIsNewerVersion(t *testing.T) { + testCases := []struct { + version1 string + version2 string + expected bool + }{ + {"202308.28.0", "202411.12.0", false}, + {"202411.12.0", "202308.28.0", true}, + {"202202.08.29", "202405.20.0", false}, + {"202404.09.0", "202411.12.0", false}, + {"202405.20.0", "202404.09.0", true}, + {"2022.10.03", "2022.12.15", false}, + {"202411.12.0", "2022.12.15", true}, + {"2022.12.15", "2022.10.03", true}, + {"202411.12.0", "202411.12.0", false}, + {"2o2411.12.0", "202411.12.0", false}, // invalid version strings should be ignored and return false + } + + for _, tc := range testCases { + t.Run(tc.version1+"_"+tc.version2, func(t *testing.T) { + result := isNewerVersion(tc.version1, tc.version2) + if result != tc.expected { + t.Errorf("isNewerVersion(%q, %q) = %v; want %v", tc.version1, tc.version2, result, tc.expected) + } + }) + } +} diff --git a/pkg/providers/imagefamily/types.go b/pkg/providers/imagefamily/types.go index b3c3510aa..009131069 100644 --- a/pkg/providers/imagefamily/types.go +++ b/pkg/providers/imagefamily/types.go @@ -17,6 +17,9 @@ limitations under the License. package imagefamily import ( + "context" + "strings" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" armcomputev5 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5" "sigs.k8s.io/karpenter/pkg/scheduling" @@ -25,17 +28,57 @@ import ( const ( AKSUbuntuPublicGalleryURL = "AKSUbuntu-38d80f77-467a-481f-a8d4-09b6d4220bd2" AKSAzureLinuxPublicGalleryURL = "AKSAzureLinux-f7c7cda5-1c9a-4bdc-a222-9614c968580b" + + AKSUbuntuResourceGroup = "AKS-Ubuntu" + AKSAzureLinuxResourceGroup = "AKS-AzureLinux" + + AKSUbuntuGalleryName = "AKSUbuntu" + AKSAzureLinuxGalleryName = "AKSAzureLinux" ) -// DefaultImageOutput is the Stub of an Image we return from an ImageFamily De +// DefaultImageOutput is a stub describing our desired image with an image's name and requirements to run that image type DefaultImageOutput struct { - CommunityImage string + // Community Image Gallery PublicGalleryURL string - Requirements scheduling.Requirements - Distro string + // Shared Image Gallery + GalleryResourceGroup string + GalleryName string + // Common + ImageDefinition string + Distro string + Requirements scheduling.Requirements +} + +func (d *DefaultImageOutput) PopulateImageTraitsFromID(imageID string) { + // We want to take a community image gallery image id or a shared image gallery id and populate the contents of DefaultImageOutput + imageIDParts := strings.Split(imageID, "/") + if imageIDParts[1] == "subscriptions" { // Shared Image Gallery + d.GalleryResourceGroup = imageIDParts[4] + d.GalleryName = imageIDParts[8] + d.ImageDefinition = imageIDParts[10] + } + if imageIDParts[1] == "CommunityGalleries" { // Community Image Gallery + d.PublicGalleryURL = imageIDParts[2] + d.ImageDefinition = imageIDParts[4] + } } // CommunityGalleryImageVersionsAPI is used for listing community gallery image versions. type CommunityGalleryImageVersionsAPI interface { NewListPager(location string, publicGalleryName string, galleryImageName string, options *armcomputev5.CommunityGalleryImageVersionsClientListOptions) *runtime.Pager[armcomputev5.CommunityGalleryImageVersionsClientListResponse] } + +type NodeImageVersion struct { + FullName string `json:"fullName"` + OS string `json:"os"` + SKU string `json:"sku"` + Version string `json:"version"` +} + +type NodeImageVersionsResponse struct { + Values []NodeImageVersion `json:"values"` +} + +type NodeImageVersionsAPI interface { + List(ctx context.Context, location, subscription string) (NodeImageVersionsResponse, error) +} diff --git a/pkg/providers/imagefamily/ubuntu_2204.go b/pkg/providers/imagefamily/ubuntu_2204.go index 54806cba2..df034db8a 100644 --- a/pkg/providers/imagefamily/ubuntu_2204.go +++ b/pkg/providers/imagefamily/ubuntu_2204.go @@ -30,9 +30,9 @@ import ( ) const ( - Ubuntu2204Gen2CommunityImage = "2204gen2containerd" - Ubuntu2204Gen1CommunityImage = "2204containerd" - Ubuntu2204Gen2ArmCommunityImage = "2204gen2arm64containerd" + Ubuntu2204Gen2ImageDefinition = "2204gen2containerd" + Ubuntu2204Gen1ImageDefinition = "2204containerd" + Ubuntu2204Gen2ArmImageDefinition = "2204gen2arm64containerd" ) type Ubuntu2204 struct { @@ -47,8 +47,10 @@ func (u Ubuntu2204) DefaultImages() []DefaultImageOutput { // image provider will select these images in order, first match wins. This is why we chose to put Ubuntu2204Gen2containerd first in the defaultImages return []DefaultImageOutput{ { - CommunityImage: Ubuntu2204Gen2CommunityImage, - PublicGalleryURL: AKSUbuntuPublicGalleryURL, + PublicGalleryURL: AKSUbuntuPublicGalleryURL, + GalleryResourceGroup: AKSUbuntuResourceGroup, + GalleryName: AKSUbuntuGalleryName, + ImageDefinition: Ubuntu2204Gen2ImageDefinition, Requirements: scheduling.NewRequirements( scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, karpv1.ArchitectureAmd64), scheduling.NewRequirement(v1alpha2.LabelSKUHyperVGeneration, v1.NodeSelectorOpIn, v1alpha2.HyperVGenerationV2), @@ -56,8 +58,10 @@ func (u Ubuntu2204) DefaultImages() []DefaultImageOutput { Distro: "aks-ubuntu-containerd-22.04-gen2", }, { - CommunityImage: Ubuntu2204Gen1CommunityImage, - PublicGalleryURL: AKSUbuntuPublicGalleryURL, + PublicGalleryURL: AKSUbuntuPublicGalleryURL, + GalleryResourceGroup: AKSUbuntuResourceGroup, + GalleryName: AKSUbuntuGalleryName, + ImageDefinition: Ubuntu2204Gen1ImageDefinition, Requirements: scheduling.NewRequirements( scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, karpv1.ArchitectureAmd64), scheduling.NewRequirement(v1alpha2.LabelSKUHyperVGeneration, v1.NodeSelectorOpIn, v1alpha2.HyperVGenerationV1), @@ -65,8 +69,10 @@ func (u Ubuntu2204) DefaultImages() []DefaultImageOutput { Distro: "aks-ubuntu-containerd-22.04", }, { - CommunityImage: Ubuntu2204Gen2ArmCommunityImage, - PublicGalleryURL: AKSUbuntuPublicGalleryURL, + PublicGalleryURL: AKSUbuntuPublicGalleryURL, + GalleryResourceGroup: AKSUbuntuResourceGroup, + GalleryName: AKSUbuntuGalleryName, + ImageDefinition: Ubuntu2204Gen2ArmImageDefinition, Requirements: scheduling.NewRequirements( scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, karpv1.ArchitectureArm64), scheduling.NewRequirement(v1alpha2.LabelSKUHyperVGeneration, v1.NodeSelectorOpIn, v1alpha2.HyperVGenerationV2), diff --git a/pkg/providers/instance/azure_client.go b/pkg/providers/instance/azure_client.go index 62a41f99b..ff797d4d2 100644 --- a/pkg/providers/instance/azure_client.go +++ b/pkg/providers/instance/azure_client.go @@ -63,7 +63,8 @@ type AZClient struct { virtualMachinesExtensionClient VirtualMachineExtensionsAPI networkInterfacesClient NetworkInterfacesAPI - ImageVersionsClient imagefamily.CommunityGalleryImageVersionsAPI + NodeImageVersionsClient imagefamily.NodeImageVersionsAPI + ImageVersionsClient imagefamily.CommunityGalleryImageVersionsAPI // SKU CLIENT is still using track 1 because skewer does not support the track 2 path. We need to refactor this once skewer supports track 2 SKUClient skuclient.SkuClient LoadBalancersClient loadbalancer.LoadBalancersAPI @@ -76,6 +77,7 @@ func NewAZClientFromAPI( interfacesClient NetworkInterfacesAPI, loadBalancersClient loadbalancer.LoadBalancersAPI, imageVersionsClient imagefamily.CommunityGalleryImageVersionsAPI, + nodeImageVersionsClient imagefamily.NodeImageVersionsAPI, skuClient skuclient.SkuClient, ) *AZClient { return &AZClient{ @@ -84,6 +86,7 @@ func NewAZClientFromAPI( virtualMachinesExtensionClient: virtualMachinesExtensionClient, networkInterfacesClient: interfacesClient, ImageVersionsClient: imageVersionsClient, + NodeImageVersionsClient: nodeImageVersionsClient, SKUClient: skuClient, LoadBalancersClient: loadBalancersClient, } @@ -137,11 +140,13 @@ func NewAZClient(ctx context.Context, cfg *auth.Config, env *azure.Environment) } klog.V(5).Infof("Created azure resource graph client %v, using a token credential", azureResourceGraphClient) - imageVersionsClient, err := armcomputev5.NewCommunityGalleryImageVersionsClient(cfg.SubscriptionID, cred, opts) + communityImageVersionsClient, err := armcomputev5.NewCommunityGalleryImageVersionsClient(cfg.SubscriptionID, cred, opts) if err != nil { return nil, err } - klog.V(5).Infof("Created image versions client %v, using a token credential", imageVersionsClient) + klog.V(5).Infof("Created image versions client %v, using a token credential", communityImageVersionsClient) + + nodeImageVersionsClient := imagefamily.NewNodeImageVersionsClient(cred) loadBalancersClient, err := armnetwork.NewLoadBalancersClient(cfg.SubscriptionID, cred, opts) if err != nil { @@ -158,6 +163,7 @@ func NewAZClient(ctx context.Context, cfg *auth.Config, env *azure.Environment) extensionsClient, interfacesClient, loadBalancersClient, - imageVersionsClient, + communityImageVersionsClient, + nodeImageVersionsClient, skuClient), nil } diff --git a/pkg/providers/instance/instance.go b/pkg/providers/instance/instance.go index 89939746e..0149c875e 100644 --- a/pkg/providers/instance/instance.go +++ b/pkg/providers/instance/instance.go @@ -314,19 +314,14 @@ func newVMObject( vmName, nicReference, zone, - capacityType string, - location string, + capacityType, + location, sshPublicKey string, nodeIdentities []string, nodeClass *v1alpha2.AKSNodeClass, launchTemplate *launchtemplate.Template, instanceType *corecloudprovider.InstanceType, - provisionMode string) armcompute.VirtualMachine { - // Build the image reference from template - imageReference := armcompute.ImageReference{ - CommunityGalleryImageID: &launchTemplate.ImageID, - } - + provisionMode string, useSIG bool) armcompute.VirtualMachine { if launchTemplate.IsWindows { return armcompute.VirtualMachine{} // TODO(Windows) } @@ -346,7 +341,6 @@ func newVMObject( CreateOption: lo.ToPtr(armcompute.DiskCreateOptionTypesFromImage), DeleteOption: lo.ToPtr(armcompute.DiskDeleteOptionTypesDelete), }, - ImageReference: &imageReference, }, NetworkProfile: &armcompute.NetworkProfile{ @@ -384,6 +378,7 @@ func newVMObject( Tags: launchTemplate.Tags, } setVMPropertiesOSDiskType(vm.Properties, launchTemplate.StorageProfile) + setImageReference(vm.Properties, launchTemplate.ImageID, useSIG) setVMPropertiesBillingProfile(vm.Properties, capacityType) if provisionMode == consts.ProvisionModeBootstrappingClient { @@ -406,6 +401,19 @@ func setVMPropertiesOSDiskType(vmProperties *armcompute.VirtualMachineProperties } } +// setImageReference sets the image reference for the VM based on if we are using self hosted karpenter or the node auto provisioning addon +func setImageReference(vmProperties *armcompute.VirtualMachineProperties, imageID string, useSIG bool) { + if useSIG { + vmProperties.StorageProfile.ImageReference = &armcompute.ImageReference{ + ID: lo.ToPtr(imageID), + } + return + } + vmProperties.StorageProfile.ImageReference = &armcompute.ImageReference{ + CommunityGalleryImageID: lo.ToPtr(imageID), + } +} + // setVMPropertiesBillingProfile sets a default MaxPrice of -1 for Spot func setVMPropertiesBillingProfile(vmProperties *armcompute.VirtualMachineProperties, capacityType string) { if capacityType == karpv1.CapacityTypeSpot { @@ -471,7 +479,8 @@ func (p *DefaultProvider) launchInstance( sshPublicKey := options.FromContext(ctx).SSHPublicKey nodeIdentityIDs := options.FromContext(ctx).NodeIdentities - vm := newVMObject(resourceName, nicReference, zone, capacityType, p.location, sshPublicKey, nodeIdentityIDs, nodeClass, launchTemplate, instanceType, p.provisionMode) + useSIG := options.FromContext(ctx).UseSIG + vm := newVMObject(resourceName, nicReference, zone, capacityType, p.location, sshPublicKey, nodeIdentityIDs, nodeClass, launchTemplate, instanceType, p.provisionMode, useSIG) logging.FromContext(ctx).Debugf("Creating virtual machine %s (%s)", resourceName, instanceType.Name) // Uses AZ Client to create a new virtual machine using the vm object we prepared earlier diff --git a/pkg/providers/instancetype/instancetypes.go b/pkg/providers/instancetype/instancetypes.go index f5b400f68..946db59e8 100644 --- a/pkg/providers/instancetype/instancetypes.go +++ b/pkg/providers/instancetype/instancetypes.go @@ -36,6 +36,7 @@ import ( "github.com/Azure/go-autorest/autorest/to" "github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2" kcache "github.com/Azure/karpenter-provider-azure/pkg/cache" + "github.com/Azure/karpenter-provider-azure/pkg/operator/options" "github.com/Azure/karpenter-provider-azure/pkg/utils" "github.com/patrickmn/go-cache" "k8s.io/apimachinery/pkg/util/sets" @@ -274,8 +275,7 @@ func (p *DefaultProvider) getInstanceTypes(ctx context.Context) (map[string]*ske logging.FromContext(ctx).Errorf("parsing VM size %s, %v", *skus[i].Size, err) continue } - - useSIG := false // replace with options.FromContext(ctx).UseSIG when available + useSIG := options.FromContext(ctx).UseSIG if !skus[i].HasLocationRestriction(p.region) && p.isSupported(&skus[i], vmsize, useSIG) { instanceTypes[skus[i].GetName()] = &skus[i] } diff --git a/pkg/providers/instancetype/suite_test.go b/pkg/providers/instancetype/suite_test.go index ce27f1290..ae1f0d515 100644 --- a/pkg/providers/instancetype/suite_test.go +++ b/pkg/providers/instancetype/suite_test.go @@ -128,6 +128,7 @@ var _ = Describe("InstanceType Provider", func() { }, }) + ctx = options.ToContext(ctx, test.Options()) cluster.Reset() clusterNonZonal.Reset() azureEnv.Reset() @@ -885,7 +886,79 @@ var _ = Describe("InstanceType Provider", func() { }) }) + Context("ImageReference", func() { + It("should use shared image gallery images when options are set to UseSIG", func() { + options := test.Options(test.OptionsFields{ + UseSIG: lo.ToPtr(true), + }) + ctx = options.ToContext(ctx) + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + pod := coretest.UnschedulablePod(coretest.PodOptions{}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod) + ExpectScheduled(ctx, env.Client, pod) + + // Expect virtual machine to have a shared image gallery id set on it + Expect(azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Len()).To(Equal(1)) + vm := azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Pop().VM + Expect(vm.Properties.StorageProfile.ImageReference).ToNot(BeNil()) + Expect(vm.Properties.StorageProfile.ImageReference.ID).ShouldNot(BeNil()) + Expect(vm.Properties.StorageProfile.ImageReference.CommunityGalleryImageID).Should(BeNil()) + + Expect(*vm.Properties.StorageProfile.ImageReference.ID).To(ContainSubstring(options.SIGSubscriptionID)) + Expect(*vm.Properties.StorageProfile.ImageReference.ID).To(ContainSubstring("AKSUbuntu")) + }) + It("should use Community Images when options are set to UseSIG=false", func() { + options := test.Options(test.OptionsFields{ + UseSIG: lo.ToPtr(false), + }) + ctx = options.ToContext(ctx) + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + pod := coretest.UnschedulablePod(coretest.PodOptions{}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod) + ExpectScheduled(ctx, env.Client, pod) + + Expect(azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Len()).To(Equal(1)) + vm := azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Pop().VM + Expect(vm.Properties.StorageProfile.ImageReference.CommunityGalleryImageID).Should(Not(BeNil())) + + }) + + }) Context("ImageProvider + Image Family", func() { + DescribeTable("should select the right Shared Image Gallery image for a given instance type", func(instanceType string, imageFamily string, expectedImageDefinition string, expectedGalleryRG string, expectedGalleryURL string) { + options := test.Options(test.OptionsFields{ + UseSIG: lo.ToPtr(true), + }) + ctx = options.ToContext(ctx) + nodeClass.Spec.ImageFamily = lo.ToPtr(imageFamily) + coretest.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelInstanceTypeStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{instanceType}, + }}) + nodePool.Spec.Template.Spec.NodeClassRef = &karpv1.NodeClassReference{Name: nodeClass.Name} + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + pod := coretest.UnschedulablePod(coretest.PodOptions{}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod) + ExpectScheduled(ctx, env.Client, pod) + + Expect(azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Len()).To(Equal(1)) + vm := azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Pop().VM + Expect(vm.Properties.StorageProfile.ImageReference).ToNot(BeNil()) + + expectedPrefix := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/galleries/%s/images/%s", options.SIGSubscriptionID, expectedGalleryRG, expectedGalleryURL, expectedImageDefinition) + Expect(*vm.Properties.StorageProfile.ImageReference.ID).To(ContainSubstring(expectedPrefix)) + + }, + + Entry("Gen2, Gen1 instance type with AKSUbuntu image family", "Standard_D2_v5", v1alpha2.Ubuntu2204ImageFamily, imagefamily.Ubuntu2204Gen2ImageDefinition, imagefamily.AKSUbuntuResourceGroup, imagefamily.AKSUbuntuGalleryName), + Entry("Gen1 instance type with AKSUbuntu image family", "Standard_D2_v3", v1alpha2.Ubuntu2204ImageFamily, imagefamily.Ubuntu2204Gen1ImageDefinition, imagefamily.AKSUbuntuResourceGroup, imagefamily.AKSUbuntuGalleryName), + Entry("ARM instance type with AKSUbuntu image family", "Standard_D16plds_v5", v1alpha2.Ubuntu2204ImageFamily, imagefamily.Ubuntu2204Gen2ArmImageDefinition, imagefamily.AKSUbuntuResourceGroup, imagefamily.AKSUbuntuGalleryName), + Entry("Gen2 instance type with AzureLinux image family", "Standard_D2_v5", v1alpha2.AzureLinuxImageFamily, imagefamily.AzureLinuxGen2ImageDefinition, imagefamily.AKSAzureLinuxResourceGroup, imagefamily.AKSAzureLinuxGalleryName), + Entry("Gen1 instance type with AzureLinux image family", "Standard_D2_v3", v1alpha2.AzureLinuxImageFamily, imagefamily.AzureLinuxGen1ImageDefinition, imagefamily.AKSAzureLinuxResourceGroup, imagefamily.AKSAzureLinuxGalleryName), + Entry("ARM instance type with AzureLinux image family", "Standard_D16plds_v5", v1alpha2.AzureLinuxImageFamily, imagefamily.AzureLinuxGen2ArmImageDefinition, imagefamily.AKSAzureLinuxResourceGroup, imagefamily.AKSAzureLinuxGalleryName), + ) DescribeTable("should select the right image for a given instance type", func(instanceType string, imageFamily string, expectedImageDefinition string, expectedGalleryURL string) { nodeClass.Spec.ImageFamily = lo.ToPtr(imageFamily) @@ -914,17 +987,17 @@ var _ = Describe("InstanceType Provider", func() { azureEnv.Reset() }, Entry("Gen2, Gen1 instance type with AKSUbuntu image family", - "Standard_D2_v5", v1alpha2.Ubuntu2204ImageFamily, imagefamily.Ubuntu2204Gen2CommunityImage, imagefamily.AKSUbuntuPublicGalleryURL), + "Standard_D2_v5", v1alpha2.Ubuntu2204ImageFamily, imagefamily.Ubuntu2204Gen2ImageDefinition, imagefamily.AKSUbuntuPublicGalleryURL), Entry("Gen1 instance type with AKSUbuntu image family", - "Standard_D2_v3", v1alpha2.Ubuntu2204ImageFamily, imagefamily.Ubuntu2204Gen1CommunityImage, imagefamily.AKSUbuntuPublicGalleryURL), + "Standard_D2_v3", v1alpha2.Ubuntu2204ImageFamily, imagefamily.Ubuntu2204Gen1ImageDefinition, imagefamily.AKSUbuntuPublicGalleryURL), Entry("ARM instance type with AKSUbuntu image family", - "Standard_D16plds_v5", v1alpha2.Ubuntu2204ImageFamily, imagefamily.Ubuntu2204Gen2ArmCommunityImage, imagefamily.AKSUbuntuPublicGalleryURL), + "Standard_D16plds_v5", v1alpha2.Ubuntu2204ImageFamily, imagefamily.Ubuntu2204Gen2ArmImageDefinition, imagefamily.AKSUbuntuPublicGalleryURL), Entry("Gen2 instance type with AzureLinux image family", - "Standard_D2_v5", v1alpha2.AzureLinuxImageFamily, imagefamily.AzureLinuxGen2CommunityImage, imagefamily.AKSAzureLinuxPublicGalleryURL), + "Standard_D2_v5", v1alpha2.AzureLinuxImageFamily, imagefamily.AzureLinuxGen2ImageDefinition, imagefamily.AKSAzureLinuxPublicGalleryURL), Entry("Gen1 instance type with AzureLinux image family", - "Standard_D2_v3", v1alpha2.AzureLinuxImageFamily, imagefamily.AzureLinuxGen1CommunityImage, imagefamily.AKSAzureLinuxPublicGalleryURL), + "Standard_D2_v3", v1alpha2.AzureLinuxImageFamily, imagefamily.AzureLinuxGen1ImageDefinition, imagefamily.AKSAzureLinuxPublicGalleryURL), Entry("ARM instance type with AzureLinux image family", - "Standard_D16plds_v5", v1alpha2.AzureLinuxImageFamily, imagefamily.AzureLinuxGen2ArmCommunityImage, imagefamily.AKSAzureLinuxPublicGalleryURL), + "Standard_D16plds_v5", v1alpha2.AzureLinuxImageFamily, imagefamily.AzureLinuxGen2ArmImageDefinition, imagefamily.AKSAzureLinuxPublicGalleryURL), ) }) Context("Instance Types", func() { diff --git a/pkg/test/environment.go b/pkg/test/environment.go index 30b6777bf..11d3faf0e 100644 --- a/pkg/test/environment.go +++ b/pkg/test/environment.go @@ -46,6 +46,7 @@ func init() { var ( resourceGroup = "test-resourceGroup" + subscription = "12345678-1234-1234-1234-123456789012" ) type Environment struct { @@ -98,6 +99,7 @@ func NewRegionalEnvironment(ctx context.Context, env *coretest.Environment, regi skuClientSingleton := &fake.MockSkuClientSingleton{SKUClient: &fake.ResourceSKUsAPI{Location: region}} communityImageVersionsAPI := &fake.CommunityGalleryImageVersionsAPI{} loadBalancersAPI := &fake.LoadBalancersAPI{} + nodeImageVersionsAPI := &fake.NodeImageVersionsAPI{} // Cache kubernetesVersionCache := cache.New(azurecache.KubernetesVersionTTL, azurecache.DefaultCleanupInterval) @@ -107,7 +109,7 @@ func NewRegionalEnvironment(ctx context.Context, env *coretest.Environment, regi // Providers pricingProvider := pricing.NewProvider(ctx, pricingAPI, region, make(chan struct{})) - imageFamilyProvider := imagefamily.NewProvider(env.KubernetesInterface, kubernetesVersionCache, communityImageVersionsAPI, region) + imageFamilyProvider := imagefamily.NewProvider(env.KubernetesInterface, kubernetesVersionCache, communityImageVersionsAPI, region, subscription, nodeImageVersionsAPI) imageFamilyResolver := imagefamily.New(env.Client, imageFamilyProvider) instanceTypesProvider := instancetype.NewDefaultProvider(region, instanceTypeCache, skuClientSingleton, pricingProvider, unavailableOfferingsCache) launchTemplateProvider := launchtemplate.NewProvider( @@ -117,7 +119,7 @@ func NewRegionalEnvironment(ctx context.Context, env *coretest.Environment, regi ptr.String("ca-bundle"), testOptions.ClusterEndpoint, "test-tenant", - "test-subscription", + subscription, "test-cluster-resource-group", "test-kubelet-identity-client-id", testOptions.NodeResourceGroup, @@ -137,6 +139,7 @@ func NewRegionalEnvironment(ctx context.Context, env *coretest.Environment, regi networkInterfacesAPI, loadBalancersAPI, communityImageVersionsAPI, + nodeImageVersionsAPI, skuClientSingleton, ) instanceProvider := instance.NewDefaultProvider( @@ -147,7 +150,7 @@ func NewRegionalEnvironment(ctx context.Context, env *coretest.Environment, regi unavailableOfferingsCache, region, testOptions.NodeResourceGroup, - "", // subscriptionID + subscription, testOptions.ProvisionMode, ) diff --git a/pkg/test/options.go b/pkg/test/options.go index 32e2b2734..4cdbbb3a4 100644 --- a/pkg/test/options.go +++ b/pkg/test/options.go @@ -41,6 +41,10 @@ type OptionsFields struct { NodeResourceGroup *string ProvisionMode *string NodeBootstrappingServerURL *string + + // UseSIG Flags not required by the self hosted offering + UseSIG *bool + SIGSubscriptionID *string } func Options(overrides ...OptionsFields) *azoptions.Options { @@ -65,5 +69,7 @@ func Options(overrides ...OptionsFields) *azoptions.Options { SubnetID: lo.FromPtrOr(options.SubnetID, "/subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/sillygeese/providers/Microsoft.Network/virtualNetworks/karpentervnet/subnets/karpentersub"), NodeResourceGroup: lo.FromPtrOr(options.NodeResourceGroup, "test-resourceGroup"), ProvisionMode: lo.FromPtrOr(options.ProvisionMode, "aksscriptless"), + UseSIG: lo.FromPtrOr(options.UseSIG, false), + SIGSubscriptionID: lo.FromPtrOr(options.SIGSubscriptionID, "10945678-1234-1234-1234-123456789012"), } } From 79120ab588db5d1a039ad84a3c27aab511d3f388 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jan 2025 20:25:26 -0800 Subject: [PATCH 4/6] chore(deps): bump the actions-deps group with 2 updates (#630) Bumps the actions-deps group with 2 updates: [actions/upload-artifact](https://github.com/actions/upload-artifact) and [github/codeql-action](https://github.com/github/codeql-action). Updates `actions/upload-artifact` from 4.4.3 to 4.5.0 - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882...6f51ac03b9356f520e9adb1b1b7802705f340c2b) Updates `github/codeql-action` from 3.27.9 to 3.28.0 - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/df409f7d9260372bd5f19e5b04e83cb3c43714ae...48ab28a6f5dbc2a99bf1e0131198dd8f1df78169) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions-deps - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions-deps ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/approval-comment.yaml | 2 +- .github/workflows/codeql-analysis.yml | 6 +++--- .github/workflows/scorecards.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/approval-comment.yaml b/.github/workflows/approval-comment.yaml index 1de04c76f..20c680785 100644 --- a/.github/workflows/approval-comment.yaml +++ b/.github/workflows/approval-comment.yaml @@ -30,7 +30,7 @@ jobs: mkdir -p /tmp/artifacts { echo ${{ github.event.pull_request.number }}; echo ${{ github.event.review.commit_id }}; } >> /tmp/artifacts/metadata.txt cat /tmp/artifacts/metadata.txt - - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 with: name: artifacts path: /tmp/artifacts diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 18c291681..98dc425a3 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -46,8 +46,8 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - uses: ./.github/actions/install-deps - run: make vulncheck - - uses: github/codeql-action/init@df409f7d9260372bd5f19e5b04e83cb3c43714ae # v3.27.9 + - uses: github/codeql-action/init@48ab28a6f5dbc2a99bf1e0131198dd8f1df78169 # v3.28.0 with: languages: ${{ matrix.language }} - - uses: github/codeql-action/autobuild@df409f7d9260372bd5f19e5b04e83cb3c43714ae # v3.27.9 - - uses: github/codeql-action/analyze@df409f7d9260372bd5f19e5b04e83cb3c43714ae # v3.27.9 + - uses: github/codeql-action/autobuild@48ab28a6f5dbc2a99bf1e0131198dd8f1df78169 # v3.28.0 + - uses: github/codeql-action/analyze@48ab28a6f5dbc2a99bf1e0131198dd8f1df78169 # v3.28.0 diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index 5679d2b6a..9439f8681 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -82,7 +82,7 @@ jobs: # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: "Upload artifact" - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 with: name: SARIF file path: results.sarif @@ -90,6 +90,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@df409f7d9260372bd5f19e5b04e83cb3c43714ae # v3.27.9 + uses: github/codeql-action/upload-sarif@48ab28a6f5dbc2a99bf1e0131198dd8f1df78169 # v3.28.0 with: sarif_file: results.sarif From 9e953524ca6ca63e321345e1e8a96590e4492aac Mon Sep 17 00:00:00 2001 From: Alex Leites <18728999+tallaxes@users.noreply.github.com> Date: Tue, 7 Jan 2025 15:23:31 -0800 Subject: [PATCH 5/6] chore: update controller-gen (#638) --- .../templates/karpenter.azure.com_aksnodeclasses.yaml | 2 +- pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/karpenter-crd/templates/karpenter.azure.com_aksnodeclasses.yaml b/charts/karpenter-crd/templates/karpenter.azure.com_aksnodeclasses.yaml index d7a769ea9..db6f5f00a 100644 --- a/charts/karpenter-crd/templates/karpenter.azure.com_aksnodeclasses.yaml +++ b/charts/karpenter-crd/templates/karpenter.azure.com_aksnodeclasses.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.5 + controller-gen.kubebuilder.io/version: v0.17.0 name: aksnodeclasses.karpenter.azure.com spec: group: karpenter.azure.com diff --git a/pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml b/pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml index d7a769ea9..db6f5f00a 100644 --- a/pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml +++ b/pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.5 + controller-gen.kubebuilder.io/version: v0.17.0 name: aksnodeclasses.karpenter.azure.com spec: group: karpenter.azure.com From 906111fe57cb940009b108d450c0991052505b2d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 7 Jan 2025 15:42:30 -0800 Subject: [PATCH 6/6] chore(deps): bump the go-deps group across 1 directory with 5 updates (#639) Bumps the go-deps group with 3 updates in the / directory: [github.com/Azure/azure-sdk-for-go/sdk/azcore](https://github.com/Azure/azure-sdk-for-go), [github.com/go-playground/validator/v10](https://github.com/go-playground/validator) and [github.com/onsi/ginkgo/v2](https://github.com/onsi/ginkgo). Updates `github.com/Azure/azure-sdk-for-go/sdk/azcore` from 1.16.0 to 1.17.0 - [Release notes](https://github.com/Azure/azure-sdk-for-go/releases) - [Changelog](https://github.com/Azure/azure-sdk-for-go/blob/main/documentation/release.md) - [Commits](https://github.com/Azure/azure-sdk-for-go/compare/sdk/azcore/v1.16.0...sdk/azcore/v1.17.0) Updates `github.com/go-playground/validator/v10` from 10.22.1 to 10.23.0 - [Release notes](https://github.com/go-playground/validator/releases) - [Commits](https://github.com/go-playground/validator/compare/v10.22.1...v10.23.0) Updates `github.com/onsi/ginkgo/v2` from 2.20.2 to 2.22.2 - [Release notes](https://github.com/onsi/ginkgo/releases) - [Changelog](https://github.com/onsi/ginkgo/blob/master/CHANGELOG.md) - [Commits](https://github.com/onsi/ginkgo/compare/v2.20.2...v2.22.2) Updates `github.com/onsi/gomega` from 1.34.2 to 1.36.2 - [Release notes](https://github.com/onsi/gomega/releases) - [Changelog](https://github.com/onsi/gomega/blob/master/CHANGELOG.md) - [Commits](https://github.com/onsi/gomega/compare/v1.34.2...v1.36.2) Updates `github.com/stretchr/testify` from 1.9.0 to 1.10.0 - [Release notes](https://github.com/stretchr/testify/releases) - [Commits](https://github.com/stretchr/testify/compare/v1.9.0...v1.10.0) --- updated-dependencies: - dependency-name: github.com/Azure/azure-sdk-for-go/sdk/azcore dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go-deps - dependency-name: github.com/go-playground/validator/v10 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go-deps - dependency-name: github.com/onsi/ginkgo/v2 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go-deps - dependency-name: github.com/onsi/gomega dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go-deps - dependency-name: github.com/stretchr/testify dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go-deps ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 24 ++++++++++++------------ go.sum | 52 ++++++++++++++++++++++++++-------------------------- 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/go.mod b/go.mod index f92355a4d..b11722a98 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/Azure/azure-kusto-go v0.16.1 github.com/Azure/azure-sdk-for-go v68.0.0+incompatible github.com/Azure/azure-sdk-for-go-extensions v0.1.8 - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute v1.0.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0 @@ -26,17 +26,17 @@ require ( github.com/go-openapi/strfmt v0.23.0 github.com/go-openapi/swag v0.23.0 github.com/go-openapi/validate v0.24.0 - github.com/go-playground/validator/v10 v10.22.1 + github.com/go-playground/validator/v10 v10.23.0 github.com/google/go-cmp v0.6.0 github.com/imdario/mergo v0.3.16 github.com/jongio/azidext/go/azidext v0.5.0 github.com/mitchellh/hashstructure/v2 v2.0.2 - github.com/onsi/ginkgo/v2 v2.20.2 - github.com/onsi/gomega v1.34.2 + github.com/onsi/ginkgo/v2 v2.22.2 + github.com/onsi/gomega v1.36.2 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/prometheus/client_golang v1.20.5 github.com/samber/lo v1.47.0 - github.com/stretchr/testify v1.9.0 + github.com/stretchr/testify v1.10.0 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.0 golang.org/x/sync v0.10.0 @@ -105,7 +105,7 @@ require ( github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 // indirect + github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad // indirect github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.21.0 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect @@ -142,21 +142,21 @@ require ( go.opentelemetry.io/otel/trace v1.24.0 // indirect go.uber.org/automaxprocs v1.5.3 // indirect go.uber.org/mock v0.4.0 // indirect - golang.org/x/crypto v0.31.0 // indirect + golang.org/x/crypto v0.32.0 // indirect golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa // indirect - golang.org/x/net v0.29.0 // indirect + golang.org/x/net v0.34.0 // indirect golang.org/x/oauth2 v0.22.0 // indirect - golang.org/x/sys v0.28.0 // indirect - golang.org/x/term v0.27.0 // indirect + golang.org/x/sys v0.29.0 // indirect + golang.org/x/term v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.6.0 // indirect - golang.org/x/tools v0.24.0 // indirect + golang.org/x/tools v0.28.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/api v0.183.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240808171019-573a1156607a // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240808171019-573a1156607a // indirect google.golang.org/grpc v1.65.0 // indirect - google.golang.org/protobuf v1.34.2 // indirect + google.golang.org/protobuf v1.36.1 // indirect gopkg.in/dnaeon/go-vcr.v3 v3.2.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 65ead3fb2..66f5b2476 100644 --- a/go.sum +++ b/go.sum @@ -41,8 +41,8 @@ github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0 github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go-extensions v0.1.8 h1:x8Vu78C4r8mh6V2yQKQRSWLU+EYBVwFsf3XECddyb6s= github.com/Azure/azure-sdk-for-go-extensions v0.1.8/go.mod h1:4su5NjJwhqFH2B/5zJSKOz7hazfr2y38Iu6W4ZK0HYA= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 h1:JZg6HRh6W6U4OLl6lk7BZ7BLisIzM9dG1R50zUk9C/M= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0/go.mod h1:YL1xnZ6QejvQHWJrX/AvhFl4WW4rqHVoKspWNVwFk0M= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0 h1:g0EZJwz7xkXQiZAI5xi9f3WWFYBlX1CPTrR+NDToRkQ= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0/go.mod h1:XCW7KnZet0Opnr7HccfUw1PLc4CjHqpcaxW8DHklNkQ= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0 h1:B/dfvscEQtew9dVuoxqxrUKKv8Ih2f55PydknDamU+g= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0/go.mod h1:fiPSssYvltE08HJchL04dOy+RD4hgrjph0cwGGMntdI= github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.0 h1:+m0M/LFxN43KvULkDNfdXOgrjtg6UYJPFBJyuEcRCAw= @@ -218,8 +218,8 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.22.1 h1:40JcKH+bBNGFczGuoBYgX4I6m/i27HYW8P9FDk5PbgA= -github.com/go-playground/validator/v10 v10.22.1/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/go-playground/validator/v10 v10.23.0 h1:/PwmTwZhS0dPkav3cdK9kV1FsAmrL8sThn8IHr/sO+o= +github.com/go-playground/validator/v10 v10.23.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= @@ -293,8 +293,8 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 h1:5iH8iuqE5apketRbSFBy+X1V0o+l+8NF1avt4HWl7cA= -github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad h1:a6HEuzUHeKH6hwfN/ZoQgRgVIWFJljSWa/zetS2WTvg= +github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -371,10 +371,10 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/onsi/ginkgo/v2 v2.20.2 h1:7NVCeyIWROIAheY21RLS+3j2bb52W0W82tkberYytp4= -github.com/onsi/ginkgo/v2 v2.20.2/go.mod h1:K9gyxPIlb+aIvnZ8bd9Ak+YP18w3APlR+5coaZoE2ag= -github.com/onsi/gomega v1.34.2 h1:pNCwDkzrsv7MS9kpaQvVb1aVLahQXyJ/Tv5oAZMI3i8= -github.com/onsi/gomega v1.34.2/go.mod h1:v1xfxRgk0KIsG+QOdm7p8UosrOzPYRo60fd3B/1Dukc= +github.com/onsi/ginkgo/v2 v2.22.2 h1:/3X8Panh8/WwhU/3Ssa6rCKqPLuAkVY2I0RoyDLySlU= +github.com/onsi/ginkgo/v2 v2.22.2/go.mod h1:oeMosUL+8LtarXBHu/c0bx2D/K9zyQ6uX3cTyztHwsk= +github.com/onsi/gomega v1.36.2 h1:koNYke6TVk6ZmnyHrCXba/T/MoLBXFjeC1PtvYgw0A8= +github.com/onsi/gomega v1.36.2/go.mod h1:DdwyADRjrc825LhMEkD76cHR5+pUnjhUN8GlHlRPHzY= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= @@ -455,8 +455,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stvp/go-udp-testing v0.0.0-20201019212854-469649b16807/go.mod h1:7jxmlfBCDBXRzr0eAQJ48XC1hBu1np4CS5+cHEYfwpc= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -506,8 +506,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= -golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= +golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -542,8 +542,8 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= -golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= +golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -583,8 +583,8 @@ golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= -golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -655,15 +655,15 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= -golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= -golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg= +golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -725,8 +725,8 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= -golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= +golang.org/x/tools v0.28.0 h1:WuB6qZ4RPCQo5aP3WdKZS7i595EdWqWR8vqJTlwTVK8= +golang.org/x/tools v0.28.0/go.mod h1:dcIOrVd3mfQKTgrDVQHqCPMWy6lnhfhtX3hLXYVLfRw= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -822,8 +822,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= +google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=