diff --git a/Makefile b/Makefile index 4106f49a6..966c2279b 100644 --- a/Makefile +++ b/Makefile @@ -75,6 +75,7 @@ coverage: go tool cover -html coverage.out -o coverage.html verify: toolchain tidy download ## Verify code. Includes dependencies, linting, formatting, etc + make az-swagger-generate-clients-raw go generate ./... hack/boilerplate.sh cp $(KARPENTER_CORE_DIR)/pkg/apis/crds/* pkg/apis/crds diff --git a/Makefile-az.mk b/Makefile-az.mk index 3e99a73d3..9feaed157 100755 --- a/Makefile-az.mk +++ b/Makefile-az.mk @@ -358,3 +358,11 @@ az-helm-install-snapshot: az-configure-values ## Install Karpenter snapshot rele az-rmcrds: ## Delete Karpenter CRDs kubectl delete crd nodepools.karpenter.sh nodeclaims.karpenter.sh aksnodeclasses.karpenter.azure.com + +az-swagger-generate-clients-raw: + cd pkg/provisionclients && swagger generate client -f swagger/*.json + hack/azure/temp_fix_get_bootstrapping_resp_error.sh + +az-swagger-generate-clients: az-swagger-generate-clients-raw + hack/boilerplate.sh + make tidy diff --git a/go.mod b/go.mod index 012a4fc93..1e12d9175 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,11 @@ require ( github.com/blang/semver/v4 v4.0.0 github.com/go-logr/logr v1.4.2 github.com/go-logr/zapr v1.3.0 + github.com/go-openapi/errors v0.22.0 + github.com/go-openapi/runtime v0.28.0 + github.com/go-openapi/strfmt v0.23.0 + github.com/go-openapi/swag v0.23.0 + github.com/go-openapi/validate v0.24.0 github.com/go-playground/validator/v10 v10.22.1 github.com/google/go-cmp v0.6.0 github.com/imdario/mergo v0.3.16 @@ -68,6 +73,7 @@ require ( github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect + github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/avast/retry-go v3.0.0+incompatible // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blendle/zapdriver v1.3.1 // indirect @@ -83,9 +89,11 @@ require ( github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.6.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-openapi/analysis v0.23.0 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.21.0 // indirect - github.com/go-openapi/swag v0.23.0 // indirect + github.com/go-openapi/loads v0.22.0 // indirect + github.com/go-openapi/spec v0.21.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect @@ -108,9 +116,12 @@ require ( github.com/kylelemons/godebug v1.1.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/oklog/ulid v1.3.1 // indirect + github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -122,6 +133,7 @@ require ( github.com/shopspring/decimal v1.3.1 // indirect github.com/spf13/cobra v1.8.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + go.mongodb.org/mongo-driver v1.14.0 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect go.opentelemetry.io/otel v1.24.0 // indirect diff --git a/go.sum b/go.sum index 591e0d7e1..9de29f5e9 100644 --- a/go.sum +++ b/go.sum @@ -121,6 +121,8 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= github.com/awslabs/operatorpkg v0.0.0-20240805231134-67d0acfb6306 h1:0dzaVod1XLEc38H4IB+KOgStoCt8RkCVI4t+XsSPrWE= @@ -190,12 +192,26 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/analysis v0.23.0 h1:aGday7OWupfMs+LbmLZG4k0MYXIANxcuBTYUC03zFCU= +github.com/go-openapi/analysis v0.23.0/go.mod h1:9mz9ZWaSlV8TvjQHLl2mUW2PbZtemkE8yA5v22ohupo= +github.com/go-openapi/errors v0.22.0 h1:c4xY/OLxUBSTiepAg3j/MHuAv5mJhnf53LLMWFB+u/w= +github.com/go-openapi/errors v0.22.0/go.mod h1:J3DmZScxCDufmIMsdOuDHxJbdOGC0xtUynjIx092vXE= github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= +github.com/go-openapi/loads v0.22.0 h1:ECPGd4jX1U6NApCGG1We+uEozOAvXvJSF4nnwHZ8Aco= +github.com/go-openapi/loads v0.22.0/go.mod h1:yLsaTCS92mnSAZX5WWoxszLj0u+Ojl+Zs5Stn1oF+rs= +github.com/go-openapi/runtime v0.28.0 h1:gpPPmWSNGo214l6n8hzdXYhPuJcGtziTOgUpvsFWGIQ= +github.com/go-openapi/runtime v0.28.0/go.mod h1:QN7OzcS+XuYmkQLw05akXk0jRH/eZ3kb18+1KwW9gyc= +github.com/go-openapi/spec v0.21.0 h1:LTVzPc3p/RzRnkQqLRndbAzjY0d0BCL72A6j3CdL9ZY= +github.com/go-openapi/spec v0.21.0/go.mod h1:78u6VdPw81XU44qEWGhtr982gJ5BWg2c0I5XwVMotYk= +github.com/go-openapi/strfmt v0.23.0 h1:nlUS6BCqcnAk0pyhi9Y+kdDVZdZMHfEKQiS4HaMgO/c= +github.com/go-openapi/strfmt v0.23.0/go.mod h1:NrtIpfKtWIygRkKVsxh7XQMDQW5HKQl6S5ik2elW+K4= github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-openapi/validate v0.24.0 h1:LdfDKwNbpB6Vn40xhTdNZAnfLECL81w+VX3BumrGD58= +github.com/go-openapi/validate v0.24.0/go.mod h1:iyeX1sEufmv3nPbBdX3ieNviWnOZaJ1+zquzJEf2BAQ= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= @@ -339,6 +355,8 @@ github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4= github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -350,10 +368,14 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo/v2 v2.20.2 h1:7NVCeyIWROIAheY21RLS+3j2bb52W0W82tkberYytp4= github.com/onsi/ginkgo/v2 v2.20.2/go.mod h1:K9gyxPIlb+aIvnZ8bd9Ak+YP18w3APlR+5coaZoE2ag= github.com/onsi/gomega v1.34.2 h1:pNCwDkzrsv7MS9kpaQvVb1aVLahQXyJ/Tv5oAZMI3i8= github.com/onsi/gomega v1.34.2/go.mod h1:v1xfxRgk0KIsG+QOdm7p8UosrOzPYRo60fd3B/1Dukc= +github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= +github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= @@ -440,6 +462,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.mongodb.org/mongo-driver v1.14.0 h1:P98w8egYRjYe3XDjxhYJagTokP/H6HzlsnojRgZRd80= +go.mongodb.org/mongo-driver v1.14.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -454,6 +478,8 @@ go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= +go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= +go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= diff --git a/hack/azure/temp_fix_get_bootstrapping_resp_error.sh b/hack/azure/temp_fix_get_bootstrapping_resp_error.sh new file mode 100755 index 000000000..28b00633c --- /dev/null +++ b/hack/azure/temp_fix_get_bootstrapping_resp_error.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# Define the file path +FILE="pkg/provisionclients/client/operations/node_bootstrapping_get_responses.go" + +# Check if the file exists +if [ ! -f "$FILE" ]; then + echo "File $FILE does not exist." + exit 1 +fi + +# Use sed to delete the readResponse() method if it exists +sed -i '/func (o \*NodeBootstrappingGetDefault) readResponse/,/^}/d' "$FILE" + +echo "readResponse() method deleted from $FILE if it existed. This is for a temporary fix that is in node_bootstrapping_get_responses_override.go." \ No newline at end of file diff --git a/hack/toolchain.sh b/hack/toolchain.sh index 3dfcb4287..80cbe5312 100755 --- a/hack/toolchain.sh +++ b/hack/toolchain.sh @@ -24,6 +24,7 @@ tools() { go install github.com/rhysd/actionlint/cmd/actionlint@v1.7.3 go install github.com/mattn/goveralls@v0.0.12 go install github.com/google/go-containerregistry/cmd/crane@v0.20.2 + go install github.com/go-swagger/go-swagger/cmd/swagger@v0.31.0 if ! echo "$PATH" | grep -q "${GOPATH:-undefined}/bin\|$HOME/go/bin"; then echo "Go workspace's \"bin\" directory is not in PATH. Run 'export PATH=\"\$PATH:\${GOPATH:-\$HOME/go}/bin\"'." diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go index bdb45c3b0..b64270faf 100644 --- a/pkg/consts/consts.go +++ b/pkg/consts/consts.go @@ -28,4 +28,7 @@ const ( NetworkDataplaneAzure = "azure" DefaultKubernetesMaxPods = 250 + + ProvisionModeAKSScriptless = "aksscriptless" + ProvisionModeBootstrappingClient = "bootstrappingclient" ) diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 4c670e038..0cf48ff27 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -117,10 +117,12 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont options.FromContext(ctx).ClusterEndpoint, azConfig.TenantID, azConfig.SubscriptionID, + azConfig.ResourceGroup, azConfig.KubeletIdentityClientID, azConfig.NodeResourceGroup, azConfig.Location, vnetGUID, + options.FromContext(ctx).ProvisionMode, ) instanceTypeProvider := instancetype.NewDefaultProvider( azConfig.Location, @@ -143,6 +145,7 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont azConfig.Location, azConfig.NodeResourceGroup, azConfig.SubscriptionID, + options.FromContext(ctx).ProvisionMode, ) return ctx, &Operator{ diff --git a/pkg/operator/options/options.go b/pkg/operator/options/options.go index 784b93d6c..3e71012bd 100644 --- a/pkg/operator/options/options.go +++ b/pkg/operator/options/options.go @@ -75,7 +75,9 @@ type Options struct { SubnetID string // => VnetSubnetID to use (for nodes in Azure CNI Overlay and Azure CNI + pod subnet; for for nodes and pods in Azure CNI), unless overridden via AKSNodeClass setFlags map[string]bool - NodeResourceGroup string + NodeResourceGroup string + ProvisionMode string + NodeBootstrappingServerURL string } func (o *Options) AddFlags(fs *coreoptions.FlagSet) { @@ -85,12 +87,14 @@ func (o *Options) AddFlags(fs *coreoptions.FlagSet) { fs.StringVar(&o.KubeletClientTLSBootstrapToken, "kubelet-bootstrap-token", env.WithDefaultString("KUBELET_BOOTSTRAP_TOKEN", ""), "[REQUIRED] The bootstrap token for new nodes to join the cluster.") fs.StringVar(&o.SSHPublicKey, "ssh-public-key", env.WithDefaultString("SSH_PUBLIC_KEY", ""), "[REQUIRED] VM SSH public key.") fs.StringVar(&o.NetworkPlugin, "network-plugin", env.WithDefaultString("NETWORK_PLUGIN", consts.NetworkPluginAzure), "The network plugin used by the cluster.") - fs.StringVar(&o.NetworkPluginMode, "network-plugin-mode", env.WithDefaultString("NETWORK_PLUGIN_MODE", consts.NetworkPluginModeOverlay), "network plugin mode of the cluster") + fs.StringVar(&o.NetworkPluginMode, "network-plugin-mode", env.WithDefaultString("NETWORK_PLUGIN_MODE", consts.NetworkPluginModeOverlay), "network plugin mode of the cluster.") fs.StringVar(&o.NetworkPolicy, "network-policy", env.WithDefaultString("NETWORK_POLICY", ""), "The network policy used by the cluster.") fs.StringVar(&o.NetworkDataplane, "network-dataplane", env.WithDefaultString("NETWORK_DATAPLANE", "cilium"), "The network dataplane used by the cluster.") - fs.StringVar(&o.SubnetID, "vnet-subnet-id", env.WithDefaultString("VNET_SUBNET_ID", ""), "The default subnet ID to use for new nodes. This must be a valid ARM resource ID for subnet that does not overlap with the service CIDR or the pod CIDR") + fs.StringVar(&o.SubnetID, "vnet-subnet-id", env.WithDefaultString("VNET_SUBNET_ID", ""), "The default subnet ID to use for new nodes. This must be a valid ARM resource ID for subnet that does not overlap with the service CIDR or the pod CIDR.") fs.Var(newNodeIdentitiesValue(env.WithDefaultString("NODE_IDENTITIES", ""), &o.NodeIdentities), "node-identities", "User assigned identities for nodes.") - fs.StringVar(&o.NodeResourceGroup, "node-resource-group", env.WithDefaultString("AZURE_NODE_RESOURCE_GROUP", ""), "[REQUIRED] the resource group created and managed by AKS where the nodes live") + fs.StringVar(&o.NodeResourceGroup, "node-resource-group", env.WithDefaultString("AZURE_NODE_RESOURCE_GROUP", ""), "[REQUIRED] the resource group created and managed by AKS where the nodes live.") + fs.StringVar(&o.ProvisionMode, "provision-mode", env.WithDefaultString("PROVISION_MODE", consts.ProvisionModeAKSScriptless), "[UNSUPPORTED] The provision mode for the cluster.") + fs.StringVar(&o.NodeBootstrappingServerURL, "nodebootstrapping-server-url", env.WithDefaultString("NODEBOOTSTRAPPING_SERVER_URL", ""), "[UNSUPPORTED] The url for the node bootstrapping provider server.") } func (o Options) GetAPIServerName() string { diff --git a/pkg/operator/options/options_validation.go b/pkg/operator/options/options_validation.go index ece1efff0..63cc14e0d 100644 --- a/pkg/operator/options/options_validation.go +++ b/pkg/operator/options/options_validation.go @@ -34,6 +34,7 @@ func (o Options) Validate() error { o.validateNetworkingOptions(), o.validateVMMemoryOverheadPercent(), o.validateVnetSubnetID(), + o.validateProvisionMode(), validate.Struct(o), ) } @@ -83,6 +84,18 @@ func (o Options) validateVMMemoryOverheadPercent() error { return nil } +func (o Options) validateProvisionMode() error { + if o.ProvisionMode != consts.ProvisionModeAKSScriptless && o.ProvisionMode != consts.ProvisionModeBootstrappingClient { + return fmt.Errorf("provision-mode is invalid: %s", o.ProvisionMode) + } + if o.ProvisionMode == consts.ProvisionModeBootstrappingClient { + if o.NodeBootstrappingServerURL == "" { + return fmt.Errorf("nodebootstrapping-server-url is required when provision-mode is bootstrappingclient") + } + } + return nil +} + func (o Options) validateRequiredFields() error { if o.ClusterEndpoint == "" { return fmt.Errorf("missing field, cluster-endpoint") diff --git a/pkg/operator/options/suite_test.go b/pkg/operator/options/suite_test.go index 9037e2dfc..fa780fabc 100644 --- a/pkg/operator/options/suite_test.go +++ b/pkg/operator/options/suite_test.go @@ -53,6 +53,8 @@ var _ = Describe("Options", func() { "NETWORK_PLUGIN", "NETWORK_POLICY", "NODE_IDENTITIES", + "PROVISION_MODE", + "NODEBOOTSTRAPPING_SERVER_URL", } var fs *coreoptions.FlagSet @@ -95,6 +97,8 @@ var _ = Describe("Options", func() { os.Setenv("NETWORK_POLICY", "env-network-policy") os.Setenv("NODE_IDENTITIES", "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/envid1,/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/envid2") os.Setenv("VNET_SUBNET_ID", "/subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/sillygeese/providers/Microsoft.Network/virtualNetworks/karpentervnet/subnets/karpentersub") + os.Setenv("PROVISION_MODE", "bootstrappingclient") + os.Setenv("NODEBOOTSTRAPPING_SERVER_URL", "https://nodebootstrapping-server-url") fs = &coreoptions.FlagSet{ FlagSet: flag.NewFlagSet("karpenter", flag.ContinueOnError), } @@ -112,6 +116,8 @@ var _ = Describe("Options", func() { NetworkPolicy: lo.ToPtr("env-network-policy"), SubnetID: lo.ToPtr("/subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/sillygeese/providers/Microsoft.Network/virtualNetworks/karpentervnet/subnets/karpentersub"), NodeIdentities: []string{"/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/envid1", "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/envid2"}, + ProvisionMode: lo.ToPtr("bootstrappingclient"), + NodeBootstrappingServerURL: lo.ToPtr("https://nodebootstrapping-server-url"), })) }) }) @@ -273,6 +279,28 @@ var _ = Describe("Options", func() { ) Expect(err).ToNot(HaveOccurred()) }) + It("should fail validation when ProvisionMode is not valid", func() { + err := opts.Parse( + fs, + "--cluster-name", "my-name", + "--cluster-endpoint", "https://karpenter-000000000000.hcp.westus2.staging.azmk8s.io", + "--kubelet-bootstrap-token", "flag-bootstrap-token", + "--ssh-public-key", "flag-ssh-public-key", + "--provision-mode", "ekeselfexposed", + ) + Expect(err).To(MatchError(ContainSubstring("provision-mode"))) + }) + It("should fail validation when ProvisionMode is bootstrappingclient but NodebootstrappingServerURL is not provided", func() { + err := opts.Parse( + fs, + "--cluster-name", "my-name", + "--cluster-endpoint", "https://karpenter-000000000000.hcp.westus2.staging.azmk8s.io", + "--kubelet-bootstrap-token", "flag-bootstrap-token", + "--ssh-public-key", "flag-ssh-public-key", + "--provision-mode", "bootstrappingclient", + ) + Expect(err).To(MatchError(ContainSubstring("nodebootstrapping-server-url"))) + }) }) }) diff --git a/pkg/providers/imagefamily/azlinux.go b/pkg/providers/imagefamily/azlinux.go index 4412f3b83..13be8fe07 100644 --- a/pkg/providers/imagefamily/azlinux.go +++ b/pkg/providers/imagefamily/azlinux.go @@ -21,6 +21,7 @@ import ( "github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2" "github.com/Azure/karpenter-provider-azure/pkg/providers/imagefamily/bootstrap" + "github.com/Azure/karpenter-provider-azure/pkg/providers/imagefamily/customscriptsbootstrap" "github.com/Azure/karpenter-provider-azure/pkg/providers/launchtemplate/parameters" karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" @@ -52,6 +53,7 @@ func (u AzureLinux) DefaultImages() []DefaultImageOutput { scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, karpv1.ArchitectureAmd64), scheduling.NewRequirement(v1alpha2.LabelSKUHyperVGeneration, v1.NodeSelectorOpIn, v1alpha2.HyperVGenerationV2), ), + Distro: "aks-azurelinux-v2-gen2", }, { CommunityImage: AzureLinuxGen1CommunityImage, @@ -60,6 +62,7 @@ func (u AzureLinux) DefaultImages() []DefaultImageOutput { scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, karpv1.ArchitectureAmd64), scheduling.NewRequirement(v1alpha2.LabelSKUHyperVGeneration, v1.NodeSelectorOpIn, v1alpha2.HyperVGenerationV1), ), + Distro: "aks-azurelinux-v2", }, { CommunityImage: AzureLinuxGen2ArmCommunityImage, @@ -68,12 +71,13 @@ func (u AzureLinux) DefaultImages() []DefaultImageOutput { scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, karpv1.ArchitectureArm64), scheduling.NewRequirement(v1alpha2.LabelSKUHyperVGeneration, v1.NodeSelectorOpIn, v1alpha2.HyperVGenerationV2), ), + Distro: "aks-azurelinux-v2-arm64-gen2", }, } } // UserData returns the default userdata script for the image Family -func (u AzureLinux) UserData(kubeletConfig *bootstrap.KubeletConfiguration, taints []v1.Taint, labels map[string]string, caBundle *string, _ *cloudprovider.InstanceType) bootstrap.Bootstrapper { +func (u AzureLinux) ScriptlessCustomData(kubeletConfig *bootstrap.KubeletConfiguration, taints []v1.Taint, labels map[string]string, caBundle *string, _ *cloudprovider.InstanceType) bootstrap.Bootstrapper { return bootstrap.AKS{ Options: bootstrap.Options{ ClusterName: u.Options.ClusterName, @@ -102,3 +106,24 @@ func (u AzureLinux) UserData(kubeletConfig *bootstrap.KubeletConfiguration, tain KubernetesVersion: u.Options.KubernetesVersion, } } + +// UserData returns the default userdata script for the image Family +func (u AzureLinux) CustomScriptsNodeBootstrapping(kubeletConfig *bootstrap.KubeletConfiguration, taints []v1.Taint, startupTaints []v1.Taint, labels map[string]string, instanceType *cloudprovider.InstanceType, imageDistro string, storageProfile string) customscriptsbootstrap.Bootstrapper { + return customscriptsbootstrap.ProvisionClientBootstrap{ + ClusterName: u.Options.ClusterName, + KubeletConfig: kubeletConfig, + Taints: taints, + StartupTaints: startupTaints, + Labels: labels, + SubnetID: u.Options.SubnetID, + Arch: u.Options.Arch, + SubscriptionID: u.Options.SubscriptionID, + ResourceGroup: u.Options.ResourceGroup, + KubeletClientTLSBootstrapToken: u.Options.KubeletClientTLSBootstrapToken, + KubernetesVersion: u.Options.KubernetesVersion, + ImageDistro: imageDistro, + InstanceType: instanceType, + StorageProfile: storageProfile, + ClusterResourceGroup: u.Options.ClusterResourceGroup, + } +} diff --git a/pkg/providers/imagefamily/customscriptsbootstrap/customscriptsbootstrap.go b/pkg/providers/imagefamily/customscriptsbootstrap/customscriptsbootstrap.go new file mode 100644 index 000000000..f6e411173 --- /dev/null +++ b/pkg/providers/imagefamily/customscriptsbootstrap/customscriptsbootstrap.go @@ -0,0 +1,28 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package customscriptsbootstrap + +import ( + "context" +) + +// Bootstrapper can be implemented to generate a bootstrap script +// that uses the params from the Bootstrap type for a specific +// bootstrapping method. +type Bootstrapper interface { + GetCustomDataAndCSE(ctx context.Context) (string, string, error) +} diff --git a/pkg/providers/imagefamily/customscriptsbootstrap/provisionclientbootstrap.go b/pkg/providers/imagefamily/customscriptsbootstrap/provisionclientbootstrap.go new file mode 100644 index 000000000..71adfacc3 --- /dev/null +++ b/pkg/providers/imagefamily/customscriptsbootstrap/provisionclientbootstrap.go @@ -0,0 +1,259 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package customscriptsbootstrap + +import ( + "context" + "encoding/base64" + "fmt" + "math" + "strings" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2" + "github.com/Azure/karpenter-provider-azure/pkg/operator/options" + "github.com/Azure/karpenter-provider-azure/pkg/providers/imagefamily/bootstrap" + "github.com/Azure/karpenter-provider-azure/pkg/provisionclients/client" + "github.com/Azure/karpenter-provider-azure/pkg/provisionclients/client/operations" + "github.com/Azure/karpenter-provider-azure/pkg/provisionclients/models" + "github.com/Azure/karpenter-provider-azure/pkg/utils" + + v1 "k8s.io/api/core/v1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" + + "sigs.k8s.io/karpenter/pkg/cloudprovider" + + httptransport "github.com/go-openapi/runtime/client" + "github.com/go-openapi/strfmt" + "github.com/samber/lo" + + "k8s.io/apimachinery/pkg/api/resource" +) + +type ProvisionClientBootstrap struct { + ClusterName string + KubeletConfig *bootstrap.KubeletConfiguration + Taints []v1.Taint `hash:"set"` + StartupTaints []v1.Taint `hash:"set"` + Labels map[string]string `hash:"set"` + SubnetID string + Arch string + SubscriptionID string + ClusterResourceGroup string + ResourceGroup string + KubeletClientTLSBootstrapToken string + KubernetesVersion string + ImageDistro string + IsWindows bool + InstanceType *cloudprovider.InstanceType + StorageProfile string + ImageFamily string +} + +var _ Bootstrapper = (*ProvisionClientBootstrap)(nil) // assert ProvisionClientBootstrap implements customscriptsbootstrapper + +func (p ProvisionClientBootstrap) GetCustomDataAndCSE(ctx context.Context) (string, string, error) { + if p.IsWindows { + // TODO(Windows) + return "", "", fmt.Errorf("windows is not supported") + } + + labels := lo.Assign(map[string]string{}, p.Labels) + getAgentbakerGeneratedLabels(p.ResourceGroup, labels) + + provisionProfile := &models.ProvisionProfile{ + Name: lo.ToPtr(""), + Architecture: lo.ToPtr(lo.Ternary(p.Arch == karpv1.ArchitectureAmd64, "x64", "Arm64")), + OsType: lo.ToPtr(lo.Ternary(p.IsWindows, models.OSTypeWindows, models.OSTypeLinux)), + VMSize: lo.ToPtr(p.InstanceType.Name), + Distro: lo.ToPtr(p.ImageDistro), + CustomNodeLabels: labels, + OrchestratorVersion: lo.ToPtr(p.KubernetesVersion), + VnetSubnetID: lo.ToPtr(p.SubnetID), + StorageProfile: lo.ToPtr(p.StorageProfile), + NodeInitializationTaints: lo.Map(p.StartupTaints, func(taint v1.Taint, _ int) string { return taint.ToString() }), + NodeTaints: lo.Map(p.Taints, func(taint v1.Taint, _ int) string { return taint.ToString() }), + SecurityProfile: &models.AgentPoolSecurityProfile{ + SSHAccess: lo.ToPtr(models.SSHAccessLocalUser), + // EnableVTPM: lo.ToPtr(false), // Unsupported as of now (Trusted launch) + // EnableSecureBoot: lo.ToPtr(false), // Unsupported as of now (Trusted launch) + }, + MaxPods: lo.ToPtr(p.KubeletConfig.MaxPods), + + VnetCidrs: []string{}, // Unsupported as of now; TODO(Windows) + // MessageOfTheDay: lo.ToPtr(""), // Unsupported as of now + // AgentPoolWindowsProfile: &models.AgentPoolWindowsProfile{}, // Unsupported as of now; TODO(Windows) + // KubeletDiskType: lo.ToPtr(models.KubeletDiskTypeUnspecified), // Unsupported as of now + // CustomLinuxOSConfig: &models.CustomLinuxOSConfig{}, // Unsupported as of now (sysctl) + // EnableFIPS: lo.ToPtr(false), // Unsupported as of now + // GpuInstanceProfile: lo.ToPtr(models.GPUInstanceProfileUnspecified), // Unsupported as of now (MIG) + // WorkloadRuntime: lo.ToPtr(models.WorkloadRuntimeUnspecified), // Unsupported as of now (Kata) + // ArtifactStreamingProfile: &models.ArtifactStreamingProfile{ + // Enabled: lo.ToPtr(false), // Unsupported as of now + // }, + } + + switch p.ImageFamily { + case v1alpha2.Ubuntu2204ImageFamily: + provisionProfile.OsSku = to.Ptr(models.OSSKUUbuntu) + case v1alpha2.AzureLinuxImageFamily: + provisionProfile.OsSku = to.Ptr(models.OSSKUAzureLinux) + default: + provisionProfile.OsSku = to.Ptr(models.OSSKUUbuntu) + } + + if p.KubeletConfig != nil { + provisionProfile.CustomKubeletConfig = &models.CustomKubeletConfig{ + CPUManagerPolicy: lo.ToPtr(p.KubeletConfig.CPUManagerPolicy), + CPUCfsQuota: p.KubeletConfig.CPUCFSQuota, + CPUCfsQuotaPeriod: lo.ToPtr(p.KubeletConfig.CPUCFSQuotaPeriod.String()), + ImageGcHighThreshold: p.KubeletConfig.ImageGCHighThresholdPercent, + ImageGcLowThreshold: p.KubeletConfig.ImageGCLowThresholdPercent, + TopologyManagerPolicy: lo.ToPtr(p.KubeletConfig.TopologyManagerPolicy), + AllowedUnsafeSysctls: p.KubeletConfig.AllowedUnsafeSysctls, + ContainerLogMaxSizeMB: convertContainerLogMaxSizeToMB(p.KubeletConfig.ContainerLogMaxSize), + ContainerLogMaxFiles: p.KubeletConfig.ContainerLogMaxFiles, + PodMaxPids: convertPodMaxPids(p.KubeletConfig.PodPidsLimit), + } + } + + if modeString, ok := p.Labels["kubernetes.azure.com/mode"]; ok && modeString == "system" { + provisionProfile.Mode = lo.ToPtr(models.AgentPoolModeSystem) + } else { + provisionProfile.Mode = lo.ToPtr(models.AgentPoolModeUser) + } + + if utils.IsNvidiaEnabledSKU(p.InstanceType.Name) { + provisionProfile.GpuProfile = &models.GPUProfile{ + DriverType: lo.ToPtr(lo.Ternary(utils.UseGridDrivers(p.InstanceType.Name), models.DriverTypeGRID, models.DriverTypeCUDA)), + InstallGPUDriver: lo.ToPtr(true), + } + } + + provisionHelperValues := &models.ProvisionHelperValues{ + SkuCPU: lo.ToPtr(p.InstanceType.Capacity.Cpu().AsApproximateFloat64()), + SkuMemory: lo.ToPtr(math.Ceil(reverseVMMemoryOverhead(options.FromContext(ctx).VMMemoryOverheadPercent, p.InstanceType.Capacity.Memory().AsApproximateFloat64()) / 1024 / 1024 / 1024)), + } + + return p.getNodeBootstrappingFromClient(ctx, provisionProfile, provisionHelperValues, p.KubeletClientTLSBootstrapToken) +} + +func (p *ProvisionClientBootstrap) getNodeBootstrappingFromClient(ctx context.Context, provisionProfile *models.ProvisionProfile, provisionHelperValues *models.ProvisionHelperValues, bootstrapToken string) (string, string, error) { + transport := httptransport.New(options.FromContext(ctx).NodeBootstrappingServerURL, "/", []string{"http"}) + client := client.New(transport, strfmt.Default) + + params := operations.NewNodeBootstrappingGetParams() + params.ResourceGroupName = p.ClusterResourceGroup + params.ResourceName = p.ClusterName + params.SubscriptionID = p.SubscriptionID + provisionValues := &models.ProvisionValues{ + ProvisionProfile: provisionProfile, + ProvisionHelperValues: provisionHelperValues, + } + params.Parameters = provisionValues + + params.WithTimeout(30 * time.Second) + params.Context = ctx + + resp, err := client.Operations.NodeBootstrappingGet(params) + if err != nil { + // As of now we just fail the provisioning given the unlikely scenario of retriable error, but could be revisted along with retriable status on the server side. + return "", "", err + } + + if resp.Payload == nil { + return "", "", fmt.Errorf("no payload in response") + } + if resp.Payload.Cse == nil || *resp.Payload.Cse == "" { + return "", "", fmt.Errorf("no CSE in response") + } + if resp.Payload.CustomData == nil || *resp.Payload.CustomData == "" { + return "", "", fmt.Errorf("no CustomData in response") + } + + cseWithoutBootstrapToken := *resp.Payload.Cse + customDataWithoutBootstrapToken := *resp.Payload.CustomData + + cseWithBootstrapToken := strings.ReplaceAll(cseWithoutBootstrapToken, "{{.TokenID}}.{{.TokenSecret}}", bootstrapToken) + + decodedCustomDataWithoutBootstrapTokenInBytes, err := base64.StdEncoding.DecodeString(customDataWithoutBootstrapToken) + if err != nil { + return "", "", err + } + decodedCustomDataWithBootstrapToken := strings.ReplaceAll(string(decodedCustomDataWithoutBootstrapTokenInBytes), "{{.TokenID}}.{{.TokenSecret}}", bootstrapToken) + customDataWithBootstrapToken := base64.StdEncoding.EncodeToString([]byte(decodedCustomDataWithBootstrapToken)) + + return customDataWithBootstrapToken, cseWithBootstrapToken, nil +} + +func getAgentbakerGeneratedLabels(nodeResourceGroup string, nodeLabels map[string]string) { + // Delegatable defaulting? + nodeLabels["kubernetes.azure.com/role"] = "agent" + nodeLabels["kubernetes.azure.com/cluster"] = normalizeResourceGroupNameForLabel(nodeResourceGroup) +} + +func normalizeResourceGroupNameForLabel(resourceGroupName string) string { + truncated := resourceGroupName + truncated = strings.ReplaceAll(truncated, "(", "-") + truncated = strings.ReplaceAll(truncated, ")", "-") + const maxLen = 63 + if len(truncated) > maxLen { + truncated = truncated[0:maxLen] + } + + if strings.HasSuffix(truncated, "-") || + strings.HasSuffix(truncated, "_") || + strings.HasSuffix(truncated, ".") { + if len(truncated) > 62 { + return truncated[0:len(truncated)-1] + "z" + } + return truncated + "z" + } + return truncated +} + +func reverseVMMemoryOverhead(vmMemoryOverheadPercent float64, adjustedMemory float64) float64 { + // This is not the best way to do it... But will be refactored later, given that retreiving the original memory properly might involves some restructure. + // Due to the fact that it is abstracted behind the cloudprovider interface. + return adjustedMemory / (1 - vmMemoryOverheadPercent) +} + +func convertContainerLogMaxSizeToMB(containerLogMaxSize string) *int32 { + q, err := resource.ParseQuantity(containerLogMaxSize) + if err == nil { + // This could be improved later + return lo.ToPtr(int32(math.Round(q.AsApproximateFloat64() / 1024 / 1024))) + } + return nil +} + +func convertPodMaxPids(podPidsLimit *int64) *int32 { + if podPidsLimit != nil { + podPidsLimitInt64 := *podPidsLimit + if podPidsLimitInt64 > int64(math.MaxInt32) { + // This could be improved later + return lo.ToPtr(int32(math.MaxInt32)) + } else if podPidsLimitInt64 < 0 { + // This as well + return lo.ToPtr(int32(-1)) + } else { + return lo.ToPtr(int32(podPidsLimitInt64)) // golint:ignore G115 already check overflow + } + } + return nil +} diff --git a/pkg/providers/imagefamily/customscriptsbootstrap/provisionclientbootstrap_test.go b/pkg/providers/imagefamily/customscriptsbootstrap/provisionclientbootstrap_test.go new file mode 100644 index 000000000..9a0e50351 --- /dev/null +++ b/pkg/providers/imagefamily/customscriptsbootstrap/provisionclientbootstrap_test.go @@ -0,0 +1,155 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package customscriptsbootstrap + +import ( + "fmt" + "math" + "testing" + + "github.com/Azure/karpenter-provider-azure/pkg/providers/instancetype" + "github.com/samber/lo" +) + +func TestReverseVMMemoryOverhead(t *testing.T) { + type Cases struct { + name string + originalGiB int64 + vmMemoryOverheadPercent float64 + } + var cases []Cases + + // Generate test cases from originalGiB 1 to 2000 x vmMemoryOverheadPercent 0 to 0.25 using for loops + // In fact, mathematically, the reverse won't be perfectly equal, but current accuracy in GiB should be enough + for i := 1; i <= 2000; i++ { + for j := 0; j <= 250; j++ { + cases = append(cases, Cases{ + name: fmt.Sprintf("Test %d - %f", i, float64(j)/1000), + originalGiB: int64(i), + vmMemoryOverheadPercent: float64(j) / 1000, + }) + } + } + t.Run("2000 x 0.25", func(t *testing.T) { + for _, tc := range cases { + subtracted := instancetype.CalculateMemoryWithoutOverhead(tc.vmMemoryOverheadPercent, float64(tc.originalGiB)).Value() + reversedGiB := int64(math.Round(reverseVMMemoryOverhead(tc.vmMemoryOverheadPercent, float64(subtracted)) / 1024 / 1024 / 1024)) + if tc.originalGiB != reversedGiB { + t.Errorf("Expected %d but got %d", tc.originalGiB, reversedGiB) + } + } + }) +} + +func TestConvertContainerLogMaxSizeToMB(t *testing.T) { + tests := []struct { + name string + containerLogMaxSize string + expected *int32 + }{ + { + name: "Default", + containerLogMaxSize: "50Mi", + expected: lo.ToPtr(int32(50)), + }, + { + name: "Valid size in Mi", + containerLogMaxSize: "1024Mi", + expected: lo.ToPtr(int32(1024)), + }, + { + name: "Valid size in Gi", + containerLogMaxSize: "1Gi", + expected: lo.ToPtr(int32(1024)), + }, + { + name: "Valid size in Ki", + containerLogMaxSize: "1048576Ki", + expected: lo.ToPtr(int32(1024)), + }, + { + name: "Invalid size", + containerLogMaxSize: "invalid", + expected: nil, + }, + { + name: "Empty size", + containerLogMaxSize: "", + expected: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := convertContainerLogMaxSizeToMB(tt.containerLogMaxSize) + if tt.expected == nil && result != nil { + t.Errorf("Expected nil but got %v", *result) + } else if tt.expected != nil && result == nil { + t.Errorf("Expected %v but got nil", *tt.expected) + } else if tt.expected != nil && result != nil && *tt.expected != *result { + t.Errorf("Expected %v but got %v", *tt.expected, *result) + } + }) + } +} + +func TestConvertPodMaxPids(t *testing.T) { + tests := []struct { + name string + podPidsLimit *int64 + expected *int32 + }{ + { + name: "Valid PIDs limit within int32 range", + podPidsLimit: lo.ToPtr(int64(1000)), + expected: lo.ToPtr(int32(1000)), + }, + { + name: "PIDs limit exceeding int32 range", + podPidsLimit: lo.ToPtr(int64(math.MaxInt32) + int64(1)), + expected: lo.ToPtr(int32(math.MaxInt32)), + }, + { + name: "PIDs limit at int32 max value", + podPidsLimit: lo.ToPtr(int64(math.MaxInt32)), + expected: lo.ToPtr(int32(math.MaxInt32)), + }, + { + name: "PIDs limit almost at int32 max value", + podPidsLimit: lo.ToPtr(int64(math.MaxInt32 - 1)), + expected: lo.ToPtr(int32(math.MaxInt32 - 1)), + }, + { + name: "Nil PIDs limit", + podPidsLimit: nil, + expected: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := convertPodMaxPids(tt.podPidsLimit) + if tt.expected == nil && result != nil { + t.Errorf("Expected nil but got %v", *result) + } else if tt.expected != nil && result == nil { + t.Errorf("Expected %v but got nil", *tt.expected) + } else if tt.expected != nil && result != nil && *tt.expected != *result { + t.Errorf("Expected %v but got %v", *tt.expected, *result) + } + }) + } +} diff --git a/pkg/providers/imagefamily/image.go b/pkg/providers/imagefamily/image.go index c2622c245..b073db7e8 100644 --- a/pkg/providers/imagefamily/image.go +++ b/pkg/providers/imagefamily/image.go @@ -62,17 +62,21 @@ func NewProvider(kubernetesInterface kubernetes.Interface, kubernetesVersionCach } } -// Get returns Image ID for the given instance type. Images may vary due to architecture, accelerator, etc -func (p *Provider) Get(ctx context.Context, nodeClass *v1alpha2.AKSNodeClass, instanceType *cloudprovider.InstanceType, imageFamily ImageFamily) (string, error) { +// Get returns Distro and Image ID for the given instance type. Images may vary due to architecture, accelerator, etc +func (p *Provider) Get(ctx context.Context, nodeClass *v1alpha2.AKSNodeClass, instanceType *cloudprovider.InstanceType, imageFamily ImageFamily) (string, string, error) { defaultImages := imageFamily.DefaultImages() for _, defaultImage := range defaultImages { if err := instanceType.Requirements.Compatible(defaultImage.Requirements, v1alpha2.AllowUndefinedWellKnownAndRestrictedLabels); err == nil { communityImageName, publicGalleryURL := defaultImage.CommunityImage, defaultImage.PublicGalleryURL - return p.GetImageID(ctx, communityImageName, publicGalleryURL) + imageID, err := p.GetImageID(ctx, communityImageName, publicGalleryURL) + if err != nil { + return "", "", err + } + return defaultImage.Distro, imageID, nil } } - return "", fmt.Errorf("no compatible images found for instance type %s", instanceType.Name) + return "", "", fmt.Errorf("no compatible images found for instance type %s", instanceType.Name) } func (p *Provider) KubeServerVersion(ctx context.Context) (string, error) { diff --git a/pkg/providers/imagefamily/resolver.go b/pkg/providers/imagefamily/resolver.go index 06b8a1dc7..df7ec7f95 100644 --- a/pkg/providers/imagefamily/resolver.go +++ b/pkg/providers/imagefamily/resolver.go @@ -18,6 +18,7 @@ package imagefamily import ( "context" + "strconv" corev1 "k8s.io/api/core/v1" "knative.dev/pkg/logging" @@ -27,6 +28,7 @@ import ( "github.com/Azure/karpenter-provider-azure/pkg/consts" "github.com/Azure/karpenter-provider-azure/pkg/metrics" "github.com/Azure/karpenter-provider-azure/pkg/providers/imagefamily/bootstrap" + "github.com/Azure/karpenter-provider-azure/pkg/providers/imagefamily/customscriptsbootstrap" "github.com/Azure/karpenter-provider-azure/pkg/providers/instancetype" template "github.com/Azure/karpenter-provider-azure/pkg/providers/launchtemplate/parameters" "github.com/Azure/karpenter-provider-azure/pkg/utils" @@ -42,13 +44,22 @@ type Resolver struct { // ImageFamily can be implemented to override the default logic for generating dynamic launch template parameters type ImageFamily interface { - UserData( + ScriptlessCustomData( kubeletConfig *bootstrap.KubeletConfiguration, taints []corev1.Taint, labels map[string]string, caBundle *string, instanceType *cloudprovider.InstanceType, ) bootstrap.Bootstrapper + CustomScriptsNodeBootstrapping( + kubeletConfig *bootstrap.KubeletConfiguration, + taints []corev1.Taint, + startupTaints []corev1.Taint, + labels map[string]string, + instanceType *cloudprovider.InstanceType, + imageDistro string, + storageProfile string, + ) customscriptsbootstrap.Bootstrapper Name() string // DefaultImages returns a list of default CommunityImage definitions for this ImageFamily. // Our Image Selection logic relies on the ordering of the default images to be ordered from most preferred to least, then we will select the latest image version available for that CommunityImage definition. @@ -67,12 +78,14 @@ func New(_ client.Client, imageProvider *Provider) *Resolver { func (r Resolver) Resolve(ctx context.Context, nodeClass *v1alpha2.AKSNodeClass, nodeClaim *karpv1.NodeClaim, instanceType *cloudprovider.InstanceType, staticParameters *template.StaticParameters) (*template.Parameters, error) { imageFamily := getImageFamily(nodeClass.Spec.ImageFamily, staticParameters) - imageID, err := r.imageProvider.Get(ctx, nodeClass, instanceType, imageFamily) + imageDistro, imageID, err := r.imageProvider.Get(ctx, nodeClass, instanceType, imageFamily) if err != nil { metrics.ImageSelectionErrorCount.WithLabelValues(imageFamily.Name()).Inc() return nil, err } + logging.FromContext(ctx).Infof("Resolved image %s for instance type %s", imageID, instanceType.Name) + taints := lo.Flatten([][]corev1.Taint{ nodeClaim.Spec.Taints, nodeClaim.Spec.StartupTaints, @@ -83,18 +96,32 @@ func (r Resolver) Resolve(ctx context.Context, nodeClass *v1alpha2.AKSNodeClass, taints = append(taints, karpv1.UnregisteredNoExecuteTaint) } - logging.FromContext(ctx).Infof("Resolved image %s for instance type %s", imageID, instanceType.Name) + storageProfile := "ManagedDisks" + if useEphemeralDisk(instanceType, nodeClass) { + storageProfile = "Ephemeral" + } template := &template.Parameters{ StaticParameters: staticParameters, - UserData: imageFamily.UserData( + ScriptlessCustomData: imageFamily.ScriptlessCustomData( prepareKubeletConfiguration(instanceType, nodeClass), taints, staticParameters.Labels, staticParameters.CABundle, instanceType, ), - ImageID: imageID, + CustomScriptsNodeBootstrapping: imageFamily.CustomScriptsNodeBootstrapping( + prepareKubeletConfiguration(instanceType, nodeClass), + nodeClaim.Spec.Taints, + nodeClaim.Spec.StartupTaints, + staticParameters.Labels, + instanceType, + imageDistro, + storageProfile, + ), + ImageID: imageID, + StorageProfile: storageProfile, + IsWindows: false, // TODO(Windows) } return template, nil @@ -131,3 +158,22 @@ func getImageFamily(familyName *string, parameters *template.StaticParameters) I return &Ubuntu2204{Options: parameters} } } + +func getEphemeralMaxSizeGB(instanceType *cloudprovider.InstanceType) int32 { + reqs := instanceType.Requirements.Get(v1alpha2.LabelSKUStorageEphemeralOSMaxSize).Values() + if len(reqs) == 0 || len(reqs) > 1 { + return 0 + } + maxSize, err := strconv.ParseFloat(reqs[0], 32) + if err != nil { + return 0 + } + // decimal places are truncated, so we round down + return int32(maxSize) +} + +// setVMPropertiesStorageProfile enables ephemeral os disk for instance types that support it +func useEphemeralDisk(instanceType *cloudprovider.InstanceType, nodeClass *v1alpha2.AKSNodeClass) bool { + // use ephemeral disk if it is large enough + return *nodeClass.Spec.OSDiskSizeGB <= getEphemeralMaxSizeGB(instanceType) +} diff --git a/pkg/providers/imagefamily/types.go b/pkg/providers/imagefamily/types.go index 9268fa2c8..b3c3510aa 100644 --- a/pkg/providers/imagefamily/types.go +++ b/pkg/providers/imagefamily/types.go @@ -32,6 +32,7 @@ type DefaultImageOutput struct { CommunityImage string PublicGalleryURL string Requirements scheduling.Requirements + Distro string } // CommunityGalleryImageVersionsAPI is used for listing community gallery image versions. diff --git a/pkg/providers/imagefamily/ubuntu_2204.go b/pkg/providers/imagefamily/ubuntu_2204.go index 64b4496e9..21d3b8c03 100644 --- a/pkg/providers/imagefamily/ubuntu_2204.go +++ b/pkg/providers/imagefamily/ubuntu_2204.go @@ -21,6 +21,7 @@ import ( "github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2" "github.com/Azure/karpenter-provider-azure/pkg/providers/imagefamily/bootstrap" + "github.com/Azure/karpenter-provider-azure/pkg/providers/imagefamily/customscriptsbootstrap" "github.com/Azure/karpenter-provider-azure/pkg/providers/launchtemplate/parameters" karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" @@ -52,6 +53,7 @@ func (u Ubuntu2204) DefaultImages() []DefaultImageOutput { scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, karpv1.ArchitectureAmd64), scheduling.NewRequirement(v1alpha2.LabelSKUHyperVGeneration, v1.NodeSelectorOpIn, v1alpha2.HyperVGenerationV2), ), + Distro: "aks-ubuntu-containerd-22.04-gen2", }, { CommunityImage: Ubuntu2204Gen1CommunityImage, @@ -60,6 +62,7 @@ func (u Ubuntu2204) DefaultImages() []DefaultImageOutput { scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, karpv1.ArchitectureAmd64), scheduling.NewRequirement(v1alpha2.LabelSKUHyperVGeneration, v1.NodeSelectorOpIn, v1alpha2.HyperVGenerationV1), ), + Distro: "aks-ubuntu-containerd-22.04", }, { CommunityImage: Ubuntu2204Gen2ArmCommunityImage, @@ -68,12 +71,13 @@ func (u Ubuntu2204) DefaultImages() []DefaultImageOutput { scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, karpv1.ArchitectureArm64), scheduling.NewRequirement(v1alpha2.LabelSKUHyperVGeneration, v1.NodeSelectorOpIn, v1alpha2.HyperVGenerationV2), ), + Distro: "aks-ubuntu-arm64-containerd-22.04-gen2", }, } } // UserData returns the default userdata script for the image Family -func (u Ubuntu2204) UserData(kubeletConfig *bootstrap.KubeletConfiguration, taints []v1.Taint, labels map[string]string, caBundle *string, _ *cloudprovider.InstanceType) bootstrap.Bootstrapper { +func (u Ubuntu2204) ScriptlessCustomData(kubeletConfig *bootstrap.KubeletConfiguration, taints []v1.Taint, labels map[string]string, caBundle *string, _ *cloudprovider.InstanceType) bootstrap.Bootstrapper { return bootstrap.AKS{ Options: bootstrap.Options{ ClusterName: u.Options.ClusterName, @@ -101,3 +105,24 @@ func (u Ubuntu2204) UserData(kubeletConfig *bootstrap.KubeletConfiguration, tain KubernetesVersion: u.Options.KubernetesVersion, } } + +// UserData returns the default userdata script for the image Family +func (u Ubuntu2204) CustomScriptsNodeBootstrapping(kubeletConfig *bootstrap.KubeletConfiguration, taints []v1.Taint, startupTaints []v1.Taint, labels map[string]string, instanceType *cloudprovider.InstanceType, imageDistro string, storageProfile string) customscriptsbootstrap.Bootstrapper { + return customscriptsbootstrap.ProvisionClientBootstrap{ + ClusterName: u.Options.ClusterName, + KubeletConfig: kubeletConfig, + Taints: taints, + StartupTaints: startupTaints, + Labels: labels, + SubnetID: u.Options.SubnetID, + Arch: u.Options.Arch, + SubscriptionID: u.Options.SubscriptionID, + ResourceGroup: u.Options.ResourceGroup, + KubeletClientTLSBootstrapToken: u.Options.KubeletClientTLSBootstrapToken, + KubernetesVersion: u.Options.KubernetesVersion, + ImageDistro: imageDistro, + InstanceType: instanceType, + StorageProfile: storageProfile, + ClusterResourceGroup: u.Options.ClusterResourceGroup, + } +} diff --git a/pkg/providers/instance/instance.go b/pkg/providers/instance/instance.go index 2faf02bdb..89939746e 100644 --- a/pkg/providers/instance/instance.go +++ b/pkg/providers/instance/instance.go @@ -23,7 +23,6 @@ import ( "fmt" "math" "sort" - "strconv" "strings" "time" @@ -101,6 +100,7 @@ type DefaultProvider struct { resourceGroup string subscriptionID string unavailableOfferings *cache.UnavailableOfferings + provisionMode string } func NewDefaultProvider( @@ -112,6 +112,7 @@ func NewDefaultProvider( location string, resourceGroup string, subscriptionID string, + provisionMode string, ) *DefaultProvider { listQuery = GetListQueryBuilder(resourceGroup).String() return &DefaultProvider{ @@ -123,6 +124,7 @@ func NewDefaultProvider( resourceGroup: resourceGroup, subscriptionID: subscriptionID, unavailableOfferings: offeringsCache, + provisionMode: provisionMode, } } @@ -132,6 +134,7 @@ func (p *DefaultProvider) Create(ctx context.Context, nodeClass *v1alpha2.AKSNod instanceTypes = orderInstanceTypesByPrice(instanceTypes, scheduling.NewNodeSelectorRequirementsWithMinValues(nodeClaim.Spec.Requirements...)) vm, instanceType, err := p.launchInstance(ctx, nodeClass, nodeClaim, instanceTypes) if err != nil { + // Currently, CSE errors will lead to here if cleanupErr := p.cleanupAzureResources(ctx, GenerateResourceName(nodeClaim.Name)); cleanupErr != nil { logging.FromContext(ctx).Errorf("failed to cleanup resources for node claim %s, %w", nodeClaim.Name, cleanupErr) } @@ -206,6 +209,19 @@ func (p *DefaultProvider) createAKSIdentifyingExtension(ctx context.Context, vmN return nil } +func (p *DefaultProvider) createCSExtension(ctx context.Context, vmName string, cse string, isWindows bool) (err error) { + vmExt := p.getCSExtension(cse, isWindows) + vmExtName := *vmExt.Name + logging.FromContext(ctx).Debugf("Creating virtual machine CSE for %s", vmName) + v, err := createVirtualMachineExtension(ctx, p.azClient.virtualMachinesExtensionClient, p.resourceGroup, vmName, vmExtName, *vmExt) + if err != nil { + logging.FromContext(ctx).Errorf("Creating VM CSE for VM %q failed, %w", vmName, err) + return fmt.Errorf("creating VM CSE for VM %q, %w failed", vmName, err) + } + logging.FromContext(ctx).Debugf("Created virtual machine CSE for %s, with an id of %s", vmName, *v.ID) + return nil +} + func (p *DefaultProvider) newNetworkInterfaceForVM(opts *createNICOptions) armnetwork.Interface { var ipv4BackendPools []*armnetwork.BackendAddressPool for _, poolID := range opts.BackendPools.IPv4PoolIDs { @@ -304,11 +320,17 @@ func newVMObject( nodeIdentities []string, nodeClass *v1alpha2.AKSNodeClass, launchTemplate *launchtemplate.Template, - instanceType *corecloudprovider.InstanceType) armcompute.VirtualMachine { + instanceType *corecloudprovider.InstanceType, + provisionMode string) armcompute.VirtualMachine { // Build the image reference from template imageReference := armcompute.ImageReference{ CommunityGalleryImageID: &launchTemplate.ImageID, } + + if launchTemplate.IsWindows { + return armcompute.VirtualMachine{} // TODO(Windows) + } + vm := armcompute.VirtualMachine{ Location: lo.ToPtr(location), Identity: ConvertToVirtualMachineIdentity(nodeIdentities), @@ -353,7 +375,6 @@ func newVMObject( }, }, }, - CustomData: lo.ToPtr(launchTemplate.UserData), }, Priority: lo.ToPtr(armcompute.VirtualMachinePriorityTypes( CapacityTypeToPriority[capacityType]), @@ -362,16 +383,21 @@ func newVMObject( Zones: lo.Ternary(len(zone) > 0, []*string{&zone}, []*string{}), Tags: launchTemplate.Tags, } - setVMPropertiesStorageProfile(vm.Properties, instanceType, nodeClass) + setVMPropertiesOSDiskType(vm.Properties, launchTemplate.StorageProfile) setVMPropertiesBillingProfile(vm.Properties, capacityType) + if provisionMode == consts.ProvisionModeBootstrappingClient { + vm.Properties.OSProfile.CustomData = lo.ToPtr(launchTemplate.CustomScriptsCustomData) + } else { + vm.Properties.OSProfile.CustomData = lo.ToPtr(launchTemplate.ScriptlessCustomData) + } + return vm } -// setVMPropertiesStorageProfile enables ephemeral os disk for instance types that support it -func setVMPropertiesStorageProfile(vmProperties *armcompute.VirtualMachineProperties, instanceType *corecloudprovider.InstanceType, nodeClass *v1alpha2.AKSNodeClass) { - // use ephemeral disk if it is large enough - if *nodeClass.Spec.OSDiskSizeGB <= getEphemeralMaxSizeGB(instanceType) { +// setVMPropertiesOSDiskType enables ephemeral os disk for instance types that support it +func setVMPropertiesOSDiskType(vmProperties *armcompute.VirtualMachineProperties, storageProfile string) { + if storageProfile == "Ephemeral" { vmProperties.StorageProfile.OSDisk.DiffDiskSettings = &armcompute.DiffDiskSettings{ Option: lo.ToPtr(armcompute.DiffDiskOptionsLocal), // placement (cache/resource) is left to CRP @@ -445,7 +471,7 @@ func (p *DefaultProvider) launchInstance( sshPublicKey := options.FromContext(ctx).SSHPublicKey nodeIdentityIDs := options.FromContext(ctx).NodeIdentities - vm := newVMObject(resourceName, nicReference, zone, capacityType, p.location, sshPublicKey, nodeIdentityIDs, nodeClass, launchTemplate, instanceType) + vm := newVMObject(resourceName, nicReference, zone, capacityType, p.location, sshPublicKey, nodeIdentityIDs, nodeClass, launchTemplate, instanceType, p.provisionMode) logging.FromContext(ctx).Debugf("Creating virtual machine %s (%s)", resourceName, instanceType.Name) // Uses AZ Client to create a new virtual machine using the vm object we prepared earlier @@ -455,6 +481,14 @@ func (p *DefaultProvider) launchInstance( return nil, nil, azErr } + if p.provisionMode == consts.ProvisionModeBootstrappingClient { + err = p.createCSExtension(ctx, resourceName, launchTemplate.CustomScriptsCSE, launchTemplate.IsWindows) + if err != nil { + // This should fall back to cleanupAzureResources + return nil, nil, err + } + } + err = p.createAKSIdentifyingExtension(ctx, resourceName) if err != nil { return nil, nil, err @@ -527,19 +561,6 @@ func (p *DefaultProvider) handleResponseErrors(ctx context.Context, instanceType return err } -func getEphemeralMaxSizeGB(instanceType *corecloudprovider.InstanceType) int32 { - reqs := instanceType.Requirements.Get(v1alpha2.LabelSKUStorageEphemeralOSMaxSize).Values() - if len(reqs) == 0 || len(reqs) > 1 { - return 0 - } - maxSize, err := strconv.ParseFloat(reqs[0], 32) - if err != nil { - return 0 - } - // decimal places are truncated, so we round down - return int32(maxSize) -} - func cpuLimitIsZero(err error) bool { return strings.Contains(err.Error(), "Current Limit: 0") } @@ -691,6 +712,36 @@ func (p *DefaultProvider) getAKSIdentifyingExtension() *armcompute.VirtualMachin return vmExtension } +func (p *DefaultProvider) getCSExtension(cse string, isWindows bool) *armcompute.VirtualMachineExtension { + const ( + vmExtensionType = "Microsoft.Compute/virtualMachines/extensions" + cseNameWindows = "windows-cse-agent-karpenter" + cseTypeWindows = "CustomScriptExtension" + csePublisherWindows = "Microsoft.Compute" + cseVersionWindows = "1.10" + cseNameLinux = "cse-agent-karpenter" + cseTypeLinux = "CustomScript" + csePublisherLinux = "Microsoft.Azure.Extensions" + cseVersionLinux = "2.0" + ) + + return &armcompute.VirtualMachineExtension{ + Location: lo.ToPtr(p.location), + Name: lo.ToPtr(lo.Ternary(isWindows, cseNameWindows, cseNameLinux)), + Type: lo.ToPtr(vmExtensionType), + Properties: &armcompute.VirtualMachineExtensionProperties{ + AutoUpgradeMinorVersion: lo.ToPtr(true), + Type: lo.ToPtr(lo.Ternary(isWindows, cseTypeWindows, cseTypeLinux)), + Publisher: lo.ToPtr(lo.Ternary(isWindows, csePublisherWindows, csePublisherLinux)), + TypeHandlerVersion: lo.ToPtr(lo.Ternary(isWindows, cseVersionWindows, cseVersionLinux)), + Settings: &map[string]interface{}{}, + ProtectedSettings: &map[string]interface{}{ + "commandToExecute": cse, + }, + }, + } +} + // GetZoneID returns the zone ID for the given virtual machine, or an empty string if there is no zone specified func GetZoneID(vm *armcompute.VirtualMachine) (string, error) { if vm == nil { diff --git a/pkg/providers/instance/instance_test.go b/pkg/providers/instance/instance_test.go index afacb0fa3..67ee915f3 100644 --- a/pkg/providers/instance/instance_test.go +++ b/pkg/providers/instance/instance_test.go @@ -87,6 +87,7 @@ func TestGetPriorityCapacityAndInstanceType(t *testing.T) { "westus-2", "MC_xxxxx_yyyy-region", "0000000-0000-0000-0000-0000000000", + "", ) for _, c := range cases { t.Run(c.name, func(t *testing.T) { diff --git a/pkg/providers/instancetype/instancetype.go b/pkg/providers/instancetype/instancetype.go index 920a0321c..f34dce1ee 100644 --- a/pkg/providers/instancetype/instancetype.go +++ b/pkg/providers/instancetype/instancetype.go @@ -264,7 +264,7 @@ func getArchitecture(architecture string) string { func computeCapacity(ctx context.Context, sku *skewer.SKU, nodeClass *v1alpha2.AKSNodeClass) corev1.ResourceList { return corev1.ResourceList{ corev1.ResourceCPU: *cpu(sku), - corev1.ResourceMemory: *memory(ctx, sku), + corev1.ResourceMemory: *memoryWithoutOverhead(ctx, sku), corev1.ResourceEphemeralStorage: *ephemeralStorage(nodeClass), corev1.ResourcePods: *pods(nodeClass), corev1.ResourceName("nvidia.com/gpu"): *gpuNvidiaCount(sku), @@ -296,11 +296,15 @@ func memoryMiB(sku *skewer.SKU) int64 { return int64(memoryGiB(sku) * 1024) } -func memory(ctx context.Context, sku *skewer.SKU) *resource.Quantity { - memory := resources.Quantity(fmt.Sprintf("%dGi", int64(memoryGiB(sku)))) - // Account for VM overhead in calculation - memory.Sub(resource.MustParse(fmt.Sprintf("%dMi", int64(math.Ceil( - float64(memory.Value())*options.FromContext(ctx).VMMemoryOverheadPercent/1024/1024))))) +func memoryWithoutOverhead(ctx context.Context, sku *skewer.SKU) *resource.Quantity { + return CalculateMemoryWithoutOverhead(options.FromContext(ctx).VMMemoryOverheadPercent, memoryGiB(sku)) +} + +func CalculateMemoryWithoutOverhead(vmMemoryOverheadPercent float64, skuMemoryGiB float64) *resource.Quantity { + // Consistency in abstractions could be improved here (e.g., units, returning types) + memory := resources.Quantity(fmt.Sprintf("%dGi", int64(skuMemoryGiB))) + memory.Sub(*resource.NewQuantity(int64(math.Ceil( + float64(memory.Value())*vmMemoryOverheadPercent)), resource.DecimalSI)) return memory } diff --git a/pkg/providers/launchtemplate/launchtemplate.go b/pkg/providers/launchtemplate/launchtemplate.go index ae9a72054..9fed3b89d 100644 --- a/pkg/providers/launchtemplate/launchtemplate.go +++ b/pkg/providers/launchtemplate/launchtemplate.go @@ -45,10 +45,14 @@ const ( ) type Template struct { - UserData string - ImageID string - SubnetID string - Tags map[string]*string + ScriptlessCustomData string + ImageID string + SubnetID string + Tags map[string]*string + CustomScriptsCustomData string + CustomScriptsCSE string + IsWindows bool + StorageProfile string } type Provider struct { @@ -60,14 +64,16 @@ type Provider struct { subscriptionID string kubeletIdentityClientID string resourceGroup string + clusterResourceGroup string location string vnetGUID string + provisionMode string } // TODO: add caching of launch templates func NewProvider(_ context.Context, imageFamily *imagefamily.Resolver, imageProvider *imagefamily.Provider, caBundle *string, clusterEndpoint string, - tenantID, subscriptionID, kubeletIdentityClientID, resourceGroup, location, vnetGUID string, + tenantID, subscriptionID, clusterResourceGroup string, kubeletIdentityClientID, resourceGroup, location, vnetGUID, provisionMode string, ) *Provider { return &Provider{ imageFamily: imageFamily, @@ -78,8 +84,10 @@ func NewProvider(_ context.Context, imageFamily *imagefamily.Resolver, imageProv subscriptionID: subscriptionID, kubeletIdentityClientID: kubeletIdentityClientID, resourceGroup: resourceGroup, + clusterResourceGroup: clusterResourceGroup, location: location, vnetGUID: vnetGUID, + provisionMode: provisionMode, } } @@ -99,7 +107,7 @@ func (p *Provider) GetTemplate(ctx context.Context, nodeClass *v1alpha2.AKSNodeC if err != nil { return nil, err } - launchTemplate, err := p.createLaunchTemplate(templateParameters) + launchTemplate, err := p.createLaunchTemplate(ctx, templateParameters) if err != nil { return nil, err } @@ -156,24 +164,37 @@ func (p *Provider) getStaticParameters(ctx context.Context, instanceType *cloudp NetworkPlugin: options.FromContext(ctx).NetworkPlugin, NetworkPolicy: options.FromContext(ctx).NetworkPolicy, SubnetID: subnetID, + ClusterResourceGroup: p.clusterResourceGroup, }, nil } -func (p *Provider) createLaunchTemplate(options *parameters.Parameters) (*Template, error) { - // render user data - userData, err := options.UserData.Script() - if err != nil { - return nil, err - } - +func (p *Provider) createLaunchTemplate(ctx context.Context, params *parameters.Parameters) (*Template, error) { // merge and convert to ARM tags - azureTags := mergeTags(options.Tags, map[string]string{karpenterManagedTagKey: options.ClusterName}) + azureTags := mergeTags(params.Tags, map[string]string{karpenterManagedTagKey: params.ClusterName}) template := &Template{ - UserData: userData, - ImageID: options.ImageID, - Tags: azureTags, - SubnetID: options.SubnetID, + ImageID: params.ImageID, + Tags: azureTags, + SubnetID: params.SubnetID, + IsWindows: params.IsWindows, + StorageProfile: params.StorageProfile, } + + if p.provisionMode == consts.ProvisionModeBootstrappingClient { + customData, cse, err := params.CustomScriptsNodeBootstrapping.GetCustomDataAndCSE(ctx) + if err != nil { + return nil, err + } + template.CustomScriptsCustomData = customData + template.CustomScriptsCSE = cse + } else { + // render user data + userData, err := params.ScriptlessCustomData.Script() + if err != nil { + return nil, err + } + template.ScriptlessCustomData = userData + } + return template, nil } diff --git a/pkg/providers/launchtemplate/parameters/types.go b/pkg/providers/launchtemplate/parameters/types.go index 9ccbd8cb3..8f2bda9f4 100644 --- a/pkg/providers/launchtemplate/parameters/types.go +++ b/pkg/providers/launchtemplate/parameters/types.go @@ -18,6 +18,7 @@ package parameters import ( "github.com/Azure/karpenter-provider-azure/pkg/providers/imagefamily/bootstrap" + "github.com/Azure/karpenter-provider-azure/pkg/providers/imagefamily/customscriptsbootstrap" ) // StaticParameters define the static launch template parameters @@ -41,6 +42,7 @@ type StaticParameters struct { NetworkPolicy string KubernetesVersion string SubnetID string + ClusterResourceGroup string Tags map[string]string Labels map[string]string @@ -49,6 +51,9 @@ type StaticParameters struct { // Parameters adds the dynamically generated launch template parameters type Parameters struct { *StaticParameters - UserData bootstrap.Bootstrapper - ImageID string + ScriptlessCustomData bootstrap.Bootstrapper + CustomScriptsNodeBootstrapping customscriptsbootstrap.Bootstrapper + ImageID string + StorageProfile string + IsWindows bool } diff --git a/pkg/provisionclients/client/node_bootstrapping_client.go b/pkg/provisionclients/client/node_bootstrapping_client.go new file mode 100644 index 000000000..2bd257a8b --- /dev/null +++ b/pkg/provisionclients/client/node_bootstrapping_client.go @@ -0,0 +1,126 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by go-swagger; DO NOT EDIT. + +package client + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "github.com/Azure/karpenter-provider-azure/pkg/provisionclients/client/operations" + "github.com/go-openapi/runtime" + httptransport "github.com/go-openapi/runtime/client" + "github.com/go-openapi/strfmt" +) + +// Default node bootstrapping HTTP client. +var Default = NewHTTPClient(nil) + +const ( + // DefaultHost is the default Host + // found in Meta (info) section of spec file + DefaultHost string = "localhost" + // DefaultBasePath is the default BasePath + // found in Meta (info) section of spec file + DefaultBasePath string = "/" +) + +// DefaultSchemes are the default schemes found in Meta (info) section of spec file +var DefaultSchemes = []string{"https"} + +// NewHTTPClient creates a new node bootstrapping HTTP client. +func NewHTTPClient(formats strfmt.Registry) *NodeBootstrapping { + return NewHTTPClientWithConfig(formats, nil) +} + +// NewHTTPClientWithConfig creates a new node bootstrapping HTTP client, +// using a customizable transport config. +func NewHTTPClientWithConfig(formats strfmt.Registry, cfg *TransportConfig) *NodeBootstrapping { + // ensure nullable parameters have default + if cfg == nil { + cfg = DefaultTransportConfig() + } + + // create transport and client + transport := httptransport.New(cfg.Host, cfg.BasePath, cfg.Schemes) + return New(transport, formats) +} + +// New creates a new node bootstrapping client +func New(transport runtime.ClientTransport, formats strfmt.Registry) *NodeBootstrapping { + // ensure nullable parameters have default + if formats == nil { + formats = strfmt.Default + } + + cli := new(NodeBootstrapping) + cli.Transport = transport + cli.Operations = operations.New(transport, formats) + return cli +} + +// DefaultTransportConfig creates a TransportConfig with the +// default settings taken from the meta section of the spec file. +func DefaultTransportConfig() *TransportConfig { + return &TransportConfig{ + Host: DefaultHost, + BasePath: DefaultBasePath, + Schemes: DefaultSchemes, + } +} + +// TransportConfig contains the transport related info, +// found in the meta section of the spec file. +type TransportConfig struct { + Host string + BasePath string + Schemes []string +} + +// WithHost overrides the default host, +// provided by the meta section of the spec file. +func (cfg *TransportConfig) WithHost(host string) *TransportConfig { + cfg.Host = host + return cfg +} + +// WithBasePath overrides the default basePath, +// provided by the meta section of the spec file. +func (cfg *TransportConfig) WithBasePath(basePath string) *TransportConfig { + cfg.BasePath = basePath + return cfg +} + +// WithSchemes overrides the default schemes, +// provided by the meta section of the spec file. +func (cfg *TransportConfig) WithSchemes(schemes []string) *TransportConfig { + cfg.Schemes = schemes + return cfg +} + +// NodeBootstrapping is a client for node bootstrapping +type NodeBootstrapping struct { + Operations operations.ClientService + + Transport runtime.ClientTransport +} + +// SetTransport changes the transport on the client and all its subresources +func (c *NodeBootstrapping) SetTransport(transport runtime.ClientTransport) { + c.Transport = transport + c.Operations.SetTransport(transport) +} diff --git a/pkg/provisionclients/client/operations/node_bootstrapping_get_parameters.go b/pkg/provisionclients/client/operations/node_bootstrapping_get_parameters.go new file mode 100644 index 000000000..72168cc4a --- /dev/null +++ b/pkg/provisionclients/client/operations/node_bootstrapping_get_parameters.go @@ -0,0 +1,225 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by go-swagger; DO NOT EDIT. + +package operations + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "context" + "net/http" + "time" + + "github.com/go-openapi/errors" + "github.com/go-openapi/runtime" + cr "github.com/go-openapi/runtime/client" + "github.com/go-openapi/strfmt" + + "github.com/Azure/karpenter-provider-azure/pkg/provisionclients/models" +) + +// NewNodeBootstrappingGetParams creates a new NodeBootstrappingGetParams object, +// with the default timeout for this client. +// +// Default values are not hydrated, since defaults are normally applied by the API server side. +// +// To enforce default values in parameter, use SetDefaults or WithDefaults. +func NewNodeBootstrappingGetParams() *NodeBootstrappingGetParams { + return &NodeBootstrappingGetParams{ + timeout: cr.DefaultTimeout, + } +} + +// NewNodeBootstrappingGetParamsWithTimeout creates a new NodeBootstrappingGetParams object +// with the ability to set a timeout on a request. +func NewNodeBootstrappingGetParamsWithTimeout(timeout time.Duration) *NodeBootstrappingGetParams { + return &NodeBootstrappingGetParams{ + timeout: timeout, + } +} + +// NewNodeBootstrappingGetParamsWithContext creates a new NodeBootstrappingGetParams object +// with the ability to set a context for a request. +func NewNodeBootstrappingGetParamsWithContext(ctx context.Context) *NodeBootstrappingGetParams { + return &NodeBootstrappingGetParams{ + Context: ctx, + } +} + +// NewNodeBootstrappingGetParamsWithHTTPClient creates a new NodeBootstrappingGetParams object +// with the ability to set a custom HTTPClient for a request. +func NewNodeBootstrappingGetParamsWithHTTPClient(client *http.Client) *NodeBootstrappingGetParams { + return &NodeBootstrappingGetParams{ + HTTPClient: client, + } +} + +/* +NodeBootstrappingGetParams contains all the parameters to send to the API endpoint + + for the node bootstrapping get operation. + + Typically these are written to a http.Request. +*/ +type NodeBootstrappingGetParams struct { + + /* Parameters. + + Values required to determine NodeBootstrapping values. + */ + Parameters *models.ProvisionValues + + // ResourceGroupName. + ResourceGroupName string + + // ResourceName. + ResourceName string + + // SubscriptionID. + SubscriptionID string + + timeout time.Duration + Context context.Context + HTTPClient *http.Client +} + +// WithDefaults hydrates default values in the node bootstrapping get params (not the query body). +// +// All values with no default are reset to their zero value. +func (o *NodeBootstrappingGetParams) WithDefaults() *NodeBootstrappingGetParams { + o.SetDefaults() + return o +} + +// SetDefaults hydrates default values in the node bootstrapping get params (not the query body). +// +// All values with no default are reset to their zero value. +func (o *NodeBootstrappingGetParams) SetDefaults() { + // no default values defined for this parameter +} + +// WithTimeout adds the timeout to the node bootstrapping get params +func (o *NodeBootstrappingGetParams) WithTimeout(timeout time.Duration) *NodeBootstrappingGetParams { + o.SetTimeout(timeout) + return o +} + +// SetTimeout adds the timeout to the node bootstrapping get params +func (o *NodeBootstrappingGetParams) SetTimeout(timeout time.Duration) { + o.timeout = timeout +} + +// WithContext adds the context to the node bootstrapping get params +func (o *NodeBootstrappingGetParams) WithContext(ctx context.Context) *NodeBootstrappingGetParams { + o.SetContext(ctx) + return o +} + +// SetContext adds the context to the node bootstrapping get params +func (o *NodeBootstrappingGetParams) SetContext(ctx context.Context) { + o.Context = ctx +} + +// WithHTTPClient adds the HTTPClient to the node bootstrapping get params +func (o *NodeBootstrappingGetParams) WithHTTPClient(client *http.Client) *NodeBootstrappingGetParams { + o.SetHTTPClient(client) + return o +} + +// SetHTTPClient adds the HTTPClient to the node bootstrapping get params +func (o *NodeBootstrappingGetParams) SetHTTPClient(client *http.Client) { + o.HTTPClient = client +} + +// WithParameters adds the parameters to the node bootstrapping get params +func (o *NodeBootstrappingGetParams) WithParameters(parameters *models.ProvisionValues) *NodeBootstrappingGetParams { + o.SetParameters(parameters) + return o +} + +// SetParameters adds the parameters to the node bootstrapping get params +func (o *NodeBootstrappingGetParams) SetParameters(parameters *models.ProvisionValues) { + o.Parameters = parameters +} + +// WithResourceGroupName adds the resourceGroupName to the node bootstrapping get params +func (o *NodeBootstrappingGetParams) WithResourceGroupName(resourceGroupName string) *NodeBootstrappingGetParams { + o.SetResourceGroupName(resourceGroupName) + return o +} + +// SetResourceGroupName adds the resourceGroupName to the node bootstrapping get params +func (o *NodeBootstrappingGetParams) SetResourceGroupName(resourceGroupName string) { + o.ResourceGroupName = resourceGroupName +} + +// WithResourceName adds the resourceName to the node bootstrapping get params +func (o *NodeBootstrappingGetParams) WithResourceName(resourceName string) *NodeBootstrappingGetParams { + o.SetResourceName(resourceName) + return o +} + +// SetResourceName adds the resourceName to the node bootstrapping get params +func (o *NodeBootstrappingGetParams) SetResourceName(resourceName string) { + o.ResourceName = resourceName +} + +// WithSubscriptionID adds the subscriptionID to the node bootstrapping get params +func (o *NodeBootstrappingGetParams) WithSubscriptionID(subscriptionID string) *NodeBootstrappingGetParams { + o.SetSubscriptionID(subscriptionID) + return o +} + +// SetSubscriptionID adds the subscriptionId to the node bootstrapping get params +func (o *NodeBootstrappingGetParams) SetSubscriptionID(subscriptionID string) { + o.SubscriptionID = subscriptionID +} + +// WriteToRequest writes these params to a swagger request +func (o *NodeBootstrappingGetParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { + + if err := r.SetTimeout(o.timeout); err != nil { + return err + } + var res []error + if o.Parameters != nil { + if err := r.SetBodyParam(o.Parameters); err != nil { + return err + } + } + + // path param resourceGroupName + if err := r.SetPathParam("resourceGroupName", o.ResourceGroupName); err != nil { + return err + } + + // path param resourceName + if err := r.SetPathParam("resourceName", o.ResourceName); err != nil { + return err + } + + // path param subscriptionId + if err := r.SetPathParam("subscriptionId", o.SubscriptionID); err != nil { + return err + } + + if len(res) > 0 { + return errors.CompositeValidationError(res...) + } + return nil +} diff --git a/pkg/provisionclients/client/operations/node_bootstrapping_get_responses.go b/pkg/provisionclients/client/operations/node_bootstrapping_get_responses.go new file mode 100644 index 000000000..86e75f225 --- /dev/null +++ b/pkg/provisionclients/client/operations/node_bootstrapping_get_responses.go @@ -0,0 +1,191 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by go-swagger; DO NOT EDIT. + +package operations + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "encoding/json" + "fmt" + "io" + + "github.com/go-openapi/runtime" + "github.com/go-openapi/strfmt" + + "github.com/Azure/karpenter-provider-azure/pkg/provisionclients/models" +) + +// NodeBootstrappingGetReader is a Reader for the NodeBootstrappingGet structure. +type NodeBootstrappingGetReader struct { + formats strfmt.Registry +} + +// ReadResponse reads a server response into the received o. +func (o *NodeBootstrappingGetReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { + switch response.Code() { + case 200: + result := NewNodeBootstrappingGetOK() + if err := result.readResponse(response, consumer, o.formats); err != nil { + return nil, err + } + return result, nil + default: + result := NewNodeBootstrappingGetDefault(response.Code()) + if err := result.readResponse(response, consumer, o.formats); err != nil { + return nil, err + } + if response.Code()/100 == 2 { + return result, nil + } + return nil, result + } +} + +// NewNodeBootstrappingGetOK creates a NodeBootstrappingGetOK with default headers values +func NewNodeBootstrappingGetOK() *NodeBootstrappingGetOK { + return &NodeBootstrappingGetOK{} +} + +/* +NodeBootstrappingGetOK describes a response with status code 200, with default header values. + +OK +*/ +type NodeBootstrappingGetOK struct { + Payload *models.NodeBootstrapping +} + +// IsSuccess returns true when this node bootstrapping get o k response has a 2xx status code +func (o *NodeBootstrappingGetOK) IsSuccess() bool { + return true +} + +// IsRedirect returns true when this node bootstrapping get o k response has a 3xx status code +func (o *NodeBootstrappingGetOK) IsRedirect() bool { + return false +} + +// IsClientError returns true when this node bootstrapping get o k response has a 4xx status code +func (o *NodeBootstrappingGetOK) IsClientError() bool { + return false +} + +// IsServerError returns true when this node bootstrapping get o k response has a 5xx status code +func (o *NodeBootstrappingGetOK) IsServerError() bool { + return false +} + +// IsCode returns true when this node bootstrapping get o k response a status code equal to that given +func (o *NodeBootstrappingGetOK) IsCode(code int) bool { + return code == 200 +} + +// Code gets the status code for the node bootstrapping get o k response +func (o *NodeBootstrappingGetOK) Code() int { + return 200 +} + +func (o *NodeBootstrappingGetOK) Error() string { + payload, _ := json.Marshal(o.Payload) + return fmt.Sprintf("[GET /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ContainerService/managedclusters/{resourceName}/nodeBootstrapping][%d] nodeBootstrappingGetOK %s", 200, payload) +} + +func (o *NodeBootstrappingGetOK) String() string { + payload, _ := json.Marshal(o.Payload) + return fmt.Sprintf("[GET /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ContainerService/managedclusters/{resourceName}/nodeBootstrapping][%d] nodeBootstrappingGetOK %s", 200, payload) +} + +func (o *NodeBootstrappingGetOK) GetPayload() *models.NodeBootstrapping { + return o.Payload +} + +func (o *NodeBootstrappingGetOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { + + o.Payload = new(models.NodeBootstrapping) + + // response payload + if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { + return err + } + + return nil +} + +// NewNodeBootstrappingGetDefault creates a NodeBootstrappingGetDefault with default headers values +func NewNodeBootstrappingGetDefault(code int) *NodeBootstrappingGetDefault { + return &NodeBootstrappingGetDefault{ + _statusCode: code, + } +} + +/* +NodeBootstrappingGetDefault describes a response with status code -1, with default header values. + +Error response describing why the operation failed. +*/ +type NodeBootstrappingGetDefault struct { + _statusCode int + + Payload string +} + +// IsSuccess returns true when this node bootstrapping get default response has a 2xx status code +func (o *NodeBootstrappingGetDefault) IsSuccess() bool { + return o._statusCode/100 == 2 +} + +// IsRedirect returns true when this node bootstrapping get default response has a 3xx status code +func (o *NodeBootstrappingGetDefault) IsRedirect() bool { + return o._statusCode/100 == 3 +} + +// IsClientError returns true when this node bootstrapping get default response has a 4xx status code +func (o *NodeBootstrappingGetDefault) IsClientError() bool { + return o._statusCode/100 == 4 +} + +// IsServerError returns true when this node bootstrapping get default response has a 5xx status code +func (o *NodeBootstrappingGetDefault) IsServerError() bool { + return o._statusCode/100 == 5 +} + +// IsCode returns true when this node bootstrapping get default response a status code equal to that given +func (o *NodeBootstrappingGetDefault) IsCode(code int) bool { + return o._statusCode == code +} + +// Code gets the status code for the node bootstrapping get default response +func (o *NodeBootstrappingGetDefault) Code() int { + return o._statusCode +} + +func (o *NodeBootstrappingGetDefault) Error() string { + payload, _ := json.Marshal(o.Payload) + return fmt.Sprintf("[GET /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ContainerService/managedclusters/{resourceName}/nodeBootstrapping][%d] NodeBootstrapping_Get default %s", o._statusCode, payload) +} + +func (o *NodeBootstrappingGetDefault) String() string { + payload, _ := json.Marshal(o.Payload) + return fmt.Sprintf("[GET /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ContainerService/managedclusters/{resourceName}/nodeBootstrapping][%d] NodeBootstrapping_Get default %s", o._statusCode, payload) +} + +func (o *NodeBootstrappingGetDefault) GetPayload() string { + return o.Payload +} + diff --git a/pkg/provisionclients/client/operations/node_bootstrapping_get_responses_override.go b/pkg/provisionclients/client/operations/node_bootstrapping_get_responses_override.go new file mode 100644 index 000000000..93d100035 --- /dev/null +++ b/pkg/provisionclients/client/operations/node_bootstrapping_get_responses_override.go @@ -0,0 +1,47 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package operations + +import ( + "io" + + "github.com/go-openapi/runtime" + "github.com/go-openapi/strfmt" +) + +func (o *NodeBootstrappingGetDefault) readResponse(response runtime.ClientResponse, _ runtime.Consumer, _ strfmt.Registry) error { + // response payload + + // THIS IS MODIFIED FROM AUTO-GENERATED. + // There is a known issue on the server side where content type in the header (JSON) doesn't match actual (text). + // This is a work around for that, which should be safe due to the fact that string > JSON. Only affect logging format for now. + // The "consumer" that we replaced only decode into JSON, and unlikely to change. + rc := response.Body() + defer rc.Close() + + bytes, err := io.ReadAll(rc) + if err != nil { + return err + } + o.Payload = string(bytes) + + // if err := consumer.Consume(response.Body(), &o.Payload); err != nil && err != io.EOF { + // return err + // } + + return nil +} diff --git a/pkg/provisionclients/client/operations/operations_client.go b/pkg/provisionclients/client/operations/operations_client.go new file mode 100644 index 000000000..8b2778624 --- /dev/null +++ b/pkg/provisionclients/client/operations/operations_client.go @@ -0,0 +1,119 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by go-swagger; DO NOT EDIT. + +package operations + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "github.com/go-openapi/runtime" + httptransport "github.com/go-openapi/runtime/client" + "github.com/go-openapi/strfmt" +) + +// New creates a new operations API client. +func New(transport runtime.ClientTransport, formats strfmt.Registry) ClientService { + return &Client{transport: transport, formats: formats} +} + +// New creates a new operations API client with basic auth credentials. +// It takes the following parameters: +// - host: http host (github.com). +// - basePath: any base path for the API client ("/v1", "/v3"). +// - scheme: http scheme ("http", "https"). +// - user: user for basic authentication header. +// - password: password for basic authentication header. +func NewClientWithBasicAuth(host, basePath, scheme, user, password string) ClientService { + transport := httptransport.New(host, basePath, []string{scheme}) + transport.DefaultAuthentication = httptransport.BasicAuth(user, password) + return &Client{transport: transport, formats: strfmt.Default} +} + +// New creates a new operations API client with a bearer token for authentication. +// It takes the following parameters: +// - host: http host (github.com). +// - basePath: any base path for the API client ("/v1", "/v3"). +// - scheme: http scheme ("http", "https"). +// - bearerToken: bearer token for Bearer authentication header. +func NewClientWithBearerToken(host, basePath, scheme, bearerToken string) ClientService { + transport := httptransport.New(host, basePath, []string{scheme}) + transport.DefaultAuthentication = httptransport.BearerToken(bearerToken) + return &Client{transport: transport, formats: strfmt.Default} +} + +/* +Client for operations API +*/ +type Client struct { + transport runtime.ClientTransport + formats strfmt.Registry +} + +// ClientOption may be used to customize the behavior of Client methods. +type ClientOption func(*runtime.ClientOperation) + +// ClientService is the interface for Client methods +type ClientService interface { + NodeBootstrappingGet(params *NodeBootstrappingGetParams, opts ...ClientOption) (*NodeBootstrappingGetOK, error) + + SetTransport(transport runtime.ClientTransport) +} + +/* +NodeBootstrappingGet gets node bootstrapping values for a specified provisioning configuration with secrets redacted + +Gets NodeBootstrapping values for a specified provisioning configuration, with secrets redacted. +*/ +func (a *Client) NodeBootstrappingGet(params *NodeBootstrappingGetParams, opts ...ClientOption) (*NodeBootstrappingGetOK, error) { + // TODO: Validate the params before sending + if params == nil { + params = NewNodeBootstrappingGetParams() + } + op := &runtime.ClientOperation{ + ID: "NodeBootstrapping_Get", + Method: "GET", + PathPattern: "/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ContainerService/managedclusters/{resourceName}/nodeBootstrapping", + ProducesMediaTypes: []string{"application/json"}, + ConsumesMediaTypes: []string{"application/json"}, + Schemes: []string{"https"}, + Params: params, + Reader: &NodeBootstrappingGetReader{formats: a.formats}, + Context: params.Context, + Client: params.HTTPClient, + } + for _, opt := range opts { + opt(op) + } + + result, err := a.transport.Submit(op) + if err != nil { + return nil, err + } + success, ok := result.(*NodeBootstrappingGetOK) + if ok { + return success, nil + } + // unexpected success response + unexpectedSuccess := result.(*NodeBootstrappingGetDefault) + return nil, runtime.NewAPIError("unexpected success response: content available as default response in error", unexpectedSuccess, unexpectedSuccess.Code()) +} + +// SetTransport changes the transport on the client +func (a *Client) SetTransport(transport runtime.ClientTransport) { + a.transport = transport +} diff --git a/pkg/provisionclients/models/agent_pool_security_profile.go b/pkg/provisionclients/models/agent_pool_security_profile.go new file mode 100644 index 000000000..1f2ca6c98 --- /dev/null +++ b/pkg/provisionclients/models/agent_pool_security_profile.go @@ -0,0 +1,71 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by go-swagger; DO NOT EDIT. + +package models + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "context" + + "github.com/go-openapi/strfmt" + "github.com/go-openapi/swag" +) + +// AgentPoolSecurityProfile agent pool security profile +// +// swagger:model AgentPoolSecurityProfile +type AgentPoolSecurityProfile struct { + + // enable secure boot + EnableSecureBoot *bool `json:"enableSecureBoot,omitempty"` + + // enable v t p m + EnableVTPM *bool `json:"enableVTPM,omitempty"` + + // ssh access + SSHAccess *int32 `json:"sshAccess,omitempty"` +} + +// Validate validates this agent pool security profile +func (m *AgentPoolSecurityProfile) Validate(formats strfmt.Registry) error { + return nil +} + +// ContextValidate validates this agent pool security profile based on context it is used +func (m *AgentPoolSecurityProfile) ContextValidate(ctx context.Context, formats strfmt.Registry) error { + return nil +} + +// MarshalBinary interface implementation +func (m *AgentPoolSecurityProfile) MarshalBinary() ([]byte, error) { + if m == nil { + return nil, nil + } + return swag.WriteJSON(m) +} + +// UnmarshalBinary interface implementation +func (m *AgentPoolSecurityProfile) UnmarshalBinary(b []byte) error { + var res AgentPoolSecurityProfile + if err := swag.ReadJSON(b, &res); err != nil { + return err + } + *m = res + return nil +} diff --git a/pkg/provisionclients/models/agent_pool_windows_profile.go b/pkg/provisionclients/models/agent_pool_windows_profile.go new file mode 100644 index 000000000..f7720daeb --- /dev/null +++ b/pkg/provisionclients/models/agent_pool_windows_profile.go @@ -0,0 +1,71 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by go-swagger; DO NOT EDIT. + +package models + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "context" + + "github.com/go-openapi/strfmt" + "github.com/go-openapi/swag" +) + +// AgentPoolWindowsProfile agent pool windows profile +// +// swagger:model AgentPoolWindowsProfile +type AgentPoolWindowsProfile struct { + + // containerd package + ContainerdPackage *string `json:"containerdPackage,omitempty"` + + // disable outbound nat + DisableOutboundNat *bool `json:"disableOutboundNat,omitempty"` + + // enable next gen networking + EnableNextGenNetworking *bool `json:"enableNextGenNetworking,omitempty"` +} + +// Validate validates this agent pool windows profile +func (m *AgentPoolWindowsProfile) Validate(formats strfmt.Registry) error { + return nil +} + +// ContextValidate validates this agent pool windows profile based on context it is used +func (m *AgentPoolWindowsProfile) ContextValidate(ctx context.Context, formats strfmt.Registry) error { + return nil +} + +// MarshalBinary interface implementation +func (m *AgentPoolWindowsProfile) MarshalBinary() ([]byte, error) { + if m == nil { + return nil, nil + } + return swag.WriteJSON(m) +} + +// UnmarshalBinary interface implementation +func (m *AgentPoolWindowsProfile) UnmarshalBinary(b []byte) error { + var res AgentPoolWindowsProfile + if err := swag.ReadJSON(b, &res); err != nil { + return err + } + *m = res + return nil +} diff --git a/pkg/provisionclients/models/artifact_streaming_profile.go b/pkg/provisionclients/models/artifact_streaming_profile.go new file mode 100644 index 000000000..8eb413394 --- /dev/null +++ b/pkg/provisionclients/models/artifact_streaming_profile.go @@ -0,0 +1,65 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by go-swagger; DO NOT EDIT. + +package models + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "context" + + "github.com/go-openapi/strfmt" + "github.com/go-openapi/swag" +) + +// ArtifactStreamingProfile artifact streaming profile +// +// swagger:model ArtifactStreamingProfile +type ArtifactStreamingProfile struct { + + // enabled + Enabled *bool `json:"enabled,omitempty"` +} + +// Validate validates this artifact streaming profile +func (m *ArtifactStreamingProfile) Validate(formats strfmt.Registry) error { + return nil +} + +// ContextValidate validates this artifact streaming profile based on context it is used +func (m *ArtifactStreamingProfile) ContextValidate(ctx context.Context, formats strfmt.Registry) error { + return nil +} + +// MarshalBinary interface implementation +func (m *ArtifactStreamingProfile) MarshalBinary() ([]byte, error) { + if m == nil { + return nil, nil + } + return swag.WriteJSON(m) +} + +// UnmarshalBinary interface implementation +func (m *ArtifactStreamingProfile) UnmarshalBinary(b []byte) error { + var res ArtifactStreamingProfile + if err := swag.ReadJSON(b, &res); err != nil { + return err + } + *m = res + return nil +} diff --git a/pkg/provisionclients/models/azure_o_s_image_config.go b/pkg/provisionclients/models/azure_o_s_image_config.go new file mode 100644 index 000000000..a8ed719bd --- /dev/null +++ b/pkg/provisionclients/models/azure_o_s_image_config.go @@ -0,0 +1,74 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by go-swagger; DO NOT EDIT. + +package models + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "context" + + "github.com/go-openapi/strfmt" + "github.com/go-openapi/swag" +) + +// AzureOSImageConfig azure o s image config +// +// swagger:model AzureOSImageConfig +type AzureOSImageConfig struct { + + // image offer + ImageOffer *string `json:"imageOffer,omitempty"` + + // image publisher + ImagePublisher *string `json:"imagePublisher,omitempty"` + + // image sku + ImageSku *string `json:"imageSku,omitempty"` + + // image version + ImageVersion *string `json:"imageVersion,omitempty"` +} + +// Validate validates this azure o s image config +func (m *AzureOSImageConfig) Validate(formats strfmt.Registry) error { + return nil +} + +// ContextValidate validates this azure o s image config based on context it is used +func (m *AzureOSImageConfig) ContextValidate(ctx context.Context, formats strfmt.Registry) error { + return nil +} + +// MarshalBinary interface implementation +func (m *AzureOSImageConfig) MarshalBinary() ([]byte, error) { + if m == nil { + return nil, nil + } + return swag.WriteJSON(m) +} + +// UnmarshalBinary interface implementation +func (m *AzureOSImageConfig) UnmarshalBinary(b []byte) error { + var res AzureOSImageConfig + if err := swag.ReadJSON(b, &res); err != nil { + return err + } + *m = res + return nil +} diff --git a/pkg/provisionclients/models/custom_kubelet_config.go b/pkg/provisionclients/models/custom_kubelet_config.go new file mode 100644 index 000000000..44dc8c1ef --- /dev/null +++ b/pkg/provisionclients/models/custom_kubelet_config.go @@ -0,0 +1,104 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by go-swagger; DO NOT EDIT. + +package models + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "context" + + "github.com/go-openapi/strfmt" + "github.com/go-openapi/swag" +) + +// CustomKubeletConfig custom kubelet config +// +// swagger:model CustomKubeletConfig +type CustomKubeletConfig struct { + + // allowed unsafe sysctls + AllowedUnsafeSysctls []string `json:"allowedUnsafeSysctls"` + + // container log max files + ContainerLogMaxFiles *int32 `json:"containerLogMaxFiles,omitempty"` + + // container log max size m b + ContainerLogMaxSizeMB *int32 `json:"containerLogMaxSizeMB,omitempty"` + + // cpu cfs quota + CPUCfsQuota *bool `json:"cpuCfsQuota,omitempty"` + + // cpu cfs quota period + CPUCfsQuotaPeriod *string `json:"cpuCfsQuotaPeriod,omitempty"` + + // cpu manager policy + CPUManagerPolicy *string `json:"cpuManagerPolicy,omitempty"` + + // cpu reserved + CPUReserved *int32 `json:"cpuReserved,omitempty"` + + // fail swap on + FailSwapOn *bool `json:"failSwapOn,omitempty"` + + // image gc high threshold + ImageGcHighThreshold *int32 `json:"imageGcHighThreshold,omitempty"` + + // image gc low threshold + ImageGcLowThreshold *int32 `json:"imageGcLowThreshold,omitempty"` + + // memory reserved + MemoryReserved *int32 `json:"memoryReserved,omitempty"` + + // pod max pids + PodMaxPids *int32 `json:"podMaxPids,omitempty"` + + // seccomp default + SeccompDefault *string `json:"seccompDefault,omitempty"` + + // topology manager policy + TopologyManagerPolicy *string `json:"topologyManagerPolicy,omitempty"` +} + +// Validate validates this custom kubelet config +func (m *CustomKubeletConfig) Validate(formats strfmt.Registry) error { + return nil +} + +// ContextValidate validates this custom kubelet config based on context it is used +func (m *CustomKubeletConfig) ContextValidate(ctx context.Context, formats strfmt.Registry) error { + return nil +} + +// MarshalBinary interface implementation +func (m *CustomKubeletConfig) MarshalBinary() ([]byte, error) { + if m == nil { + return nil, nil + } + return swag.WriteJSON(m) +} + +// UnmarshalBinary interface implementation +func (m *CustomKubeletConfig) UnmarshalBinary(b []byte) error { + var res CustomKubeletConfig + if err := swag.ReadJSON(b, &res); err != nil { + return err + } + *m = res + return nil +} diff --git a/pkg/provisionclients/models/custom_linux_o_s_config.go b/pkg/provisionclients/models/custom_linux_o_s_config.go new file mode 100644 index 000000000..57c1bb1ba --- /dev/null +++ b/pkg/provisionclients/models/custom_linux_o_s_config.go @@ -0,0 +1,184 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by go-swagger; DO NOT EDIT. + +package models + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "context" + + "github.com/go-openapi/errors" + "github.com/go-openapi/strfmt" + "github.com/go-openapi/swag" +) + +// CustomLinuxOSConfig custom linux o s config +// +// swagger:model CustomLinuxOSConfig +type CustomLinuxOSConfig struct { + + // swap file size m b + SwapFileSizeMB *int32 `json:"swapFileSizeMB,omitempty"` + + // sysctls + Sysctls *SysctlConfig `json:"sysctls,omitempty"` + + // transparent huge page defrag + TransparentHugePageDefrag *string `json:"transparentHugePageDefrag,omitempty"` + + // transparent huge page enabled + TransparentHugePageEnabled *string `json:"transparentHugePageEnabled,omitempty"` + + // ulimits + Ulimits *UlimitConfig `json:"ulimits,omitempty"` +} + +// Validate validates this custom linux o s config +func (m *CustomLinuxOSConfig) Validate(formats strfmt.Registry) error { + var res []error + + if err := m.validateSysctls(formats); err != nil { + res = append(res, err) + } + + if err := m.validateUlimits(formats); err != nil { + res = append(res, err) + } + + if len(res) > 0 { + return errors.CompositeValidationError(res...) + } + return nil +} + +func (m *CustomLinuxOSConfig) validateSysctls(formats strfmt.Registry) error { + if swag.IsZero(m.Sysctls) { // not required + return nil + } + + if m.Sysctls != nil { + if err := m.Sysctls.Validate(formats); err != nil { + if ve, ok := err.(*errors.Validation); ok { + return ve.ValidateName("sysctls") + } else if ce, ok := err.(*errors.CompositeError); ok { + return ce.ValidateName("sysctls") + } + return err + } + } + + return nil +} + +func (m *CustomLinuxOSConfig) validateUlimits(formats strfmt.Registry) error { + if swag.IsZero(m.Ulimits) { // not required + return nil + } + + if m.Ulimits != nil { + if err := m.Ulimits.Validate(formats); err != nil { + if ve, ok := err.(*errors.Validation); ok { + return ve.ValidateName("ulimits") + } else if ce, ok := err.(*errors.CompositeError); ok { + return ce.ValidateName("ulimits") + } + return err + } + } + + return nil +} + +// ContextValidate validate this custom linux o s config based on the context it is used +func (m *CustomLinuxOSConfig) ContextValidate(ctx context.Context, formats strfmt.Registry) error { + var res []error + + if err := m.contextValidateSysctls(ctx, formats); err != nil { + res = append(res, err) + } + + if err := m.contextValidateUlimits(ctx, formats); err != nil { + res = append(res, err) + } + + if len(res) > 0 { + return errors.CompositeValidationError(res...) + } + return nil +} + +func (m *CustomLinuxOSConfig) contextValidateSysctls(ctx context.Context, formats strfmt.Registry) error { + + if m.Sysctls != nil { + + if swag.IsZero(m.Sysctls) { // not required + return nil + } + + if err := m.Sysctls.ContextValidate(ctx, formats); err != nil { + if ve, ok := err.(*errors.Validation); ok { + return ve.ValidateName("sysctls") + } else if ce, ok := err.(*errors.CompositeError); ok { + return ce.ValidateName("sysctls") + } + return err + } + } + + return nil +} + +func (m *CustomLinuxOSConfig) contextValidateUlimits(ctx context.Context, formats strfmt.Registry) error { + + if m.Ulimits != nil { + + if swag.IsZero(m.Ulimits) { // not required + return nil + } + + if err := m.Ulimits.ContextValidate(ctx, formats); err != nil { + if ve, ok := err.(*errors.Validation); ok { + return ve.ValidateName("ulimits") + } else if ce, ok := err.(*errors.CompositeError); ok { + return ce.ValidateName("ulimits") + } + return err + } + } + + return nil +} + +// MarshalBinary interface implementation +func (m *CustomLinuxOSConfig) MarshalBinary() ([]byte, error) { + if m == nil { + return nil, nil + } + return swag.WriteJSON(m) +} + +// UnmarshalBinary interface implementation +func (m *CustomLinuxOSConfig) UnmarshalBinary(b []byte) error { + var res CustomLinuxOSConfig + if err := swag.ReadJSON(b, &res); err != nil { + return err + } + *m = res + return nil +} diff --git a/pkg/provisionclients/models/enums.go b/pkg/provisionclients/models/enums.go new file mode 100644 index 000000000..35a61ae5c --- /dev/null +++ b/pkg/provisionclients/models/enums.go @@ -0,0 +1,65 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This is NOT auto-generated. + +package models + +const ( + OSTypeUnspecified int32 = 0 + OSTypeWindows int32 = 1 + OSTypeLinux int32 = 2 +) + +const ( + OSSKUUnspecified int32 = 0 + OSSKUUbuntu int32 = 1 + OSSKUAzureLinux int32 = 7 + OSSKUWindows2019 int32 = 3 + OSSKUWindows2022 int32 = 4 + OSSKUWindowsAnnual int32 = 8 +) + +const ( + KubeletDiskTypeUnspecified int32 = 0 + KubeletDiskTypeOS int32 = 1 + KubeletDiskTypeTemporary int32 = 2 +) + +const ( + AgentPoolModeUnspecified int32 = 0 + AgentPoolModeSystem int32 = 1 + AgentPoolModeUser int32 = 2 +) + +const ( + GPUInstanceProfileUnspecified int32 = 0 +) + +const ( + WorkloadRuntimeUnspecified int32 = 0 +) + +const ( + SSHAccessLocalUser int32 = 0 + SSHAccessDisabled int32 = 1 +) + +const ( + DriverTypeUnspecified int32 = 0 + DriverTypeGRID int32 = 2 + DriverTypeCUDA int32 = 3 +) diff --git a/pkg/provisionclients/models/g_p_u_profile.go b/pkg/provisionclients/models/g_p_u_profile.go new file mode 100644 index 000000000..40e1fbe9f --- /dev/null +++ b/pkg/provisionclients/models/g_p_u_profile.go @@ -0,0 +1,68 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by go-swagger; DO NOT EDIT. + +package models + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "context" + + "github.com/go-openapi/strfmt" + "github.com/go-openapi/swag" +) + +// GPUProfile g p u profile +// +// swagger:model GPUProfile +type GPUProfile struct { + + // driver type + DriverType *int32 `json:"driverType,omitempty"` + + // install g p u driver + InstallGPUDriver *bool `json:"installGPUDriver,omitempty"` +} + +// Validate validates this g p u profile +func (m *GPUProfile) Validate(formats strfmt.Registry) error { + return nil +} + +// ContextValidate validates this g p u profile based on context it is used +func (m *GPUProfile) ContextValidate(ctx context.Context, formats strfmt.Registry) error { + return nil +} + +// MarshalBinary interface implementation +func (m *GPUProfile) MarshalBinary() ([]byte, error) { + if m == nil { + return nil, nil + } + return swag.WriteJSON(m) +} + +// UnmarshalBinary interface implementation +func (m *GPUProfile) UnmarshalBinary(b []byte) error { + var res GPUProfile + if err := swag.ReadJSON(b, &res); err != nil { + return err + } + *m = res + return nil +} diff --git a/pkg/provisionclients/models/node_bootstrapping.go b/pkg/provisionclients/models/node_bootstrapping.go new file mode 100644 index 000000000..0fc7dfcff --- /dev/null +++ b/pkg/provisionclients/models/node_bootstrapping.go @@ -0,0 +1,206 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by go-swagger; DO NOT EDIT. + +package models + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "context" + + "github.com/go-openapi/errors" + "github.com/go-openapi/strfmt" + "github.com/go-openapi/swag" + "github.com/go-openapi/validate" +) + +// NodeBootstrapping node bootstrapping +// +// swagger:model NodeBootstrapping +type NodeBootstrapping struct { + + // cse + // Required: true + Cse *string `json:"cse"` + + // custom data + // Required: true + CustomData *string `json:"customData"` + + // os image config + // Required: true + OsImageConfig *AzureOSImageConfig `json:"osImageConfig"` + + // sig image config + // Required: true + SigImageConfig *SigImageConfig `json:"sigImageConfig"` +} + +// Validate validates this node bootstrapping +func (m *NodeBootstrapping) Validate(formats strfmt.Registry) error { + var res []error + + if err := m.validateCse(formats); err != nil { + res = append(res, err) + } + + if err := m.validateCustomData(formats); err != nil { + res = append(res, err) + } + + if err := m.validateOsImageConfig(formats); err != nil { + res = append(res, err) + } + + if err := m.validateSigImageConfig(formats); err != nil { + res = append(res, err) + } + + if len(res) > 0 { + return errors.CompositeValidationError(res...) + } + return nil +} + +func (m *NodeBootstrapping) validateCse(formats strfmt.Registry) error { + + if err := validate.Required("cse", "body", m.Cse); err != nil { + return err + } + + return nil +} + +func (m *NodeBootstrapping) validateCustomData(formats strfmt.Registry) error { + + if err := validate.Required("customData", "body", m.CustomData); err != nil { + return err + } + + return nil +} + +func (m *NodeBootstrapping) validateOsImageConfig(formats strfmt.Registry) error { + + if err := validate.Required("osImageConfig", "body", m.OsImageConfig); err != nil { + return err + } + + if m.OsImageConfig != nil { + if err := m.OsImageConfig.Validate(formats); err != nil { + if ve, ok := err.(*errors.Validation); ok { + return ve.ValidateName("osImageConfig") + } else if ce, ok := err.(*errors.CompositeError); ok { + return ce.ValidateName("osImageConfig") + } + return err + } + } + + return nil +} + +func (m *NodeBootstrapping) validateSigImageConfig(formats strfmt.Registry) error { + + if err := validate.Required("sigImageConfig", "body", m.SigImageConfig); err != nil { + return err + } + + if m.SigImageConfig != nil { + if err := m.SigImageConfig.Validate(formats); err != nil { + if ve, ok := err.(*errors.Validation); ok { + return ve.ValidateName("sigImageConfig") + } else if ce, ok := err.(*errors.CompositeError); ok { + return ce.ValidateName("sigImageConfig") + } + return err + } + } + + return nil +} + +// ContextValidate validate this node bootstrapping based on the context it is used +func (m *NodeBootstrapping) ContextValidate(ctx context.Context, formats strfmt.Registry) error { + var res []error + + if err := m.contextValidateOsImageConfig(ctx, formats); err != nil { + res = append(res, err) + } + + if err := m.contextValidateSigImageConfig(ctx, formats); err != nil { + res = append(res, err) + } + + if len(res) > 0 { + return errors.CompositeValidationError(res...) + } + return nil +} + +func (m *NodeBootstrapping) contextValidateOsImageConfig(ctx context.Context, formats strfmt.Registry) error { + + if m.OsImageConfig != nil { + + if err := m.OsImageConfig.ContextValidate(ctx, formats); err != nil { + if ve, ok := err.(*errors.Validation); ok { + return ve.ValidateName("osImageConfig") + } else if ce, ok := err.(*errors.CompositeError); ok { + return ce.ValidateName("osImageConfig") + } + return err + } + } + + return nil +} + +func (m *NodeBootstrapping) contextValidateSigImageConfig(ctx context.Context, formats strfmt.Registry) error { + + if m.SigImageConfig != nil { + + if err := m.SigImageConfig.ContextValidate(ctx, formats); err != nil { + if ve, ok := err.(*errors.Validation); ok { + return ve.ValidateName("sigImageConfig") + } else if ce, ok := err.(*errors.CompositeError); ok { + return ce.ValidateName("sigImageConfig") + } + return err + } + } + + return nil +} + +// MarshalBinary interface implementation +func (m *NodeBootstrapping) MarshalBinary() ([]byte, error) { + if m == nil { + return nil, nil + } + return swag.WriteJSON(m) +} + +// UnmarshalBinary interface implementation +func (m *NodeBootstrapping) UnmarshalBinary(b []byte) error { + var res NodeBootstrapping + if err := swag.ReadJSON(b, &res); err != nil { + return err + } + *m = res + return nil +} diff --git a/pkg/provisionclients/models/provision_helper_values.go b/pkg/provisionclients/models/provision_helper_values.go new file mode 100644 index 000000000..a86782d55 --- /dev/null +++ b/pkg/provisionclients/models/provision_helper_values.go @@ -0,0 +1,103 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by go-swagger; DO NOT EDIT. + +package models + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "context" + + "github.com/go-openapi/errors" + "github.com/go-openapi/strfmt" + "github.com/go-openapi/swag" + "github.com/go-openapi/validate" +) + +// ProvisionHelperValues provision helper values +// +// swagger:model ProvisionHelperValues +type ProvisionHelperValues struct { + + // sku CPU + // Required: true + SkuCPU *float64 `json:"skuCPU"` + + // sku memory + // Required: true + SkuMemory *float64 `json:"skuMemory"` +} + +// Validate validates this provision helper values +func (m *ProvisionHelperValues) Validate(formats strfmt.Registry) error { + var res []error + + if err := m.validateSkuCPU(formats); err != nil { + res = append(res, err) + } + + if err := m.validateSkuMemory(formats); err != nil { + res = append(res, err) + } + + if len(res) > 0 { + return errors.CompositeValidationError(res...) + } + return nil +} + +func (m *ProvisionHelperValues) validateSkuCPU(formats strfmt.Registry) error { + + if err := validate.Required("skuCPU", "body", m.SkuCPU); err != nil { + return err + } + + return nil +} + +func (m *ProvisionHelperValues) validateSkuMemory(formats strfmt.Registry) error { + + if err := validate.Required("skuMemory", "body", m.SkuMemory); err != nil { + return err + } + + return nil +} + +// ContextValidate validates this provision helper values based on context it is used +func (m *ProvisionHelperValues) ContextValidate(ctx context.Context, formats strfmt.Registry) error { + return nil +} + +// MarshalBinary interface implementation +func (m *ProvisionHelperValues) MarshalBinary() ([]byte, error) { + if m == nil { + return nil, nil + } + return swag.WriteJSON(m) +} + +// UnmarshalBinary interface implementation +func (m *ProvisionHelperValues) UnmarshalBinary(b []byte) error { + var res ProvisionHelperValues + if err := swag.ReadJSON(b, &res); err != nil { + return err + } + *m = res + return nil +} diff --git a/pkg/provisionclients/models/provision_profile.go b/pkg/provisionclients/models/provision_profile.go new file mode 100644 index 000000000..690b279e7 --- /dev/null +++ b/pkg/provisionclients/models/provision_profile.go @@ -0,0 +1,608 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by go-swagger; DO NOT EDIT. + +package models + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "context" + + "github.com/go-openapi/errors" + "github.com/go-openapi/strfmt" + "github.com/go-openapi/swag" + "github.com/go-openapi/validate" +) + +// ProvisionProfile provision profile +// +// swagger:model ProvisionProfile +type ProvisionProfile struct { + + // agent pool windows profile + AgentPoolWindowsProfile *AgentPoolWindowsProfile `json:"agentPoolWindowsProfile,omitempty"` + + // architecture + // Required: true + Architecture *string `json:"architecture"` + + // artifact streaming profile + ArtifactStreamingProfile *ArtifactStreamingProfile `json:"artifactStreamingProfile,omitempty"` + + // custom kubelet config + CustomKubeletConfig *CustomKubeletConfig `json:"customKubeletConfig,omitempty"` + + // custom linux o s config + CustomLinuxOSConfig *CustomLinuxOSConfig `json:"customLinuxOSConfig,omitempty"` + + // custom node labels + CustomNodeLabels map[string]string `json:"customNodeLabels,omitempty"` + + // distro + // Required: true + Distro *string `json:"distro"` + + // enable f IP s + EnableFIPS *bool `json:"enableFIPS,omitempty"` + + // gpu instance profile + GpuInstanceProfile *int32 `json:"gpuInstanceProfile,omitempty"` + + // gpu profile + GpuProfile *GPUProfile `json:"gpuProfile,omitempty"` + + // kubelet disk type + KubeletDiskType *int32 `json:"kubeletDiskType,omitempty"` + + // max pods + // Required: true + MaxPods *int32 `json:"maxPods"` + + // message of the day + MessageOfTheDay *string `json:"messageOfTheDay,omitempty"` + + // mode + // Required: true + Mode *int32 `json:"mode"` + + // name + // Required: true + Name *string `json:"name"` + + // node initialization taints + NodeInitializationTaints []string `json:"nodeInitializationTaints"` + + // node taints + NodeTaints []string `json:"nodeTaints"` + + // orchestrator version + // Required: true + OrchestratorVersion *string `json:"orchestratorVersion"` + + // os sku + // Required: true + OsSku *int32 `json:"osSku"` + + // os type + // Required: true + OsType *int32 `json:"osType"` + + // security profile + SecurityProfile *AgentPoolSecurityProfile `json:"securityProfile,omitempty"` + + // storage profile + // Required: true + StorageProfile *string `json:"storageProfile"` + + // vm size + // Required: true + VMSize *string `json:"vmSize"` + + // vnet cidrs + // Required: true + VnetCidrs []string `json:"vnetCidrs"` + + // vnet subnet ID + // Required: true + VnetSubnetID *string `json:"vnetSubnetID"` + + // workload runtime + WorkloadRuntime *int32 `json:"workloadRuntime,omitempty"` +} + +// Validate validates this provision profile +func (m *ProvisionProfile) Validate(formats strfmt.Registry) error { + var res []error + + if err := m.validateAgentPoolWindowsProfile(formats); err != nil { + res = append(res, err) + } + + if err := m.validateArchitecture(formats); err != nil { + res = append(res, err) + } + + if err := m.validateArtifactStreamingProfile(formats); err != nil { + res = append(res, err) + } + + if err := m.validateCustomKubeletConfig(formats); err != nil { + res = append(res, err) + } + + if err := m.validateCustomLinuxOSConfig(formats); err != nil { + res = append(res, err) + } + + if err := m.validateDistro(formats); err != nil { + res = append(res, err) + } + + if err := m.validateGpuProfile(formats); err != nil { + res = append(res, err) + } + + if err := m.validateMaxPods(formats); err != nil { + res = append(res, err) + } + + if err := m.validateMode(formats); err != nil { + res = append(res, err) + } + + if err := m.validateName(formats); err != nil { + res = append(res, err) + } + + if err := m.validateOrchestratorVersion(formats); err != nil { + res = append(res, err) + } + + if err := m.validateOsSku(formats); err != nil { + res = append(res, err) + } + + if err := m.validateOsType(formats); err != nil { + res = append(res, err) + } + + if err := m.validateSecurityProfile(formats); err != nil { + res = append(res, err) + } + + if err := m.validateStorageProfile(formats); err != nil { + res = append(res, err) + } + + if err := m.validateVMSize(formats); err != nil { + res = append(res, err) + } + + if err := m.validateVnetCidrs(formats); err != nil { + res = append(res, err) + } + + if err := m.validateVnetSubnetID(formats); err != nil { + res = append(res, err) + } + + if len(res) > 0 { + return errors.CompositeValidationError(res...) + } + return nil +} + +func (m *ProvisionProfile) validateAgentPoolWindowsProfile(formats strfmt.Registry) error { + if swag.IsZero(m.AgentPoolWindowsProfile) { // not required + return nil + } + + if m.AgentPoolWindowsProfile != nil { + if err := m.AgentPoolWindowsProfile.Validate(formats); err != nil { + if ve, ok := err.(*errors.Validation); ok { + return ve.ValidateName("agentPoolWindowsProfile") + } else if ce, ok := err.(*errors.CompositeError); ok { + return ce.ValidateName("agentPoolWindowsProfile") + } + return err + } + } + + return nil +} + +func (m *ProvisionProfile) validateArchitecture(formats strfmt.Registry) error { + + if err := validate.Required("architecture", "body", m.Architecture); err != nil { + return err + } + + return nil +} + +func (m *ProvisionProfile) validateArtifactStreamingProfile(formats strfmt.Registry) error { + if swag.IsZero(m.ArtifactStreamingProfile) { // not required + return nil + } + + if m.ArtifactStreamingProfile != nil { + if err := m.ArtifactStreamingProfile.Validate(formats); err != nil { + if ve, ok := err.(*errors.Validation); ok { + return ve.ValidateName("artifactStreamingProfile") + } else if ce, ok := err.(*errors.CompositeError); ok { + return ce.ValidateName("artifactStreamingProfile") + } + return err + } + } + + return nil +} + +func (m *ProvisionProfile) validateCustomKubeletConfig(formats strfmt.Registry) error { + if swag.IsZero(m.CustomKubeletConfig) { // not required + return nil + } + + if m.CustomKubeletConfig != nil { + if err := m.CustomKubeletConfig.Validate(formats); err != nil { + if ve, ok := err.(*errors.Validation); ok { + return ve.ValidateName("customKubeletConfig") + } else if ce, ok := err.(*errors.CompositeError); ok { + return ce.ValidateName("customKubeletConfig") + } + return err + } + } + + return nil +} + +func (m *ProvisionProfile) validateCustomLinuxOSConfig(formats strfmt.Registry) error { + if swag.IsZero(m.CustomLinuxOSConfig) { // not required + return nil + } + + if m.CustomLinuxOSConfig != nil { + if err := m.CustomLinuxOSConfig.Validate(formats); err != nil { + if ve, ok := err.(*errors.Validation); ok { + return ve.ValidateName("customLinuxOSConfig") + } else if ce, ok := err.(*errors.CompositeError); ok { + return ce.ValidateName("customLinuxOSConfig") + } + return err + } + } + + return nil +} + +func (m *ProvisionProfile) validateDistro(formats strfmt.Registry) error { + + if err := validate.Required("distro", "body", m.Distro); err != nil { + return err + } + + return nil +} + +func (m *ProvisionProfile) validateGpuProfile(formats strfmt.Registry) error { + if swag.IsZero(m.GpuProfile) { // not required + return nil + } + + if m.GpuProfile != nil { + if err := m.GpuProfile.Validate(formats); err != nil { + if ve, ok := err.(*errors.Validation); ok { + return ve.ValidateName("gpuProfile") + } else if ce, ok := err.(*errors.CompositeError); ok { + return ce.ValidateName("gpuProfile") + } + return err + } + } + + return nil +} + +func (m *ProvisionProfile) validateMaxPods(formats strfmt.Registry) error { + + if err := validate.Required("maxPods", "body", m.MaxPods); err != nil { + return err + } + + return nil +} + +func (m *ProvisionProfile) validateMode(formats strfmt.Registry) error { + + if err := validate.Required("mode", "body", m.Mode); err != nil { + return err + } + + return nil +} + +func (m *ProvisionProfile) validateName(formats strfmt.Registry) error { + + if err := validate.Required("name", "body", m.Name); err != nil { + return err + } + + return nil +} + +func (m *ProvisionProfile) validateOrchestratorVersion(formats strfmt.Registry) error { + + if err := validate.Required("orchestratorVersion", "body", m.OrchestratorVersion); err != nil { + return err + } + + return nil +} + +func (m *ProvisionProfile) validateOsSku(formats strfmt.Registry) error { + + if err := validate.Required("osSku", "body", m.OsSku); err != nil { + return err + } + + return nil +} + +func (m *ProvisionProfile) validateOsType(formats strfmt.Registry) error { + + if err := validate.Required("osType", "body", m.OsType); err != nil { + return err + } + + return nil +} + +func (m *ProvisionProfile) validateSecurityProfile(formats strfmt.Registry) error { + if swag.IsZero(m.SecurityProfile) { // not required + return nil + } + + if m.SecurityProfile != nil { + if err := m.SecurityProfile.Validate(formats); err != nil { + if ve, ok := err.(*errors.Validation); ok { + return ve.ValidateName("securityProfile") + } else if ce, ok := err.(*errors.CompositeError); ok { + return ce.ValidateName("securityProfile") + } + return err + } + } + + return nil +} + +func (m *ProvisionProfile) validateStorageProfile(formats strfmt.Registry) error { + + if err := validate.Required("storageProfile", "body", m.StorageProfile); err != nil { + return err + } + + return nil +} + +func (m *ProvisionProfile) validateVMSize(formats strfmt.Registry) error { + + if err := validate.Required("vmSize", "body", m.VMSize); err != nil { + return err + } + + return nil +} + +func (m *ProvisionProfile) validateVnetCidrs(formats strfmt.Registry) error { + + if err := validate.Required("vnetCidrs", "body", m.VnetCidrs); err != nil { + return err + } + + return nil +} + +func (m *ProvisionProfile) validateVnetSubnetID(formats strfmt.Registry) error { + + if err := validate.Required("vnetSubnetID", "body", m.VnetSubnetID); err != nil { + return err + } + + return nil +} + +// ContextValidate validate this provision profile based on the context it is used +func (m *ProvisionProfile) ContextValidate(ctx context.Context, formats strfmt.Registry) error { + var res []error + + if err := m.contextValidateAgentPoolWindowsProfile(ctx, formats); err != nil { + res = append(res, err) + } + + if err := m.contextValidateArtifactStreamingProfile(ctx, formats); err != nil { + res = append(res, err) + } + + if err := m.contextValidateCustomKubeletConfig(ctx, formats); err != nil { + res = append(res, err) + } + + if err := m.contextValidateCustomLinuxOSConfig(ctx, formats); err != nil { + res = append(res, err) + } + + if err := m.contextValidateGpuProfile(ctx, formats); err != nil { + res = append(res, err) + } + + if err := m.contextValidateSecurityProfile(ctx, formats); err != nil { + res = append(res, err) + } + + if len(res) > 0 { + return errors.CompositeValidationError(res...) + } + return nil +} + +func (m *ProvisionProfile) contextValidateAgentPoolWindowsProfile(ctx context.Context, formats strfmt.Registry) error { + + if m.AgentPoolWindowsProfile != nil { + + if swag.IsZero(m.AgentPoolWindowsProfile) { // not required + return nil + } + + if err := m.AgentPoolWindowsProfile.ContextValidate(ctx, formats); err != nil { + if ve, ok := err.(*errors.Validation); ok { + return ve.ValidateName("agentPoolWindowsProfile") + } else if ce, ok := err.(*errors.CompositeError); ok { + return ce.ValidateName("agentPoolWindowsProfile") + } + return err + } + } + + return nil +} + +func (m *ProvisionProfile) contextValidateArtifactStreamingProfile(ctx context.Context, formats strfmt.Registry) error { + + if m.ArtifactStreamingProfile != nil { + + if swag.IsZero(m.ArtifactStreamingProfile) { // not required + return nil + } + + if err := m.ArtifactStreamingProfile.ContextValidate(ctx, formats); err != nil { + if ve, ok := err.(*errors.Validation); ok { + return ve.ValidateName("artifactStreamingProfile") + } else if ce, ok := err.(*errors.CompositeError); ok { + return ce.ValidateName("artifactStreamingProfile") + } + return err + } + } + + return nil +} + +func (m *ProvisionProfile) contextValidateCustomKubeletConfig(ctx context.Context, formats strfmt.Registry) error { + + if m.CustomKubeletConfig != nil { + + if swag.IsZero(m.CustomKubeletConfig) { // not required + return nil + } + + if err := m.CustomKubeletConfig.ContextValidate(ctx, formats); err != nil { + if ve, ok := err.(*errors.Validation); ok { + return ve.ValidateName("customKubeletConfig") + } else if ce, ok := err.(*errors.CompositeError); ok { + return ce.ValidateName("customKubeletConfig") + } + return err + } + } + + return nil +} + +func (m *ProvisionProfile) contextValidateCustomLinuxOSConfig(ctx context.Context, formats strfmt.Registry) error { + + if m.CustomLinuxOSConfig != nil { + + if swag.IsZero(m.CustomLinuxOSConfig) { // not required + return nil + } + + if err := m.CustomLinuxOSConfig.ContextValidate(ctx, formats); err != nil { + if ve, ok := err.(*errors.Validation); ok { + return ve.ValidateName("customLinuxOSConfig") + } else if ce, ok := err.(*errors.CompositeError); ok { + return ce.ValidateName("customLinuxOSConfig") + } + return err + } + } + + return nil +} + +func (m *ProvisionProfile) contextValidateGpuProfile(ctx context.Context, formats strfmt.Registry) error { + + if m.GpuProfile != nil { + + if swag.IsZero(m.GpuProfile) { // not required + return nil + } + + if err := m.GpuProfile.ContextValidate(ctx, formats); err != nil { + if ve, ok := err.(*errors.Validation); ok { + return ve.ValidateName("gpuProfile") + } else if ce, ok := err.(*errors.CompositeError); ok { + return ce.ValidateName("gpuProfile") + } + return err + } + } + + return nil +} + +func (m *ProvisionProfile) contextValidateSecurityProfile(ctx context.Context, formats strfmt.Registry) error { + + if m.SecurityProfile != nil { + + if swag.IsZero(m.SecurityProfile) { // not required + return nil + } + + if err := m.SecurityProfile.ContextValidate(ctx, formats); err != nil { + if ve, ok := err.(*errors.Validation); ok { + return ve.ValidateName("securityProfile") + } else if ce, ok := err.(*errors.CompositeError); ok { + return ce.ValidateName("securityProfile") + } + return err + } + } + + return nil +} + +// MarshalBinary interface implementation +func (m *ProvisionProfile) MarshalBinary() ([]byte, error) { + if m == nil { + return nil, nil + } + return swag.WriteJSON(m) +} + +// UnmarshalBinary interface implementation +func (m *ProvisionProfile) UnmarshalBinary(b []byte) error { + var res ProvisionProfile + if err := swag.ReadJSON(b, &res); err != nil { + return err + } + *m = res + return nil +} diff --git a/pkg/provisionclients/models/provision_values.go b/pkg/provisionclients/models/provision_values.go new file mode 100644 index 000000000..bbfc5aa32 --- /dev/null +++ b/pkg/provisionclients/models/provision_values.go @@ -0,0 +1,172 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by go-swagger; DO NOT EDIT. + +package models + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "context" + + "github.com/go-openapi/errors" + "github.com/go-openapi/strfmt" + "github.com/go-openapi/swag" + "github.com/go-openapi/validate" +) + +// ProvisionValues provision values +// +// swagger:model ProvisionValues +type ProvisionValues struct { + + // provision helper values + // Required: true + ProvisionHelperValues *ProvisionHelperValues `json:"provisionHelperValues"` + + // provision profile + // Required: true + ProvisionProfile *ProvisionProfile `json:"provisionProfile"` +} + +// Validate validates this provision values +func (m *ProvisionValues) Validate(formats strfmt.Registry) error { + var res []error + + if err := m.validateProvisionHelperValues(formats); err != nil { + res = append(res, err) + } + + if err := m.validateProvisionProfile(formats); err != nil { + res = append(res, err) + } + + if len(res) > 0 { + return errors.CompositeValidationError(res...) + } + return nil +} + +func (m *ProvisionValues) validateProvisionHelperValues(formats strfmt.Registry) error { + + if err := validate.Required("provisionHelperValues", "body", m.ProvisionHelperValues); err != nil { + return err + } + + if m.ProvisionHelperValues != nil { + if err := m.ProvisionHelperValues.Validate(formats); err != nil { + if ve, ok := err.(*errors.Validation); ok { + return ve.ValidateName("provisionHelperValues") + } else if ce, ok := err.(*errors.CompositeError); ok { + return ce.ValidateName("provisionHelperValues") + } + return err + } + } + + return nil +} + +func (m *ProvisionValues) validateProvisionProfile(formats strfmt.Registry) error { + + if err := validate.Required("provisionProfile", "body", m.ProvisionProfile); err != nil { + return err + } + + if m.ProvisionProfile != nil { + if err := m.ProvisionProfile.Validate(formats); err != nil { + if ve, ok := err.(*errors.Validation); ok { + return ve.ValidateName("provisionProfile") + } else if ce, ok := err.(*errors.CompositeError); ok { + return ce.ValidateName("provisionProfile") + } + return err + } + } + + return nil +} + +// ContextValidate validate this provision values based on the context it is used +func (m *ProvisionValues) ContextValidate(ctx context.Context, formats strfmt.Registry) error { + var res []error + + if err := m.contextValidateProvisionHelperValues(ctx, formats); err != nil { + res = append(res, err) + } + + if err := m.contextValidateProvisionProfile(ctx, formats); err != nil { + res = append(res, err) + } + + if len(res) > 0 { + return errors.CompositeValidationError(res...) + } + return nil +} + +func (m *ProvisionValues) contextValidateProvisionHelperValues(ctx context.Context, formats strfmt.Registry) error { + + if m.ProvisionHelperValues != nil { + + if err := m.ProvisionHelperValues.ContextValidate(ctx, formats); err != nil { + if ve, ok := err.(*errors.Validation); ok { + return ve.ValidateName("provisionHelperValues") + } else if ce, ok := err.(*errors.CompositeError); ok { + return ce.ValidateName("provisionHelperValues") + } + return err + } + } + + return nil +} + +func (m *ProvisionValues) contextValidateProvisionProfile(ctx context.Context, formats strfmt.Registry) error { + + if m.ProvisionProfile != nil { + + if err := m.ProvisionProfile.ContextValidate(ctx, formats); err != nil { + if ve, ok := err.(*errors.Validation); ok { + return ve.ValidateName("provisionProfile") + } else if ce, ok := err.(*errors.CompositeError); ok { + return ce.ValidateName("provisionProfile") + } + return err + } + } + + return nil +} + +// MarshalBinary interface implementation +func (m *ProvisionValues) MarshalBinary() ([]byte, error) { + if m == nil { + return nil, nil + } + return swag.WriteJSON(m) +} + +// UnmarshalBinary interface implementation +func (m *ProvisionValues) UnmarshalBinary(b []byte) error { + var res ProvisionValues + if err := swag.ReadJSON(b, &res); err != nil { + return err + } + *m = res + return nil +} diff --git a/pkg/provisionclients/models/sig_image_config.go b/pkg/provisionclients/models/sig_image_config.go new file mode 100644 index 000000000..dcbeca86f --- /dev/null +++ b/pkg/provisionclients/models/sig_image_config.go @@ -0,0 +1,127 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by go-swagger; DO NOT EDIT. + +package models + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "context" + + "github.com/go-openapi/errors" + "github.com/go-openapi/strfmt" + "github.com/go-openapi/swag" +) + +// SigImageConfig sig image config +// +// swagger:model SigImageConfig +type SigImageConfig struct { + + // sig image config template + SigImageConfigTemplate *SigImageConfigTemplate `json:"sigImageConfigTemplate,omitempty"` + + // subscription ID + SubscriptionID *string `json:"subscriptionID,omitempty"` +} + +// Validate validates this sig image config +func (m *SigImageConfig) Validate(formats strfmt.Registry) error { + var res []error + + if err := m.validateSigImageConfigTemplate(formats); err != nil { + res = append(res, err) + } + + if len(res) > 0 { + return errors.CompositeValidationError(res...) + } + return nil +} + +func (m *SigImageConfig) validateSigImageConfigTemplate(formats strfmt.Registry) error { + if swag.IsZero(m.SigImageConfigTemplate) { // not required + return nil + } + + if m.SigImageConfigTemplate != nil { + if err := m.SigImageConfigTemplate.Validate(formats); err != nil { + if ve, ok := err.(*errors.Validation); ok { + return ve.ValidateName("sigImageConfigTemplate") + } else if ce, ok := err.(*errors.CompositeError); ok { + return ce.ValidateName("sigImageConfigTemplate") + } + return err + } + } + + return nil +} + +// ContextValidate validate this sig image config based on the context it is used +func (m *SigImageConfig) ContextValidate(ctx context.Context, formats strfmt.Registry) error { + var res []error + + if err := m.contextValidateSigImageConfigTemplate(ctx, formats); err != nil { + res = append(res, err) + } + + if len(res) > 0 { + return errors.CompositeValidationError(res...) + } + return nil +} + +func (m *SigImageConfig) contextValidateSigImageConfigTemplate(ctx context.Context, formats strfmt.Registry) error { + + if m.SigImageConfigTemplate != nil { + + if swag.IsZero(m.SigImageConfigTemplate) { // not required + return nil + } + + if err := m.SigImageConfigTemplate.ContextValidate(ctx, formats); err != nil { + if ve, ok := err.(*errors.Validation); ok { + return ve.ValidateName("sigImageConfigTemplate") + } else if ce, ok := err.(*errors.CompositeError); ok { + return ce.ValidateName("sigImageConfigTemplate") + } + return err + } + } + + return nil +} + +// MarshalBinary interface implementation +func (m *SigImageConfig) MarshalBinary() ([]byte, error) { + if m == nil { + return nil, nil + } + return swag.WriteJSON(m) +} + +// UnmarshalBinary interface implementation +func (m *SigImageConfig) UnmarshalBinary(b []byte) error { + var res SigImageConfig + if err := swag.ReadJSON(b, &res); err != nil { + return err + } + *m = res + return nil +} diff --git a/pkg/provisionclients/models/sig_image_config_template.go b/pkg/provisionclients/models/sig_image_config_template.go new file mode 100644 index 000000000..d064f02f3 --- /dev/null +++ b/pkg/provisionclients/models/sig_image_config_template.go @@ -0,0 +1,74 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by go-swagger; DO NOT EDIT. + +package models + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "context" + + "github.com/go-openapi/strfmt" + "github.com/go-openapi/swag" +) + +// SigImageConfigTemplate sig image config template +// +// swagger:model SigImageConfigTemplate +type SigImageConfigTemplate struct { + + // definition + Definition *string `json:"definition,omitempty"` + + // gallery + Gallery *string `json:"gallery,omitempty"` + + // resource group + ResourceGroup *string `json:"resourceGroup,omitempty"` + + // version + Version *string `json:"version,omitempty"` +} + +// Validate validates this sig image config template +func (m *SigImageConfigTemplate) Validate(formats strfmt.Registry) error { + return nil +} + +// ContextValidate validates this sig image config template based on context it is used +func (m *SigImageConfigTemplate) ContextValidate(ctx context.Context, formats strfmt.Registry) error { + return nil +} + +// MarshalBinary interface implementation +func (m *SigImageConfigTemplate) MarshalBinary() ([]byte, error) { + if m == nil { + return nil, nil + } + return swag.WriteJSON(m) +} + +// UnmarshalBinary interface implementation +func (m *SigImageConfigTemplate) UnmarshalBinary(b []byte) error { + var res SigImageConfigTemplate + if err := swag.ReadJSON(b, &res); err != nil { + return err + } + *m = res + return nil +} diff --git a/pkg/provisionclients/models/sysctl_config.go b/pkg/provisionclients/models/sysctl_config.go new file mode 100644 index 000000000..1c7354f85 --- /dev/null +++ b/pkg/provisionclients/models/sysctl_config.go @@ -0,0 +1,146 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by go-swagger; DO NOT EDIT. + +package models + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "context" + + "github.com/go-openapi/strfmt" + "github.com/go-openapi/swag" +) + +// SysctlConfig sysctl config +// +// swagger:model SysctlConfig +type SysctlConfig struct { + + // fs aio max nr + FsAioMaxNr *int32 `json:"fsAioMaxNr,omitempty"` + + // fs file max + FsFileMax *int32 `json:"fsFileMax,omitempty"` + + // fs inotify max user watches + FsInotifyMaxUserWatches *int32 `json:"fsInotifyMaxUserWatches,omitempty"` + + // fs nr open + FsNrOpen *int32 `json:"fsNrOpen,omitempty"` + + // kernel threads max + KernelThreadsMax *int32 `json:"kernelThreadsMax,omitempty"` + + // net core netdev max backlog + NetCoreNetdevMaxBacklog *int32 `json:"netCoreNetdevMaxBacklog,omitempty"` + + // net core optmem max + NetCoreOptmemMax *int32 `json:"netCoreOptmemMax,omitempty"` + + // net core rmem default + NetCoreRmemDefault *int32 `json:"netCoreRmemDefault,omitempty"` + + // net core rmem max + NetCoreRmemMax *int32 `json:"netCoreRmemMax,omitempty"` + + // net core somaxconn + NetCoreSomaxconn *int32 `json:"netCoreSomaxconn,omitempty"` + + // net core wmem default + NetCoreWmemDefault *int32 `json:"netCoreWmemDefault,omitempty"` + + // net core wmem max + NetCoreWmemMax *int32 `json:"netCoreWmemMax,omitempty"` + + // net Ipv4 Ip local port range + NetIPV4IPLocalPortRange *string `json:"netIpv4IpLocalPortRange,omitempty"` + + // net Ipv4 neigh default gc thresh1 + NetIPV4NeighDefaultGcThresh1 *int32 `json:"netIpv4NeighDefaultGcThresh1,omitempty"` + + // net Ipv4 neigh default gc thresh2 + NetIPV4NeighDefaultGcThresh2 *int32 `json:"netIpv4NeighDefaultGcThresh2,omitempty"` + + // net Ipv4 neigh default gc thresh3 + NetIPV4NeighDefaultGcThresh3 *int32 `json:"netIpv4NeighDefaultGcThresh3,omitempty"` + + // net Ipv4 Tcp fin timeout + NetIPV4TCPFinTimeout *int32 `json:"netIpv4TcpFinTimeout,omitempty"` + + // net Ipv4 Tcp keepalive probes + NetIPV4TCPKeepaliveProbes *int32 `json:"netIpv4TcpKeepaliveProbes,omitempty"` + + // net Ipv4 Tcp keepalive time + NetIPV4TCPKeepaliveTime *int32 `json:"netIpv4TcpKeepaliveTime,omitempty"` + + // net Ipv4 Tcp max syn backlog + NetIPV4TCPMaxSynBacklog *int32 `json:"netIpv4TcpMaxSynBacklog,omitempty"` + + // net Ipv4 Tcp max tw buckets + NetIPV4TCPMaxTwBuckets *int32 `json:"netIpv4TcpMaxTwBuckets,omitempty"` + + // net Ipv4 Tcp tw reuse + NetIPV4TCPTwReuse *bool `json:"netIpv4TcpTwReuse,omitempty"` + + // net Ipv4 tcpkeepalive intvl + NetIPV4TcpkeepaliveIntvl *int32 `json:"netIpv4TcpkeepaliveIntvl,omitempty"` + + // net netfilter nf conntrack buckets + NetNetfilterNfConntrackBuckets *int32 `json:"netNetfilterNfConntrackBuckets,omitempty"` + + // net netfilter nf conntrack max + NetNetfilterNfConntrackMax *int32 `json:"netNetfilterNfConntrackMax,omitempty"` + + // vm max map count + VMMaxMapCount *int32 `json:"vmMaxMapCount,omitempty"` + + // vm swappiness + VMSwappiness *int32 `json:"vmSwappiness,omitempty"` + + // vm vfs cache pressure + VMVfsCachePressure *int32 `json:"vmVfsCachePressure,omitempty"` +} + +// Validate validates this sysctl config +func (m *SysctlConfig) Validate(formats strfmt.Registry) error { + return nil +} + +// ContextValidate validates this sysctl config based on context it is used +func (m *SysctlConfig) ContextValidate(ctx context.Context, formats strfmt.Registry) error { + return nil +} + +// MarshalBinary interface implementation +func (m *SysctlConfig) MarshalBinary() ([]byte, error) { + if m == nil { + return nil, nil + } + return swag.WriteJSON(m) +} + +// UnmarshalBinary interface implementation +func (m *SysctlConfig) UnmarshalBinary(b []byte) error { + var res SysctlConfig + if err := swag.ReadJSON(b, &res); err != nil { + return err + } + *m = res + return nil +} diff --git a/pkg/provisionclients/models/ulimit_config.go b/pkg/provisionclients/models/ulimit_config.go new file mode 100644 index 000000000..7c554b81c --- /dev/null +++ b/pkg/provisionclients/models/ulimit_config.go @@ -0,0 +1,68 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by go-swagger; DO NOT EDIT. + +package models + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "context" + + "github.com/go-openapi/strfmt" + "github.com/go-openapi/swag" +) + +// UlimitConfig ulimit config +// +// swagger:model UlimitConfig +type UlimitConfig struct { + + // max locked memory + MaxLockedMemory *string `json:"maxLockedMemory,omitempty"` + + // no file + NoFile *string `json:"noFile,omitempty"` +} + +// Validate validates this ulimit config +func (m *UlimitConfig) Validate(formats strfmt.Registry) error { + return nil +} + +// ContextValidate validates this ulimit config based on context it is used +func (m *UlimitConfig) ContextValidate(ctx context.Context, formats strfmt.Registry) error { + return nil +} + +// MarshalBinary interface implementation +func (m *UlimitConfig) MarshalBinary() ([]byte, error) { + if m == nil { + return nil, nil + } + return swag.WriteJSON(m) +} + +// UnmarshalBinary interface implementation +func (m *UlimitConfig) UnmarshalBinary(b []byte) error { + var res UlimitConfig + if err := swag.ReadJSON(b, &res); err != nil { + return err + } + *m = res + return nil +} diff --git a/pkg/provisionclients/swagger/nodebootstrapping.json b/pkg/provisionclients/swagger/nodebootstrapping.json new file mode 100644 index 000000000..2271bc8fd --- /dev/null +++ b/pkg/provisionclients/swagger/nodebootstrapping.json @@ -0,0 +1,653 @@ +{ + "swagger": "2.0", + "info": { + "title": "NodeBootstrapping", + "description": "A swagger spec that defines the get NodeBootstrapping operation.", + "version": "2024-10-01" + }, + "schemes": [ + "https" + ], + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "paths": { + "/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ContainerService/managedclusters/{resourceName}/nodeBootstrapping": { + "get": { + "operationId": "NodeBootstrapping_Get", + "summary": "Gets NodeBootstrapping values for a specified provisioning configuration, with secrets redacted.", + "description": "Gets NodeBootstrapping values for a specified provisioning configuration, with secrets redacted.", + "parameters": [ + { + "$ref": "#/parameters/SubscriptionIdParameter" + }, + { + "$ref": "#/parameters/ResourceGroupNameParameter" + }, + { + "$ref": "#/parameters/ResourceNameParameter" + }, + { + "name": "parameters", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/ProvisionValues" + }, + "description": "Values required to determine NodeBootstrapping values." + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/NodeBootstrapping" + } + }, + "default": { + "description": "Error response describing why the operation failed.", + "schema": { + "type": "string" + } + } + } + } + } + }, + "definitions": { + "ProvisionValues": { + "type": "object", + "properties": { + "provisionProfile": { + "$ref": "#/definitions/ProvisionProfile", + "x-nullable": true + }, + "provisionHelperValues": { + "$ref": "#/definitions/ProvisionHelperValues", + "x-nullable": true + } + }, + "required": [ + "provisionProfile", + "provisionHelperValues" + ] + }, + "ProvisionProfile": { + "type": "object", + "properties": { + "name": { + "type": "string", + "x-nullable": true + }, + "vmSize": { + "type": "string", + "x-nullable": true + }, + "osType": { + "type": "integer", + "format": "int32", + "x-nullable": true + }, + "osSku": { + "type": "integer", + "format": "int32", + "x-nullable": true + }, + "storageProfile": { + "type": "string", + "x-nullable": true + }, + "distro": { + "type": "string", + "x-nullable": true + }, + "customNodeLabels": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "x-nullable": true + }, + "orchestratorVersion": { + "type": "string", + "x-nullable": true + }, + "vnetCidrs": { + "type": "array", + "items": { + "type": "string" + }, + "x-nullable": true + }, + "kubeletDiskType": { + "type": "integer", + "format": "int32", + "x-nullable": true + }, + "vnetSubnetID": { + "type": "string", + "x-nullable": true + }, + "customKubeletConfig": { + "$ref": "#/definitions/CustomKubeletConfig", + "x-nullable": true + }, + "customLinuxOSConfig": { + "$ref": "#/definitions/CustomLinuxOSConfig", + "x-nullable": true + }, + "enableFIPS": { + "type": "boolean", + "x-nullable": true + }, + "mode": { + "type": "integer", + "format": "int32", + "x-nullable": true + }, + "gpuInstanceProfile": { + "type": "integer", + "format": "int32", + "x-nullable": true + }, + "workloadRuntime": { + "type": "integer", + "format": "int32", + "x-nullable": true + }, + "messageOfTheDay": { + "type": "string", + "x-nullable": true + }, + "architecture": { + "type": "string", + "x-nullable": true + }, + "agentPoolWindowsProfile": { + "$ref": "#/definitions/AgentPoolWindowsProfile", + "x-nullable": true + }, + "securityProfile": { + "$ref": "#/definitions/AgentPoolSecurityProfile", + "x-nullable": true + }, + "gpuProfile": { + "$ref": "#/definitions/GPUProfile", + "x-nullable": true + }, + "artifactStreamingProfile": { + "$ref": "#/definitions/ArtifactStreamingProfile", + "x-nullable": true + }, + "nodeInitializationTaints": { + "type": "array", + "items": { + "type": "string" + }, + "x-nullable": true + }, + "nodeTaints": { + "type": "array", + "items": { + "type": "string" + }, + "x-nullable": true + }, + "maxPods": { + "type": "integer", + "format": "int32", + "x-nullable": true + } + }, + "required": [ + "name", + "vmSize", + "osType", + "osSku", + "storageProfile", + "distro", + "orchestratorVersion", + "vnetCidrs", + "vnetSubnetID", + "mode", + "architecture", + "maxPods" + ] + }, + "CustomKubeletConfig": { + "type": "object", + "properties": { + "cpuManagerPolicy": { + "type": "string", + "x-nullable": true + }, + "cpuCfsQuota": { + "type": "boolean", + "x-nullable": true + }, + "cpuCfsQuotaPeriod": { + "type": "string", + "x-nullable": true + }, + "imageGcHighThreshold": { + "type": "integer", + "format": "int32", + "x-nullable": true + }, + "imageGcLowThreshold": { + "type": "integer", + "format": "int32", + "x-nullable": true + }, + "topologyManagerPolicy": { + "type": "string", + "x-nullable": true + }, + "allowedUnsafeSysctls": { + "type": "array", + "items": { + "type": "string" + }, + "x-nullable": true + }, + "failSwapOn": { + "type": "boolean", + "x-nullable": true + }, + "containerLogMaxSizeMB": { + "type": "integer", + "format": "int32", + "x-nullable": true + }, + "containerLogMaxFiles": { + "type": "integer", + "format": "int32", + "x-nullable": true + }, + "podMaxPids": { + "type": "integer", + "format": "int32", + "x-nullable": true + }, + "cpuReserved": { + "type": "integer", + "format": "int32", + "x-nullable": true + }, + "memoryReserved": { + "type": "integer", + "format": "int32", + "x-nullable": true + }, + "seccompDefault": { + "type": "string", + "x-nullable": true + } + } + }, + "CustomLinuxOSConfig": { + "type": "object", + "properties": { + "sysctls": { + "$ref": "#/definitions/SysctlConfig", + "x-nullable": true + }, + "transparentHugePageEnabled": { + "type": "string", + "x-nullable": true + }, + "transparentHugePageDefrag": { + "type": "string", + "x-nullable": true + }, + "swapFileSizeMB": { + "type": "integer", + "format": "int32", + "x-nullable": true + }, + "ulimits": { + "$ref": "#/definitions/UlimitConfig", + "x-nullable": true + } + } + }, + "SysctlConfig": { + "type": "object", + "properties": { + "netCoreSomaxconn": { + "type": "integer", + "format": "int32", + "x-nullable": true + }, + "netCoreNetdevMaxBacklog": { + "type": "integer", + "format": "int32", + "x-nullable": true + }, + "netCoreRmemDefault": { + "type": "integer", + "format": "int32", + "x-nullable": true + }, + "netCoreRmemMax": { + "type": "integer", + "format": "int32", + "x-nullable": true + }, + "netCoreWmemDefault": { + "type": "integer", + "format": "int32", + "x-nullable": true + }, + "netCoreWmemMax": { + "type": "integer", + "format": "int32", + "x-nullable": true + }, + "netCoreOptmemMax": { + "type": "integer", + "format": "int32", + "x-nullable": true + }, + "netIpv4TcpMaxSynBacklog": { + "type": "integer", + "format": "int32", + "x-nullable": true + }, + "netIpv4TcpMaxTwBuckets": { + "type": "integer", + "format": "int32", + "x-nullable": true + }, + "netIpv4TcpFinTimeout": { + "type": "integer", + "format": "int32", + "x-nullable": true + }, + "netIpv4TcpKeepaliveTime": { + "type": "integer", + "format": "int32", + "x-nullable": true + }, + "netIpv4TcpKeepaliveProbes": { + "type": "integer", + "format": "int32", + "x-nullable": true + }, + "netIpv4TcpkeepaliveIntvl": { + "type": "integer", + "format": "int32", + "x-nullable": true + }, + "netIpv4TcpTwReuse": { + "type": "boolean", + "x-nullable": true + }, + "netIpv4IpLocalPortRange": { + "type": "string", + "x-nullable": true + }, + "netIpv4NeighDefaultGcThresh1": { + "type": "integer", + "format": "int32", + "x-nullable": true + }, + "netIpv4NeighDefaultGcThresh2": { + "type": "integer", + "format": "int32", + "x-nullable": true + }, + "netIpv4NeighDefaultGcThresh3": { + "type": "integer", + "format": "int32", + "x-nullable": true + }, + "netNetfilterNfConntrackMax": { + "type": "integer", + "format": "int32", + "x-nullable": true + }, + "netNetfilterNfConntrackBuckets": { + "type": "integer", + "format": "int32", + "x-nullable": true + }, + "fsInotifyMaxUserWatches": { + "type": "integer", + "format": "int32", + "x-nullable": true + }, + "fsFileMax": { + "type": "integer", + "format": "int32", + "x-nullable": true + }, + "fsAioMaxNr": { + "type": "integer", + "format": "int32", + "x-nullable": true + }, + "fsNrOpen": { + "type": "integer", + "format": "int32", + "x-nullable": true + }, + "kernelThreadsMax": { + "type": "integer", + "format": "int32", + "x-nullable": true + }, + "vmMaxMapCount": { + "type": "integer", + "format": "int32", + "x-nullable": true + }, + "vmSwappiness": { + "type": "integer", + "format": "int32", + "x-nullable": true + }, + "vmVfsCachePressure": { + "type": "integer", + "format": "int32", + "x-nullable": true + } + } + }, + "UlimitConfig": { + "type": "object", + "properties": { + "noFile": { + "type": "string", + "x-nullable": true + }, + "maxLockedMemory": { + "type": "string", + "x-nullable": true + } + } + }, + "AgentPoolWindowsProfile": { + "type": "object", + "properties": { + "disableOutboundNat": { + "type": "boolean", + "x-nullable": true + }, + "containerdPackage": { + "type": "string", + "x-nullable": true + }, + "enableNextGenNetworking": { + "type": "boolean", + "x-nullable": true + } + } + }, + "AgentPoolSecurityProfile": { + "type": "object", + "properties": { + "sshAccess": { + "type": "integer", + "format": "int32", + "x-nullable": true + }, + "enableVTPM": { + "type": "boolean", + "x-nullable": true + }, + "enableSecureBoot": { + "type": "boolean", + "x-nullable": true + } + } + }, + "GPUProfile": { + "type": "object", + "properties": { + "installGPUDriver": { + "type": "boolean", + "x-nullable": true + }, + "driverType": { + "type": "integer", + "format": "int32", + "x-nullable": true + } + } + }, + "ArtifactStreamingProfile": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "x-nullable": true + } + } + }, + "ProvisionHelperValues": { + "type": "object", + "properties": { + "skuCPU": { + "type": "number", + "format": "double", + "x-nullable": true + }, + "skuMemory": { + "type": "number", + "format": "double", + "x-nullable": true + } + }, + "required": [ + "skuCPU", + "skuMemory" + ] + }, + "SigImageConfigTemplate": { + "type": "object", + "properties": { + "resourceGroup": { + "type": "string", + "x-nullable": true + }, + "gallery": { + "type": "string", + "x-nullable": true + }, + "definition": { + "type": "string", + "x-nullable": true + }, + "version": { + "type": "string", + "x-nullable": true + } + } + }, + "SigImageConfig": { + "type": "object", + "properties": { + "sigImageConfigTemplate": { + "$ref": "#/definitions/SigImageConfigTemplate", + "x-nullable": true + }, + "subscriptionID": { + "type": "string", + "x-nullable": true + } + } + }, + "AzureOSImageConfig": { + "type": "object", + "properties": { + "imageOffer": { + "type": "string", + "x-nullable": true + }, + "imageSku": { + "type": "string", + "x-nullable": true + }, + "imagePublisher": { + "type": "string", + "x-nullable": true + }, + "imageVersion": { + "type": "string", + "x-nullable": true + } + } + }, + "NodeBootstrapping": { + "type": "object", + "properties": { + "customData": { + "type": "string", + "x-nullable": true + }, + "cse": { + "type": "string", + "x-nullable": true + }, + "osImageConfig": { + "$ref": "#/definitions/AzureOSImageConfig", + "x-nullable": true + }, + "sigImageConfig": { + "$ref": "#/definitions/SigImageConfig", + "x-nullable": true + } + }, + "required": [ + "customData", + "cse", + "osImageConfig", + "sigImageConfig" + ] + } + }, + "parameters": { + "SubscriptionIdParameter": { + "name": "subscriptionId", + "in": "path", + "required": true, + "type": "string" + }, + "ResourceGroupNameParameter": { + "name": "resourceGroupName", + "in": "path", + "required": true, + "type": "string" + }, + "ResourceNameParameter": { + "name": "resourceName", + "in": "path", + "required": true, + "type": "string" + } + } +} diff --git a/pkg/test/environment.go b/pkg/test/environment.go index 39257ede4..30b6777bf 100644 --- a/pkg/test/environment.go +++ b/pkg/test/environment.go @@ -118,10 +118,12 @@ func NewRegionalEnvironment(ctx context.Context, env *coretest.Environment, regi testOptions.ClusterEndpoint, "test-tenant", "test-subscription", - "test-userAssignedIdentity", + "test-cluster-resource-group", + "test-kubelet-identity-client-id", testOptions.NodeResourceGroup, region, "test-vnet-guid", + testOptions.ProvisionMode, ) loadBalancerProvider := loadbalancer.NewProvider( loadBalancersAPI, @@ -146,6 +148,7 @@ func NewRegionalEnvironment(ctx context.Context, env *coretest.Environment, regi region, testOptions.NodeResourceGroup, "", // subscriptionID + testOptions.ProvisionMode, ) return &Environment{ diff --git a/pkg/test/options.go b/pkg/test/options.go index ab0a10ba7..32e2b2734 100644 --- a/pkg/test/options.go +++ b/pkg/test/options.go @@ -39,6 +39,8 @@ type OptionsFields struct { NodeIdentities []string SubnetID *string NodeResourceGroup *string + ProvisionMode *string + NodeBootstrappingServerURL *string } func Options(overrides ...OptionsFields) *azoptions.Options { @@ -62,5 +64,6 @@ func Options(overrides ...OptionsFields) *azoptions.Options { NodeIdentities: options.NodeIdentities, SubnetID: lo.FromPtrOr(options.SubnetID, "/subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/sillygeese/providers/Microsoft.Network/virtualNetworks/karpentervnet/subnets/karpentersub"), NodeResourceGroup: lo.FromPtrOr(options.NodeResourceGroup, "test-resourceGroup"), + ProvisionMode: lo.FromPtrOr(options.ProvisionMode, "aksscriptless"), } } diff --git a/pkg/utils/gpu.go b/pkg/utils/gpu.go index f9a7dc416..75dc5bcc7 100644 --- a/pkg/utils/gpu.go +++ b/pkg/utils/gpu.go @@ -33,7 +33,7 @@ const ( ) func GetAKSGPUImageSHA(size string) string { - if useGridDrivers(size) { + if UseGridDrivers(size) { return AKSGPUGridSHA } return AKSGPUCudaSHA @@ -142,7 +142,7 @@ func IsMarinerEnabledGPUSKU(vmSize string) bool { // NVv1 seems to run with CUDA, NVv5 requires GRID. // NVv3 is untested on AKS, NVv4 is AMD so n/a, and NVv2 no longer seems to exist (?). func GetGPUDriverVersion(size string) string { - if useGridDrivers(size) { + if UseGridDrivers(size) { return Nvidia535GridDriverVersion } if isStandardNCv1(size) { @@ -156,7 +156,7 @@ func isStandardNCv1(size string) bool { return strings.HasPrefix(tmp, "standard_nc") && !strings.Contains(tmp, "_v") } -func useGridDrivers(size string) bool { +func UseGridDrivers(size string) bool { return ConvergedGPUDriverSizes[strings.ToLower(size)] }