Merge branch 'main' into charliedmcb/disableFlakyTestPVC

Azure · Nov 17, 2023 · 275d2ad · 275d2ad
2 parents 5f020b8 + 334b902
commit 275d2ad
Show file tree

Hide file tree

Showing 35 changed files with 482 additions and 224 deletions.
diff --git a/.github/actions/e2e/create-acr/action.yaml b/.github/actions/e2e/create-acr/action.yaml
@@ -19,6 +19,10 @@ inputs:
   git_ref:
     description: "The git commit, tag, or branch to check out"
     required: false
+  location:
+    type: string
+    description: "the azure location to run the e2e test in"
+    default: "westus2"
 runs:
   using: "composite"
   steps:
@@ -36,4 +40,4 @@ runs:
     run: az account set --subscription ${{ inputs.subscription-id }}
   - name: create ACR
     shell: bash
-    run: AZURE_RESOURCE_GROUP=${{ inputs.resource_group }} AZURE_ACR_NAME=${{ inputs.acr_name }} make az-mkacr
+    run: AZURE_RESOURCE_GROUP=${{ inputs.resource_group }} AZURE_ACR_NAME=${{ inputs.acr_name }} AZURE_LOCATION=${{ inputs.location }} make az-mkacr
diff --git a/.github/actions/e2e/create-cluster/action.yaml b/.github/actions/e2e/create-cluster/action.yaml
@@ -14,9 +14,6 @@ inputs:
   subscription-id:
     description: 
     required: true
-  # region:
-  #   description: "Region to create aks cluster"
-  #   required: true
   resource_group:
     description: "Name of the resource group to create the cluster within"
     required: true
@@ -29,6 +26,10 @@ inputs:
   git_ref:
     description: "The git commit, tag, or branch to check out"
     required: false
+  location:
+    type: string
+    description: "the azure location to run the e2e test in"
+    default: "westus2"
 runs:
   using: "composite"
   steps:
@@ -46,7 +47,7 @@ runs:
     run: az account set --subscription ${{ inputs.subscription-id }}
   - name: create cluster
     shell: bash
-    run: AZURE_CLUSTER_NAME=${{ inputs.cluster_name }} AZURE_RESOURCE_GROUP=${{ inputs.resource_group }} AZURE_ACR_NAME=${{ inputs.acr_name }} make az-mkaks-cilium
+    run: AZURE_CLUSTER_NAME=${{ inputs.cluster_name }} AZURE_RESOURCE_GROUP=${{ inputs.resource_group }} AZURE_ACR_NAME=${{ inputs.acr_name }} AZURE_LOCATION=${{ inputs.location }} make az-mkaks-cilium
   - name: az login 2
     uses: azure/login@v1
     with:
@@ -55,4 +56,4 @@ runs:
       subscription-id: ${{ inputs.subscription-id }}
   - name: update azure perms
     shell: bash
-    run: AZURE_CLUSTER_NAME=${{ inputs.cluster_name }} AZURE_RESOURCE_GROUP=${{ inputs.resource_group }} make az-perm
+    run: AZURE_CLUSTER_NAME=${{ inputs.cluster_name }} AZURE_RESOURCE_GROUP=${{ inputs.resource_group }} AZURE_LOCATION=${{ inputs.location }} make az-perm
diff --git a/.github/actions/e2e/install-karpenter/action.yaml b/.github/actions/e2e/install-karpenter/action.yaml
@@ -25,6 +25,10 @@ inputs:
   git_ref:
     description: "The git commit, tag, or branch to check out"
     required: false
+  location:
+    type: string
+    description: "the azure location to run the e2e test in"
+    default: "westus2"
 runs:
   using: "composite"
   steps:
@@ -42,15 +46,10 @@ runs:
     run: az account set --subscription ${{ inputs.subscription-id }}
   - name: patch skaffold and cilium
     shell: bash
-    run: AZURE_CLUSTER_NAME=${{ inputs.cluster_name }} AZURE_RESOURCE_GROUP=${{ inputs.resource_group }} make az-patch-skaffold-azureoverlay
+    run: AZURE_CLUSTER_NAME=${{ inputs.cluster_name }} AZURE_RESOURCE_GROUP=${{ inputs.resource_group }} AZURE_LOCATION=${{ inputs.location }} make az-patch-skaffold-azureoverlay
   - name: deploy karpenter to cluster
     shell: bash
     run: AZURE_ACR_NAME=${{ inputs.acr_name }} make az-run
-  - name: wait for pods to settle
-    shell: bash
-    run: |
-      ${GITHUB_WORKSPACE}/.github/actions/e2e/install-karpenter/waitforgatekeeper.sh
-      sleep 10
   - name: taint nodes as karpenter-system
     shell: bash
     run: |

diff --git a/.github/actions/e2e/install-karpenter/waitforgatekeeper.sh b/.github/actions/e2e/install-karpenter/waitforgatekeeper.sh
diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml
@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        k8sVersion: ["1.25.x", "1.26.x", "1.27.x"]
+        k8sVersion: ["1.25.x", "1.26.x", "1.27.x", "1.28.x"]
     env:
       K8S_VERSION: ${{ matrix.k8sVersion }}
     steps:
@@ -18,7 +18,9 @@ jobs:
       with:
         k8sVersion: ${{ matrix.k8sVersion }}    
     - run: K8S_VERSION=${{ matrix.k8sVersion }} make ci-test
-#    - uses: shogo82148/actions-goveralls@v1
-#      if: env.K8S_VERSION == '1.25.x'
-#      with:
-#        path-to-profile: coverage.out
+    - name: Send coverage
+      # should only send converage once https://docs.coveralls.io/parallel-builds
+      if: matrix.k8sVersion == '1.27.x'
+      env:
+        COVERALLS_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: goveralls -coverprofile=coverage.out -service=github
diff --git a/.github/workflows/e2e-matrix-trigger.yaml b/.github/workflows/e2e-matrix-trigger.yaml
@@ -1,6 +1,11 @@
 name: E2EMatrixTrigger
 on:
   workflow_dispatch:
+    inputs:
+      location:
+        type: string
+        description: "the azure location to run the e2e test in"
+        default: "westus2"
   push:
     branches: [main]
   workflow_run:
@@ -19,6 +24,7 @@ jobs:
     uses: ./.github/workflows/e2e-matrix.yaml
     with:
       git_ref: ${{ needs.resolve.outputs.GIT_REF }}
+      location: ${{ inputs.location || 'westus2' }}
     secrets:
       E2E_CLIENT_ID: ${{ secrets.E2E_CLIENT_ID }}
       E2E_TENANT_ID: ${{ secrets.E2E_TENANT_ID }}

diff --git a/.github/workflows/e2e-matrix.yaml b/.github/workflows/e2e-matrix.yaml
@@ -4,9 +4,10 @@ on:
     inputs:
       git_ref:
         type: string
-  #     region:
-  #       type: string
-  #       default: "us-east-2"
+      location:
+        type: string
+        description: "the azure location to run the e2e test in"
+        default: "westus2"
   #     k8s_version:
   #       type: string
   #       default: "1.27"
@@ -39,7 +40,7 @@ jobs:
       git_ref: ${{ inputs.git_ref }}
       suite: ${{ matrix.suite }}
       hash: ${{ needs.initialize-generative-params.outputs.E2E_HASH }}
-  #     region: ${{ inputs.region }}
+      location: ${{ inputs.location }}
   #     k8s_version: ${{ inputs.k8s_version }}
     secrets:
       E2E_CLIENT_ID: ${{ secrets.E2E_CLIENT_ID }}

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
@@ -2,9 +2,6 @@ name: E2E
 on:
   workflow_call:
     inputs:
-    #   region:
-    #     type: string
-    #     default: "us-east-2"
       git_ref:
         type: string
       suite:
@@ -13,6 +10,10 @@ on:
       hash:
         type: string
         required: true
+      location:
+        type: string
+        description: "the azure location to run the e2e test in"
+        default: "westus2"
     #   k8s_version:
     #     type: string
     #     default: "1.27"
@@ -75,6 +76,7 @@ jobs:
           resource_group: ${{ env.RG_NAME }}
           acr_name: ${{ env.ACR_NAME }}
           git_ref: ${{ inputs.git_ref }}
+          location: ${{ inputs.location }}
       - name: add jitter on cluster creation
         run: |
           # Creating jitter so that we can stagger cluster creation to avoid throttling
@@ -90,11 +92,11 @@ jobs:
           client-id: ${{ secrets.E2E_CLIENT_ID }}
           tenant-id: ${{ secrets.E2E_TENANT_ID }}
           subscription-id: ${{ secrets.E2E_SUBSCRIPTION_ID }}
-          # region: westus2
           resource_group: ${{ env.RG_NAME }}
           cluster_name: ${{ env.CLUSTER_NAME }}
           acr_name: ${{ env.ACR_NAME }}
           git_ref: ${{ inputs.git_ref }}
+          location: ${{ inputs.location }}
       - name: build and publish karpenter
         shell: bash
         run: AZURE_ACR_NAME=${{ env.ACR_NAME }} make az-build
@@ -108,6 +110,7 @@ jobs:
           cluster_name: ${{ env.CLUSTER_NAME }}
           acr_name: ${{ env.ACR_NAME }}
           git_ref: ${{ inputs.git_ref }}
+          location: ${{ inputs.location }}
       - name: run the ${{ inputs.suite }} test suite
         if: inputs.suite != 'Nonbehavioral'
         run: |
@@ -130,7 +133,6 @@ jobs:
           client-id: ${{ secrets.E2E_CLIENT_ID }}
           tenant-id: ${{ secrets.E2E_TENANT_ID }}
           subscription-id: ${{ secrets.E2E_SUBSCRIPTION_ID }}
-          # region: westus2
           resource_group: ${{ env.RG_NAME }}
           cluster_name: ${{ env.CLUSTER_NAME }}
           acr_name: ${{ env.ACR_NAME }}

diff --git a/.github/workflows/resolve-args.yaml b/.github/workflows/resolve-args.yaml
@@ -18,6 +18,8 @@ jobs:
         run: |
           if [[ "${{ github.event_name }}" == "workflow_run" ]]; then
             echo GIT_REF="$(tail -n 1 /tmp/artifacts/metadata.txt)" >> "$GITHUB_OUTPUT"
+          elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
+            echo GIT_REF="${{ github.sha }}" >> "$GITHUB_OUTPUT"
           else
             echo GIT_REF="" >> "$GITHUB_OUTPUT"
           fi
diff --git a/Makefile-az.mk b/Makefile-az.mk
@@ -25,7 +25,7 @@ az-mkacr: az-mkrg ## Create test ACR
 	az acr login  --name $(AZURE_ACR_NAME)
 
 az-mkaks: az-mkacr ## Create test AKS cluster (with --vm-set-type AvailabilitySet for compatibility with standalone VMs)
-	az aks create          --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) --attach-acr $(AZURE_ACR_NAME) \
+	az aks create          --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) --attach-acr $(AZURE_ACR_NAME) --location $(AZURE_LOCATION) \
 		--enable-managed-identity --node-count 3 --generate-ssh-keys --vm-set-type AvailabilitySet -o none
 	az aks get-credentials --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) --overwrite-existing
 	skaffold config set default-repo $(AZURE_ACR_NAME).azurecr.io/karpenter
@@ -94,10 +94,10 @@ az-rmvmss-vms: ## Delete all VMs in VMSS Flex (use with care!)
 az-perm: ## Create role assignments to let Karpenter manage VMs and Network
 	# Note (charliedmcb): need to be objectId for E2E workflow as the pipeline identity doesn't have permissions to "query Graph API"
 	$(eval AZURE_OBJECT_ID=$(shell az aks show --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) | jq  -r ".identityProfile.kubeletidentity.objectId"))
-	az role assignment create --assignee $(AZURE_OBJECT_ID) --resource-group $(AZURE_RESOURCE_GROUP_MC) --role "Virtual Machine Contributor"
-	az role assignment create --assignee $(AZURE_OBJECT_ID) --resource-group $(AZURE_RESOURCE_GROUP_MC) --role "Network Contributor"
-	az role assignment create --assignee $(AZURE_OBJECT_ID) --resource-group $(AZURE_RESOURCE_GROUP_MC) --role "Managed Identity Operator"
-	az role assignment create --assignee $(AZURE_OBJECT_ID) --resource-group $(AZURE_RESOURCE_GROUP)    --role "Network Contributor" # in some case we create vnet here
+	az role assignment create --assignee $(AZURE_OBJECT_ID) --scope /subscriptions/$(AZURE_SUBSCRIPTION_ID)/resourceGroups/$(AZURE_RESOURCE_GROUP_MC) --role "Virtual Machine Contributor"
+	az role assignment create --assignee $(AZURE_OBJECT_ID) --scope /subscriptions/$(AZURE_SUBSCRIPTION_ID)/resourceGroups/$(AZURE_RESOURCE_GROUP_MC) --role "Network Contributor"
+	az role assignment create --assignee $(AZURE_OBJECT_ID) --scope /subscriptions/$(AZURE_SUBSCRIPTION_ID)/resourceGroups/$(AZURE_RESOURCE_GROUP_MC) --role "Managed Identity Operator"
+	az role assignment create --assignee $(AZURE_OBJECT_ID) --scope /subscriptions/$(AZURE_SUBSCRIPTION_ID)/resourceGroups/$(AZURE_RESOURCE_GROUP)    --role "Network Contributor" # in some case we create vnet here
 	@echo Consider "make az-patch-skaffold"!
 
 az-perm-acr:
@@ -263,12 +263,6 @@ az-klogs: ## Karpenter logs
 az-kevents: ## Karpenter events
 	kubectl get events -A --field-selector source=karpenter
 
-az-provision-gpus:
-	kubectl apply -f examples/workloads/device-plugin.yaml
-	kubectl apply -f examples/provisioner/gpu-provisioner.yaml
-	kubectl apply -f examples/workloads/samples-mnist.yaml
-
-
 az-node-viewer: ## Watch nodes using eks-node-viewer
 	eks-node-viewer --disable-pricing --node-selector "karpenter.sh/nodepool" # --resources cpu,memory
 

diff --git a/charts/karpenter/templates/webhooks-core.yaml b/charts/karpenter/templates/webhooks-core.yaml
@@ -38,7 +38,6 @@ webhooks:
         operations:
           - CREATE
           - UPDATE
-          - DELETE
         resources:
           - nodeclaims
           - nodeclaims/status

diff --git a/examples/v1beta1/general-purpose.yaml b/examples/v1beta1/general-purpose.yaml
@@ -18,9 +18,6 @@ spec:
         - key: kubernetes.io/os
           operator: In
           values: ["linux"]
-        - key: karpenter.azure.com/sku-hyperv-generation
-          operator: In
-          values: ["2"]
         - key: karpenter.sh/capacity-type
           operator: In
           values: ["on-demand"]

diff --git a/examples/v1beta1/multi-arch.yaml b/examples/v1beta1/multi-arch.yaml
@@ -15,9 +15,6 @@ spec:
         - key: kubernetes.io/arch
           operator: In
           values: ["arm64"]
-        - key: karpenter.azure.com/sku-hyperv-generation
-          operator: In
-          values: ["2"]
         - key: kubernetes.io/os
           operator: In
           values: ["linux"]
@@ -45,9 +42,6 @@ spec:
         - key: kubernetes.io/arch
           operator: In
           values: ["amd64"]
-        - key: karpenter.azure.com/sku-hyperv-generation
-          operator: In
-          values: ["2"]
         - key: kubernetes.io/os
           operator: In
           values: ["linux"]

diff --git a/examples/v1beta1/system-surge.yaml b/examples/v1beta1/system-surge.yaml
@@ -22,9 +22,6 @@ spec:
         - key: kubernetes.io/arch
           operator: In
           values: ["amd64"]
-        - key: karpenter.azure.com/sku-hyperv-generation
-          operator: In
-          values: ["2"]
         - key: kubernetes.io/os
           operator: In
           values: ["linux"]

diff --git a/go.mod b/go.mod
@@ -17,10 +17,11 @@ require (
 	github.com/Azure/go-autorest/autorest/adal v0.9.23
 	github.com/Azure/go-autorest/autorest/to v0.4.0
 	github.com/Azure/skewer v0.0.19
+	github.com/Pallinder/go-randomdata v1.2.0
 	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137
-	github.com/aws/karpenter-core v0.32.1
-	github.com/go-logr/logr v1.2.4
-	github.com/go-logr/zapr v1.2.4
+	github.com/aws/karpenter-core v0.32.2-0.20231109191441-e32aafc81fb5
+	github.com/go-logr/logr v1.3.0
+	github.com/go-logr/zapr v1.3.0
 	github.com/go-playground/validator/v10 v10.13.0
 	github.com/imdario/mergo v0.3.16
 	github.com/mitchellh/hashstructure/v2 v2.0.2
@@ -37,7 +38,7 @@ require (
 	k8s.io/apiextensions-apiserver v0.28.3
 	k8s.io/apimachinery v0.28.3
 	k8s.io/client-go v0.28.3
-	k8s.io/klog/v2 v2.100.1
+	k8s.io/klog/v2 v2.110.1
 	k8s.io/utils v0.0.0-20230726121419-3b25d923346b
 	knative.dev/pkg v0.0.0-20231010144348-ca8c009405dd
 	sigs.k8s.io/cloud-provider-azure v1.28.2
@@ -55,7 +56,6 @@ require (
 	github.com/Azure/go-autorest/logger v0.2.1 // indirect
 	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 // indirect
-	github.com/Pallinder/go-randomdata v1.2.0 // indirect
 	github.com/avast/retry-go v3.0.0+incompatible // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
@@ -115,11 +115,11 @@ require (
 	golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
 	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/oauth2 v0.13.0 // indirect
-	golang.org/x/sync v0.4.0 // indirect
+	golang.org/x/sync v0.5.0 // indirect
 	golang.org/x/sys v0.13.0 // indirect
 	golang.org/x/term v0.13.0 // indirect
-	golang.org/x/text v0.13.0 // indirect
-	golang.org/x/time v0.3.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/time v0.4.0 // indirect
 	golang.org/x/tools v0.14.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
 	google.golang.org/api v0.146.0 // indirect