handle Interface already exists due to SetNetworkStatus failed

Signed-off-by: zhuanlan <[email protected]>

pkg/controller/pod.go

@@ -201,9 +201,8 @@ func (pnc *PodNetworksController) processNextWorkItem() bool {
 
 	var results []annotations.AttachmentResult
 	var pod *corev1.Pod
-	defer func() {
-		pnc.handleResult(err, podNamespacedName, pod, results)
-	}()
+	var netnsPath, podSandboxID string
+	var addFailedIndex int = -1
 
 	pod, err = pnc.podsLister.Pods(podNamespace).Get(podName)
 	if err != nil {
@@ -219,13 +218,13 @@ func (pnc *PodNetworksController) processNextWorkItem() bool {
 	indexedNetworkSelectionElements := annotations.IndexNetworkSelectionElements(networkSelectionElements)
 	indexedNetworkStatus := annotations.IndexNetworkStatus(networkStatus)
 
-	netnsPath, err := pnc.containerRuntime.NetworkNamespace(ctx, string(pod.UID))
+	netnsPath, err = pnc.containerRuntime.NetworkNamespace(ctx, string(pod.UID))
 	if err != nil {
 		klog.Errorf("failed to figure out the pod's network namespace: %v", err)
 		return true
 	}
 
-	podSandboxID, err := pnc.containerRuntime.PodSandboxID(ctx, string(pod.UID))
+	podSandboxID, err = pnc.containerRuntime.PodSandboxID(ctx, string(pod.UID))
 	if err != nil {
 		klog.Errorf("failed to figure out the PodSandboxID: %v", err)
 		return true
@@ -244,6 +243,10 @@ func (pnc *PodNetworksController) processNextWorkItem() bool {
 		}
 	}
 
+	defer func() {
+		pnc.handleResult(err, podNamespacedName, pod, results, attachmentsToAdd, netnsPath, podSandboxID, addFailedIndex)
+	}()
+
 	if len(attachmentsToAdd) > 0 {
 		results, err = pnc.handleDynamicInterfaceRequest(
 			&DynamicAttachmentRequest{
@@ -255,6 +258,7 @@ func (pnc *PodNetworksController) processNextWorkItem() bool {
 			})
 		if err != nil {
 			klog.Errorf("error adding attachments: %v", err)
+			addFailedIndex = len(results)
 			return true
 		}
 	}
@@ -312,12 +316,17 @@ func (pnc *PodNetworksController) handleResult(
 	namespacedPodName *string,
 	pod *corev1.Pod,
 	results []annotations.AttachmentResult,
+	attachmentsToAdd []nadv1.NetworkSelectionElement,
+	netnsPath, podSandboxID string,
+	addfailedIndex int,
 ) {
 	namespacedPodNameString := "<nil>"
 	if namespacedPodName != nil {
 		namespacedPodNameString = *namespacedPodName
 	}
 
+	var shouldDeleteattachmentsAdded bool
+
 	if results != nil {
 		updatedStatus, podNetworkStatusUpdateError := annotations.UpdatePodNetworkStatus(pod, results)
 		if podNetworkStatusUpdateError != nil {
@@ -326,6 +335,7 @@ func (pnc *PodNetworksController) handleResult(
 				namespacedPodNameString,
 				podNetworkStatusUpdateError,
 			)
+			shouldDeleteattachmentsAdded = true
 		}
 
 		if setNetworkStatusError := nadutils.SetNetworkStatus(
@@ -334,13 +344,35 @@ func (pnc *PodNetworksController) handleResult(
 			updatedStatus,
 		); setNetworkStatusError != nil {
 			klog.Errorf("error updating pod %s network status: %v", namespacedPodNameString, setNetworkStatusError)
+			shouldDeleteattachmentsAdded = true
+		}
+	}
+
+	var attachmentsToCancal []nadv1.NetworkSelectionElement
+	if shouldDeleteattachmentsAdded {
+		attachmentsToCancal = attachmentsToAdd
+	} else if addfailedIndex != -1 {
+		attachmentsToCancal = attachmentsToAdd[addfailedIndex:]
+	}
+	if len(attachmentsToCancal) > 0 {
+		_, deleteattachmentsError := pnc.handleDynamicInterfaceRequest(
+			&DynamicAttachmentRequest{
+				Pod:          pod,
+				Attachments:  attachmentsToCancal,
+				Type:         remove,
+				PodNetNS:     netnsPath,
+				PodSandboxID: podSandboxID,
+			})
+		if deleteattachmentsError != nil {
+			klog.Errorf("error removing attachments: %v after updating pod %s network status failed", err, namespacedPodNameString)
 		}
 	}
 
 	if err == nil {
 		pnc.workqueue.Forget(namespacedPodName)
 		return
 	}
+
 
 	currentRetries := pnc.workqueue.NumRequeues(namespacedPodName)
 	if currentRetries <= maxRetries {

pkg/controller/pod_test.go

@@ -391,6 +391,15 @@ var _ = Describe("Dynamic Attachment controller", func() {
 					)
 					Eventually(<-eventRecorder.Events).Should(Equal(expectedEventPayload))
 
+					expectedEventPayload = fmt.Sprintf(
+						"Warning FailedRemovingInterface pod [%s]: failed removing interface %s from network: %s",
+						annotations.NamespacedName(namespace, podName),
+						"net-non-existing",
+						networkToAdd,
+					)
+					Eventually(<-eventRecorder.Events).Should(Equal(expectedEventPayload))
+
+
 					// reconciliation requeued without adding the next interface (net1).
 					expectedEventPayload = fmt.Sprintf(
 						"Warning FailedAddingInterface pod [%s]: failed adding interface %s to network: %s",