Merge branch 'fix-k8s-api-network-restart-error-42' into 'release/4.2'

Introduce k8s network failure reason and remove previous network related error... See merge request weblogic-cloud/weblogic-kubernetes-operator!4831
oracle · Oct 9, 2024 · 99c6522 · 99c6522
2 parents 55de432 + db297fe
commit 99c6522
Show file tree

Hide file tree

Showing 14 changed files with 57 additions and 2 deletions.
diff --git a/documentation/domains/Domain.json b/documentation/domains/Domain.json
@@ -238,6 +238,7 @@
             "DomainInvalid",
             "Introspection",
             "Kubernetes",
+            "KubernetesNetworkException",
             "ServerPod",
             "PersistentVolumeClaim",
             "ReplicasTooHigh",

diff --git a/kubernetes/crd/domain-crd.yaml b/kubernetes/crd/domain-crd.yaml
@@ -5,7 +5,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    weblogic.sha256: 3cad99b24fb84de65dc38d7734ce269fb7058c3b6aed32b85ef590e142921635
+    weblogic.sha256: 8cd5a2176fe99b104c82048d750d42f1130341bdfdba825493bc64de45025424
   name: domains.weblogic.oracle
 spec:
   group: weblogic.oracle
@@ -10322,6 +10322,7 @@ spec:
                       - DomainInvalid
                       - Introspection
                       - Kubernetes
+                      - KubernetesNetworkException
                       - ServerPod
                       - PersistentVolumeClaim
                       - ReplicasTooHigh

diff --git a/operator/src/main/java/oracle/kubernetes/operator/StuckPodProcessing.java b/operator/src/main/java/oracle/kubernetes/operator/StuckPodProcessing.java
@@ -64,6 +64,7 @@ public PodListProcessing(String namespace, OffsetDateTime dateTime) {
 
     @Override
     public NextAction onSuccess(Packet packet, CallResponse<V1PodList> callResponse) {
+      clearExistingKubernetesNetworkException(packet);
       callResponse.getResult().getItems().stream()
             .filter(pod -> isStuck(pod, now))
             .forEach(pod -> addStuckPodToPacket(packet, pod));
@@ -156,6 +157,7 @@ public ForcedDeleteResponseStep(String name, String namespace, String domainUID)
     @Override
     @SuppressWarnings("try")
     public NextAction onSuccess(Packet packet, CallResponse<Object> callResponse) {
+      clearExistingKubernetesNetworkException(packet);
       try (ThreadLoggingContext ignored =
                ThreadLoggingContext.setThreadContext().namespace(namespace).domainUid(domainUID)) {
         LOGGER.info(POD_FORCE_DELETED, name, namespace);

diff --git a/operator/src/main/java/oracle/kubernetes/operator/WaitForReadyStep.java b/operator/src/main/java/oracle/kubernetes/operator/WaitForReadyStep.java
@@ -254,6 +254,7 @@ static class MakeRightDomainStep<V> extends DefaultResponseStep<V> {
 
     @Override
     public NextAction onSuccess(Packet packet, CallResponse<V> callResponse) {
+      clearExistingKubernetesNetworkException(packet);
       MakeRightDomainOperation makeRightDomainOperation =
               (MakeRightDomainOperation)packet.get(MAKE_RIGHT_DOMAIN_OPERATION);
       if (makeRightDomainOperation != null) {

diff --git a/operator/src/main/java/oracle/kubernetes/operator/WebhookMain.java b/operator/src/main/java/oracle/kubernetes/operator/WebhookMain.java
@@ -212,6 +212,7 @@ private class  CrdPresenceResponseStep<L extends KubernetesListObject> extends D
 
     @Override
     public NextAction onSuccess(Packet packet, CallResponse<L> callResponse) {
+      clearExistingKubernetesNetworkException(packet);
       warnedOfCrdAbsence = false;
       crdPresenceCheckCount.set(0);
       return super.onSuccess(packet, callResponse);

diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/PersistentVolumeClaimHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/PersistentVolumeClaimHelper.java
@@ -144,6 +144,7 @@ private NextAction updateDomainStatus(Packet packet, CallResponse<V1PersistentVo
 
       @Override
       public NextAction onSuccess(Packet packet, CallResponse<V1PersistentVolumeClaim> callResponse) {
+        clearExistingKubernetesNetworkException(packet);
         logPersistentVolumeClaimCreated(messageKey);
         addPersistentVolumeClaimToRecord(callResponse.getResult());
         return doNext(packet);
@@ -164,6 +165,7 @@ public NextAction onFailure(Packet packet, CallResponse<V1PersistentVolumeClaim>
 
       @Override
       public NextAction onSuccess(Packet packet, CallResponse<V1PersistentVolumeClaim> callResponse) {
+        clearExistingKubernetesNetworkException(packet);
         DomainPresenceInfo info = packet.getSpi(DomainPresenceInfo.class);
         V1PersistentVolumeClaim persistentVolumeClaim = callResponse.getResult();
 

diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/PersistentVolumeHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/PersistentVolumeHelper.java
@@ -133,6 +133,7 @@ private NextAction updateDomainStatus(Packet packet, CallResponse<V1PersistentVo
 
       @Override
       public NextAction onSuccess(Packet packet, CallResponse<V1PersistentVolume> callResponse) {
+        clearExistingKubernetesNetworkException(packet);
         logPersistentVolumeCreated(messageKey);
         return doNext(packet);
       }
@@ -152,6 +153,7 @@ public NextAction onFailure(Packet packet, CallResponse<V1PersistentVolume> call
 
       @Override
       public NextAction onSuccess(Packet packet, CallResponse<V1PersistentVolume> callResponse) {
+        clearExistingKubernetesNetworkException(packet);
         DomainPresenceInfo info = packet.getSpi(DomainPresenceInfo.class);
         V1PersistentVolume persistentVolume = callResponse.getResult();
         if (persistentVolume == null) {

diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodDisruptionBudgetHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodDisruptionBudgetHelper.java
@@ -113,6 +113,7 @@ private NextAction updateDomainStatus(Packet packet, CallResponse<V1PodDisruptio
 
       @Override
       public NextAction onSuccess(Packet packet, CallResponse<V1PodDisruptionBudget> callResponse) {
+        clearExistingKubernetesNetworkException(packet);
         logPodDisruptionBudgetCreated(messageKey);
         addPodDisruptionBudgetToRecord(callResponse.getResult());
         return doNext(packet);
@@ -133,6 +134,7 @@ public NextAction onFailure(Packet packet, CallResponse<V1PodDisruptionBudget> c
 
       @Override
       public NextAction onSuccess(Packet packet, CallResponse<V1PodDisruptionBudget> callResponse) {
+        clearExistingKubernetesNetworkException(packet);
         V1PodDisruptionBudget podDisruptionBudget = callResponse.getResult();
         if (podDisruptionBudget == null) {
           removePodDisruptionBudgetFromRecord();
@@ -157,6 +159,7 @@ public NextAction onFailure(Packet packet, CallResponse<V1PodDisruptionBudget> c
 
       @Override
       public NextAction onSuccess(Packet packet, CallResponse<V1PodDisruptionBudget> callResponse) {
+        clearExistingKubernetesNetworkException(packet);
         logPodDisruptionBudgetPatched();
         return doNext(packet);
       }

diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/ResponseStep.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/ResponseStep.java
@@ -3,6 +3,8 @@
 
 package oracle.kubernetes.operator.helpers;
 
+import java.net.ConnectException;
+import java.net.SocketTimeoutException;
 import java.util.Collections;
 import java.util.Optional;
 import java.util.stream.Collectors;
@@ -25,6 +27,7 @@
 import oracle.kubernetes.operator.work.Packet;
 import oracle.kubernetes.operator.work.Step;
 import oracle.kubernetes.weblogic.domain.model.DomainCondition;
+import oracle.kubernetes.weblogic.domain.model.DomainFailureReason;
 import oracle.kubernetes.weblogic.domain.model.DomainResource;
 
 import static oracle.kubernetes.common.CommonConstants.CRD;
@@ -37,6 +40,7 @@
 import static oracle.kubernetes.operator.calls.AsyncRequestStep.accessContinue;
 import static oracle.kubernetes.weblogic.domain.model.DomainConditionType.FAILED;
 import static oracle.kubernetes.weblogic.domain.model.DomainFailureReason.KUBERNETES;
+import static oracle.kubernetes.weblogic.domain.model.DomainFailureReason.KUBERNETES_NETWORK_EXCEPTION;
 
 /**
  * Step to receive response of Kubernetes API server call.
@@ -80,6 +84,20 @@ public final void setPrevious(Step previousStep) {
     this.previousStep = previousStep;
   }
 
+  /**
+   * Clear out any existing Kubernetes network exception (ConnectException and SocketTimeoutException).
+   *
+   * @param packet packet
+   */
+  public static void clearExistingKubernetesNetworkException(Packet packet) {
+    Optional.ofNullable(packet.getSpi(DomainPresenceInfo.class))
+        .map(DomainPresenceInfo::getDomain)
+        .map(DomainResource::getStatus)
+        .ifPresent(status -> status.removeConditionsMatching(
+            c -> c.hasType(FAILED) && KUBERNETES_NETWORK_EXCEPTION == c.getReason()));
+  }
+
+
   @Override
   public final NextAction apply(Packet packet) {
     NextAction nextAction = getActionForCallResponse(packet);
@@ -233,7 +251,16 @@ private void addDomainFailureStatus(Packet packet, RequestParams requestParams,
 
   private void updateFailureStatus(
       @Nonnull DomainResource domain, RequestParams requestParams, ApiException apiException) {
-    DomainCondition condition = new DomainCondition(FAILED).withFailureInfo(domain.getSpec()).withReason(KUBERNETES)
+    DomainFailureReason reason = KUBERNETES;
+    if (apiException != null) {
+      LOGGER.fine("updateFailureStatus: apiException: " + apiException.getCause());
+      LOGGER.fine("updateFailureStatus: status code: " + apiException.getCode());
+    }
+    if (apiException != null && (apiException.getCause() instanceof ConnectException
+        || apiException.getCause() instanceof SocketTimeoutException)) {
+      reason = DomainFailureReason.KUBERNETES_NETWORK_EXCEPTION;
+    }
+    DomainCondition condition = new DomainCondition(FAILED).withFailureInfo(domain.getSpec()).withReason(reason)
         .withMessage(createMessage(requestParams, apiException));
     addFailureStatus(domain, condition);
   }

diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/SecretHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/SecretHelper.java
@@ -102,6 +102,7 @@ public NextAction onFailure(Packet packet, CallResponse<V1Secret> callResponse)
 
       @Override
       public NextAction onSuccess(Packet packet, CallResponse<V1Secret> callResponse) {
+        clearExistingKubernetesNetworkException(packet);
         V1Secret secret = callResponse.getResult();
         packet.getSpi(DomainPresenceInfo.class).setWebLogicCredentialsSecret(secret);
         insertAuthorizationSource(packet, secret);

diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/ServiceHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/ServiceHelper.java
@@ -663,6 +663,7 @@ public NextAction onFailure(Packet packet, CallResponse<V1Service> callResponse)
 
       @Override
       public NextAction onSuccess(Packet packet, CallResponse<V1Service> callResponse) {
+        clearExistingKubernetesNetworkException(packet);
         V1Service service = callResponse.getResult();
         if (service == null) {
           removeServiceFromRecord();
@@ -687,6 +688,7 @@ public NextAction onFailure(Packet packet, CallResponse<V1Service> callResponse)
 
       @Override
       public NextAction onSuccess(Packet packet, CallResponse<V1Service> callResponse) {
+        clearExistingKubernetesNetworkException(packet);
         return doNext(createReplacementService(getNext()), packet);
       }
     }
@@ -714,6 +716,7 @@ private NextAction updateDomainStatus(Packet packet, CallResponse<V1Service> cal
 
       @Override
       public NextAction onSuccess(Packet packet, CallResponse<V1Service> callResponse) {
+        clearExistingKubernetesNetworkException(packet);
         logServiceCreated(messageKey);
         addServiceToRecord(callResponse.getResult());
         return doNext(packet);

diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/WebhookHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/WebhookHelper.java
@@ -193,6 +193,7 @@ private class ReadResponseStep extends WebhookConfigResponseStep {
 
       @Override
       public NextAction onSuccess(Packet packet, CallResponse<V1ValidatingWebhookConfiguration> callResponse) {
+        clearExistingKubernetesNetworkException(packet);
         V1ValidatingWebhookConfiguration existingWebhookConfig = callResponse.getResult();
         if (existingWebhookConfig == null) {
           return doNext(createValidatingWebhookConfiguration(getNext()), packet);
@@ -273,6 +274,7 @@ private class CreateResponseStep extends WebhookConfigResponseStep {
 
       @Override
       public NextAction onSuccess(Packet packet, CallResponse<V1ValidatingWebhookConfiguration> callResponse) {
+        clearExistingKubernetesNetworkException(packet);
         LOGGER.info(VALIDATING_WEBHOOK_CONFIGURATION_CREATED, getName(callResponse.getResult()));
         return doNext(packet);
       }
@@ -306,6 +308,7 @@ public NextAction onFailure(Packet packet, CallResponse<V1ValidatingWebhookConfi
 
       @Override
       public NextAction onSuccess(Packet packet, CallResponse<V1ValidatingWebhookConfiguration> callResponse) {
+        clearExistingKubernetesNetworkException(packet);
         LOGGER.info(MessageKeys.VALIDATING_WEBHOOK_CONFIGURATION_REPLACED, getName(callResponse.getResult()));
         return doNext(packet);
       }

diff --git a/operator/src/main/java/oracle/kubernetes/operator/steps/DefaultResponseStep.java b/operator/src/main/java/oracle/kubernetes/operator/steps/DefaultResponseStep.java
@@ -36,6 +36,7 @@ public NextAction onFailure(Packet packet, CallResponse<T> callResponse) {
 
   @Override
   public NextAction onSuccess(Packet packet, CallResponse<T> callResponse) {
+    clearExistingKubernetesNetworkException(packet);
     return doNext(packet);
   }
 }
diff --git a/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/DomainFailureReason.java b/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/DomainFailureReason.java
@@ -73,6 +73,13 @@ public String getEventError() {
       return KUBERNETES_EVENT_ERROR;
     }
   },
+  @SerializedName("KubernetesNetworkException")
+  KUBERNETES_NETWORK_EXCEPTION("KubernetesNetworkException") {
+    @Override
+    public String getEventError() {
+      return KUBERNETES_EVENT_ERROR;
+    }
+  },
   @SerializedName("ServerPod")
   SERVER_POD("ServerPod") {
     @Override