diff --git a/Jenkinsfile.oke b/Jenkinsfile.oke
index 767f298134b..86d0d5e63c8 100644
--- a/Jenkinsfile.oke
+++ b/Jenkinsfile.oke
@@ -3,7 +3,9 @@
//
CRON_SETTINGS = '''H 3 * * 0-4 % MAVEN_PROFILE_NAME=oke-gate;CLUSTER_NAME=seqone;PARALLEL_RUN=false
+ H 1 * * 0-4 % MAVEN_PROFILE_NAME=oke-sequential;CLUSTER_NAME=seqtwo;PARALLEL_RUN=false
H 2 * * 0-4 % MAVEN_PROFILE_NAME=oke-parallel;CLUSTER_NAME=parone;PARALLEL_RUN=true'''
+
pipeline {
agent { label 'large' }
options {
@@ -113,9 +115,10 @@ pipeline {
)
choice(name: 'MAVEN_PROFILE_NAME',
- description: 'Profile to use in mvn command to run the tests. Possible values are oke-gate,oke-parallel. Refer to weblogic-kubernetes-operator/integration-tests/pom.xml on the branch.',
+ description: 'Profile to use in mvn command to run the tests. Possible values are oke-gate,oke-parallel, oke-sequential. Refer to weblogic-kubernetes-operator/integration-tests/pom.xml on the branch.',
choices: [
'oke-gate',
+ 'oke-sequential',
'oke-parallel'
]
)
@@ -528,6 +531,8 @@ EOF
export OCI_CLI_CONFIG_FILE=${jenkins_home_directory}/.oci/config
export OCI_CLI_PROFILE=${oci_profile}
compartment_ocid=${compartment_id}
+ export COMPARTMENT_OCID=${compartment_id}
+ echo "COMPARTMENT_OCID : ${COMPARTMENT_OCID}"
mkdir -m777 -p "${WORKSPACE}/.mvn"
touch ${WORKSPACE}/.mvn/maven.config
export KUBECONFIG=${kubeconfig_file}
@@ -599,6 +604,7 @@ EOF
echo "-Dwko.it.monitoring.exporter.branch=\"${MONITORING_EXPORTER_BRANCH}\"" >> ${WORKSPACE}/.mvn/maven.config
echo "-Dwko.it.monitoring.exporter.webapp.version=\"${MONITORING_EXPORTER_WEBAPP_VERSION}\"" >> ${WORKSPACE}/.mvn/maven.config
echo "-Dwko.it.prometheus.chart.version=\"${PROMETHEUS_CHART_VERSION}\"" >> ${WORKSPACE}/.mvn/maven.config
+ echo "-Dwko.it.oci.compartment.ocid=\"${COMPARTMENT_OCID}\"" >> ${WORKSPACE}/.mvn/maven.config
echo "-Dwko.it.grafana.chart.version=\"${GRAFANA_CHART_VERSION}\"" >> ${WORKSPACE}/.mvn/maven.config
echo "-Dwko.it.collect.logs.on.success=\"${COLLECT_LOGS_ON_SUCCESS}\"" >> ${WORKSPACE}/.mvn/maven.config
echo "-Dwko.it.remoteconsole.version=\"${REMOTECONSOLE_VERSION}\"" >> ${WORKSPACE}/.mvn/maven.config
@@ -608,6 +614,10 @@ EOF
echo "${WORKSPACE}/.mvn/maven.config contents:"
cat "${WORKSPACE}/.mvn/maven.config"
cp "${WORKSPACE}/.mvn/maven.config" "${result_root}"
+ echo "${HTTP_PROXY} HTTP_PROXY settings"
+ echo "${HTTPS_PROXY} HTTPS_PROXY settings"
+ echo "${http_proxy} http_proxy settings"
+ echo "${https_proxy} https_proxy settings"
'''
}
withMaven(globalMavenSettingsConfig: 'wkt-maven-settings-xml', publisherStrategy: 'EXPLICIT') {
@@ -620,12 +630,15 @@ EOF
export OKE_CLUSTER_PRIVATEIP="true"
export OKD="false"
export KUBECONFIG=${kubeconfig_file}
+ export COMPARTMENT_OCID=${compartment_id}
export BASE_IMAGES_REPO_USERNAME="${OCIR_USER}"
export BASE_IMAGES_REPO_PASSWORD="${OCIR_PASS}"
export BASE_IMAGES_REPO_EMAIL="noreply@oracle.com"
export TEST_IMAGES_REPO_USERNAME="${OCIR_USER}"
export TEST_IMAGES_REPO_PASSWORD="${OCIR_PASS}"
export TEST_IMAGES_REPO_EMAIL="noreply@oracle.com"
+ export no_proxy="${NO_PROXY},localhost,127.0.0.1,.us.oracle.com,.oraclecorp.com,login.oracle.com"
+ export NO_PROXY="${no_proxy}"
if ! mvn -pl integration-tests -P ${MAVEN_PROFILE_NAME} verify 2>&1 | tee "${result_root}/oketest.log"; then
echo "integration-tests failed"
@@ -670,7 +683,7 @@ EOF
${WORKSPACE}/terraform/oke.delete.sh ${OCI_PROP_FILE} ${WORKSPACE}/terraform ${AVAILABILITY_DOMAIN}
fi
- if [ "${MAVEN_PROFILE_NAME}" = "oke-gate" ] && [ "${BRANCH}" = "main" ]; then
+ if [ "${MAVEN_PROFILE_NAME}" = "oke-gate" ] && [ "${BRANCH}" = "release/4.2" ]; then
compname="wkt"
wkt_compartment_ocid=$(oci iam compartment list --compartment-id-in-subtree true --all | jq --arg compname "$compname" '.data[] | select(."name"==$compname)' | jq -r ."id")
sec_list_id=$(oci network security-list list --compartment-id="$wkt_compartment_ocid" --display-name=Security-List-wktiso1 | jq -r '.data[] | ."id"')
diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml
index e5180dd8bef..a36e854f998 100644
--- a/integration-tests/pom.xml
+++ b/integration-tests/pom.xml
@@ -48,6 +48,7 @@
${env.GRAFANA_CHART_VERSION}
${env.ISTIO_VERSION}
${env.OKE_CLUSTER}
+ ${env.COMPARTMENT_OCID}
${env.ARM}
${env.OKE_CLUSTER_PRIVATEIP}
${env.NFS_SERVER}
diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItAddNewDynamicClusterUsingWlst.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItAddNewDynamicClusterUsingWlst.java
index a16d3a75a8a..01560558930 100644
--- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItAddNewDynamicClusterUsingWlst.java
+++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItAddNewDynamicClusterUsingWlst.java
@@ -57,7 +57,7 @@
@IntegrationTest
@Tag("kind-parallel")
@Tag("olcne-mrg")
-@Tag("oke-gate")
+@Tag("oke-parallel")
class ItAddNewDynamicClusterUsingWlst {
diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItCrossDomainTransaction.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItCrossDomainTransaction.java
index d116d82a6d7..14a536e5b51 100644
--- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItCrossDomainTransaction.java
+++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItCrossDomainTransaction.java
@@ -107,7 +107,7 @@
*/
@DisplayName("Verify cross domain transaction is successful")
@IntegrationTest
-@Tag("oke-gate")
+@Tag("oke-sequential")
@Tag("kind-parallel")
@Tag("okd-wls-srg")
class ItCrossDomainTransaction {
@@ -666,7 +666,7 @@ private static void createNginxIngressPathRoutingRules() {
ingressRules.add(ingressRule);
- createIngressAndRetryIfFail(60, false, ingressName, domain1Namespace, null, ingressClassName, ingressRules, null);
+ createIngressAndRetryIfFail(20, false, ingressName, domain1Namespace, null, ingressClassName, ingressRules, null);
// check the ingress was found in the domain namespace
assertThat(assertDoesNotThrow(() -> listIngresses(domain1Namespace)))
@@ -681,6 +681,6 @@ private static void createNginxIngressPathRoutingRules() {
+ "/weblogic/ready --write-out %{http_code} -o /dev/null";
logger.info("Executing curl command {0}", curlCmd);
- assertTrue(callWebAppAndWaitTillReady(curlCmd, 60));
+ assertTrue(callWebAppAndWaitTillReady(curlCmd, 20));
}
}
diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDynamicDomainInPV.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDynamicDomainInPV.java
index bbfed633cf0..c5e405ebb89 100644
--- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDynamicDomainInPV.java
+++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDynamicDomainInPV.java
@@ -78,7 +78,7 @@
*/
@DisplayName("Test to creat a FMW dynamic domain in persistent volume using WLST")
@IntegrationTest
-@Tag("oke-gate")
+@Tag("oke-sequential")
@Tag("kind-sequential")
@Tag("okd-fmw-cert")
class ItFmwDynamicDomainInPV {
diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItIntrospectVersion.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItIntrospectVersion.java
index 9d777584f03..83da090c4d8 100644
--- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItIntrospectVersion.java
+++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItIntrospectVersion.java
@@ -177,7 +177,7 @@
@Tag("olcne-srg")
@Tag("kind-parallel")
@Tag("okd-wls-mrg")
-@Tag("oke-gate")
+@Tag("oke-sequential")
@Tag("oke-arm")
class ItIntrospectVersion {
diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItIstioMonitoringExporter.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItIstioMonitoringExporter.java
index 4f6ef983c27..b407cda69dd 100644
--- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItIstioMonitoringExporter.java
+++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItIstioMonitoringExporter.java
@@ -77,7 +77,7 @@
@DisplayName("Test the monitoring WebLogic Domain via istio provided Prometheus")
@IntegrationTest
-@Tag("oke-gate")
+@Tag("oke-parallel")
@Tag("kind-parallel")
@Tag("olcne-mrg")
class ItIstioMonitoringExporter {
diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItKubernetesDomainEvents.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItKubernetesDomainEvents.java
index f4264063c0c..c365d4eacb0 100644
--- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItKubernetesDomainEvents.java
+++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItKubernetesDomainEvents.java
@@ -150,7 +150,7 @@
@Tag("oke-arm")
@IntegrationTest
@Tag("olcne-srg")
-@Tag("oke-parallel")
+@Tag("oke-sequential")
class ItKubernetesDomainEvents {
private static String opNamespace = null;
diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java
index a0056a7051a..cac5bbceac4 100644
--- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java
+++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java
@@ -121,7 +121,7 @@
@Tag("toolkits-srg")
@Tag("okd-wls-srg")
@Tag("oke-arm")
-@Tag("oke-gate")
+@Tag("oke-parallel")
class ItMiiAuxiliaryImage {
private static String domainNamespace = null;
diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiClusterResource.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiClusterResource.java
index f3ee4ef6b26..1e0fc8b1e26 100644
--- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiClusterResource.java
+++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiClusterResource.java
@@ -104,7 +104,7 @@
@Tag("kind-parallel")
@Tag("okd-wls-srg")
@Tag("oke-arm")
-@Tag("oke-gate")
+@Tag("oke-sequential")
class ItMiiClusterResource {
private static String opNamespace = null;
diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiDynamicUpdatePart1.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiDynamicUpdatePart1.java
index cc0e8f7d44d..1dd9b556cda 100644
--- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiDynamicUpdatePart1.java
+++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiDynamicUpdatePart1.java
@@ -66,7 +66,7 @@
@Tag("kind-parallel")
@Tag("toolkits-srg")
@Tag("okd-wls-mrg")
-@Tag("oke-gate")
+@Tag("oke-sequential")
@Tag("oke-arm")
@Tag("olcne-srg")
class ItMiiDynamicUpdatePart1 {
diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiDynamicUpdatePart2.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiDynamicUpdatePart2.java
index 317c27bd945..11dd73ae817 100644
--- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiDynamicUpdatePart2.java
+++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiDynamicUpdatePart2.java
@@ -65,7 +65,7 @@
@Tag("kind-parallel")
@Tag("toolkits-srg")
@Tag("okd-wls-mrg")
-@Tag("oke-gate")
+@Tag("oke-sequential")
@Tag("oke-arm")
class ItMiiDynamicUpdatePart2 {
diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiDynamicUpdatePart3.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiDynamicUpdatePart3.java
index 169a0ef6e1c..7d69ff6dadd 100644
--- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiDynamicUpdatePart3.java
+++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiDynamicUpdatePart3.java
@@ -78,7 +78,7 @@
@DisplayName("Test dynamic updates to a model in image domain, part3")
@IntegrationTest
@Tag("olcne-mrg")
-@Tag("oke-gate")
+@Tag("oke-sequential")
@Tag("kind-parallel")
@Tag("toolkits-srg")
@Tag("okd-wls-mrg")
diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiUpdateDomainConfig.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiUpdateDomainConfig.java
index ca6affcebc9..1d12ea40fee 100644
--- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiUpdateDomainConfig.java
+++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiUpdateDomainConfig.java
@@ -119,7 +119,7 @@
@Tag("kind-parallel")
@Tag("toolkits-srg")
@Tag("okd-wls-srg")
-@Tag("oke-gate")
+@Tag("oke-sequential")
class ItMiiUpdateDomainConfig {
private static String opNamespace = null;
diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMultiDomainModelsScale.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMultiDomainModelsScale.java
index b1c61060c92..1c495e714c8 100644
--- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMultiDomainModelsScale.java
+++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMultiDomainModelsScale.java
@@ -121,7 +121,7 @@
+ "rolling restart behavior in a multi-cluster MII domain and "
+ "the sample application can be accessed via NGINX ingress controller")
@Tag("kind-sequential")
-@Tag("oke-gate")
+@Tag("oke-sequential")
@IntegrationTest
class ItMultiDomainModelsScale {
diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItOCILoadBalancer.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItOCILoadBalancer.java
index 5ea7725898a..04465a1ee48 100644
--- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItOCILoadBalancer.java
+++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItOCILoadBalancer.java
@@ -4,15 +4,12 @@
package oracle.weblogic.kubernetes;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
-import io.kubernetes.client.openapi.models.V1LoadBalancerIngress;
-import io.kubernetes.client.openapi.models.V1Service;
import oracle.weblogic.kubernetes.actions.impl.primitive.Kubernetes;
import oracle.weblogic.kubernetes.annotations.IntegrationTest;
import oracle.weblogic.kubernetes.annotations.Namespaces;
+import oracle.weblogic.kubernetes.extensions.InitializationTasks;
import oracle.weblogic.kubernetes.logging.LoggingFacade;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
@@ -25,10 +22,11 @@
import static oracle.weblogic.kubernetes.TestConstants.MII_BASIC_IMAGE_NAME;
import static oracle.weblogic.kubernetes.TestConstants.MII_BASIC_IMAGE_TAG;
import static oracle.weblogic.kubernetes.TestConstants.SKIP_CLEANUP;
-import static oracle.weblogic.kubernetes.assertions.impl.Kubernetes.getService;
import static oracle.weblogic.kubernetes.utils.ApplicationUtils.callWebAppAndCheckForServerNameInResponse;
import static oracle.weblogic.kubernetes.utils.CommonMiiTestUtils.createMiiDomainAndVerify;
import static oracle.weblogic.kubernetes.utils.ImageUtils.createTestRepoSecret;
+import static oracle.weblogic.kubernetes.utils.LoadBalancerUtils.deleteLoadBalancer;
+import static oracle.weblogic.kubernetes.utils.LoadBalancerUtils.getLoadBalancerIP;
import static oracle.weblogic.kubernetes.utils.LoadBalancerUtils.installAndVerifyOCILoadBalancer;
import static oracle.weblogic.kubernetes.utils.OperatorUtils.installAndVerifyOperator;
import static oracle.weblogic.kubernetes.utils.ThreadSafeLogger.getLogger;
@@ -45,7 +43,7 @@
+ "all managed servers in the domain through OCI Load Balancer")
@IntegrationTest
@Tag("oke-arm")
-@Tag("oke-parallel")
+@Tag("oke-gate")
class ItOCILoadBalancer {
// domain constants
private static final int replicaCount = 2;
@@ -88,6 +86,7 @@ public void tearDownAll() {
if (!SKIP_CLEANUP) {
Kubernetes.deleteService(OCI_LB_NAME, domainNamespace);
}
+ deleteLoadBalancer(loadBalancerIP);
}
/**
@@ -118,37 +117,14 @@ void testOCILoadBalancer() throws Exception {
assertDoesNotThrow(() -> installAndVerifyOCILoadBalancer(domainNamespace,
clusterHttpPort, clusterName, domainUid, OCI_LB_NAME),
"Installation of OCI Load Balancer failed");
- loadBalancerIP = getLoadBalancerIP(domainNamespace,OCI_LB_NAME);
+ loadBalancerIP = getLoadBalancerIP(domainNamespace,OCI_LB_NAME, true);
assertNotNull(loadBalancerIP, "External IP for Load Balancer is undefined");
+ InitializationTasks.registerLoadBalancerExternalIP(loadBalancerIP);
+
logger.info("LoadBalancer IP is " + loadBalancerIP);
verifyWebAppAccessThroughOCILoadBalancer(loadBalancerIP, 2, clusterHttpPort);
}
- /**
- * Retreive external IP from OCI LoadBalancer.
- */
- private static String getLoadBalancerIP(String namespace, String lbName) throws Exception {
- Map labels = new HashMap<>();
- labels.put("loadbalancer", lbName);
- V1Service service = getService(lbName, labels, namespace);
- assertNotNull(service, "Can't find service with name " + lbName);
- logger.info("Found service with name {0} in {1} namespace ", lbName, namespace);
- assertNotNull(service.getStatus(), "service status is null");
- assertNotNull(service.getStatus().getLoadBalancer(), "service loadbalancer is null");
- List ingress = service.getStatus().getLoadBalancer().getIngress();
- if (ingress != null) {
- logger.info("LoadBalancer Ingress " + ingress.toString());
- V1LoadBalancerIngress lbIng = ingress.stream().filter(c ->
- c.getIp() != null && !c.getIp().equals("pending")
- ).findAny().orElse(null);
- if (lbIng != null) {
- logger.info("OCI LoadBalancer is created with external ip" + lbIng.getIp());
- return lbIng.getIp();
- }
- }
- return null;
- }
-
/**
* Verify the sample-app app can be accessed from all managed servers in the domain through OCI Load Balancer.
*/
diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/TestConstants.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/TestConstants.java
index 4052b92fd48..d39edd904bb 100644
--- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/TestConstants.java
+++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/TestConstants.java
@@ -370,6 +370,7 @@ public interface TestConstants {
Boolean.parseBoolean(getNonEmptySystemProperty("wko.it.oke.cluster.privateip", "false"));
public static final String NFS_SERVER = System.getProperty("wko.it.nfs.server", "");
public static final String NODE_IP = System.getProperty("wko.it.node.ip", "");
+ public static final String COMPARTMENT_OCID = System.getProperty("wko.it.oci.compartment.ocid", "");
public static final String [] FSS_DIR = System.getProperty("wko.it.fss.dir","").split(",");
public static final String IMAGE_PULL_POLICY = "IfNotPresent";
diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/extensions/InitializationTasks.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/extensions/InitializationTasks.java
index 405acd0f29e..186569a27a8 100644
--- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/extensions/InitializationTasks.java
+++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/extensions/InitializationTasks.java
@@ -11,9 +11,11 @@
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -113,6 +115,7 @@
import static oracle.weblogic.kubernetes.utils.FileUtils.cleanupDirectory;
import static oracle.weblogic.kubernetes.utils.IstioUtils.installIstio;
import static oracle.weblogic.kubernetes.utils.IstioUtils.uninstallIstio;
+import static oracle.weblogic.kubernetes.utils.LoadBalancerUtils.deleteLoadBalancer;
import static oracle.weblogic.kubernetes.utils.LoadBalancerUtils.installAndVerifyTraefik;
import static oracle.weblogic.kubernetes.utils.OperatorUtils.installAndVerifyOperator;
import static oracle.weblogic.kubernetes.utils.ThreadSafeLogger.getLogger;
@@ -133,6 +136,7 @@ public class InitializationTasks implements BeforeAllCallback, ExtensionContext.
private static String wdtBasicImage;
private static Collection pushedImages = new ArrayList<>();
+ private static Set lbIPs = new HashSet<>();
private static boolean isInitializationSuccessful = false;
ConditionFactory withVeryLongRetryPolicy
@@ -355,6 +359,15 @@ public static void registerPushedImage(String imageName) {
pushedImages.add(imageName);
}
+ /**
+ * Called when load balancer is created in OCI, allowing conditional cleanup of load balancers.
+ *
+ * @param lbIP external IP of load balancer.
+ */
+ public static void registerLoadBalancerExternalIP(String lbIP) {
+ lbIPs.add(lbIP);
+ }
+
@Override
public void close() {
LoggingFacade logger = getLogger();
@@ -395,6 +408,13 @@ public void close() {
for (String image : pushedImages) {
deleteImage(image);
}
+ if (OKE_CLUSTER) {
+ logger.info("Cleanup created in OCI load balancers after all test suites are run");
+ // delete all load balancers in OCI
+ for (String ip : lbIPs) {
+ deleteLoadBalancer(ip);
+ }
+ }
}
// delete images from TEST_IMAGES_REPO, if necessary
diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/IstioUtils.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/IstioUtils.java
index 28959823cb8..d9f6e98498c 100644
--- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/IstioUtils.java
+++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/IstioUtils.java
@@ -24,6 +24,7 @@
import oracle.weblogic.domain.OnlineUpdate;
import oracle.weblogic.domain.ServerPod;
import oracle.weblogic.kubernetes.actions.impl.primitive.Command;
+import oracle.weblogic.kubernetes.extensions.InitializationTasks;
import oracle.weblogic.kubernetes.logging.LoggingFacade;
import oracle.weblogic.kubernetes.utils.SemanticVersion.Compatibility;
@@ -37,6 +38,7 @@
import static oracle.weblogic.kubernetes.TestConstants.ISTIO_VERSION;
import static oracle.weblogic.kubernetes.TestConstants.KUBERNETES_CLI;
import static oracle.weblogic.kubernetes.TestConstants.OCNE;
+import static oracle.weblogic.kubernetes.TestConstants.OKE_CLUSTER;
import static oracle.weblogic.kubernetes.TestConstants.OKE_CLUSTER_PRIVATEIP;
import static oracle.weblogic.kubernetes.TestConstants.PROMETHEUS_CONFIG_MAP_RELOAD_IMAGE_NAME;
import static oracle.weblogic.kubernetes.TestConstants.PROMETHEUS_CONFIG_MAP_RELOAD_IMAGE_TAG;
@@ -51,9 +53,12 @@
import static oracle.weblogic.kubernetes.utils.ClusterUtils.createClusterResourceAndAddReferenceToDomain;
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkServiceExists;
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.formatIPv6Host;
+import static oracle.weblogic.kubernetes.utils.CommonTestUtils.getServiceExtIPAddrtOke;
+import static oracle.weblogic.kubernetes.utils.CommonTestUtils.testUntil;
import static oracle.weblogic.kubernetes.utils.ExecCommand.exec;
import static oracle.weblogic.kubernetes.utils.FileUtils.generateFileFromTemplate;
import static oracle.weblogic.kubernetes.utils.FileUtils.replaceStringInFile;
+import static oracle.weblogic.kubernetes.utils.LoadBalancerUtils.isLoadBalancerHealthy;
import static oracle.weblogic.kubernetes.utils.PodUtils.checkPodReady;
import static oracle.weblogic.kubernetes.utils.PodUtils.setPodAntiAffinity;
import static oracle.weblogic.kubernetes.utils.ThreadSafeLogger.getLogger;
@@ -112,6 +117,17 @@ public static void installIstio() {
.command(command)
.redirect(false))
.execute());
+
+ if (OKE_CLUSTER) {
+ String loadBalancerIP = getServiceExtIPAddrtOke("istio-ingressgateway", "istio-system");
+ testUntil(
+ assertDoesNotThrow(() -> isLoadBalancerHealthy("istio-system", "istio-ingressgateway"),
+ "isLoadBalancerHealthy failed with ApiException"),
+ logger,
+ "Istio LoadBalancer to be healthy in namespace {0}",
+ "istio-system");
+ InitializationTasks.registerLoadBalancerExternalIP(loadBalancerIP);
+ }
}
/**
diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/LoadBalancerUtils.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/LoadBalancerUtils.java
index 00d4e2b5c39..92f52f28b64 100644
--- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/LoadBalancerUtils.java
+++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/LoadBalancerUtils.java
@@ -12,6 +12,7 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.Callable;
import io.kubernetes.client.custom.IntOrString;
import io.kubernetes.client.openapi.ApiException;
@@ -21,6 +22,7 @@
import io.kubernetes.client.openapi.models.V1IngressRule;
import io.kubernetes.client.openapi.models.V1IngressServiceBackend;
import io.kubernetes.client.openapi.models.V1IngressTLS;
+import io.kubernetes.client.openapi.models.V1LoadBalancerIngress;
import io.kubernetes.client.openapi.models.V1ObjectMeta;
import io.kubernetes.client.openapi.models.V1Service;
import io.kubernetes.client.openapi.models.V1ServiceBackendPort;
@@ -29,10 +31,13 @@
import oracle.weblogic.kubernetes.TestConstants;
import oracle.weblogic.kubernetes.actions.impl.NginxParams;
import oracle.weblogic.kubernetes.actions.impl.TraefikParams;
+import oracle.weblogic.kubernetes.actions.impl.primitive.Command;
import oracle.weblogic.kubernetes.actions.impl.primitive.HelmParams;
+import oracle.weblogic.kubernetes.extensions.InitializationTasks;
import oracle.weblogic.kubernetes.logging.LoggingFacade;
import static oracle.weblogic.kubernetes.TestConstants.ADMIN_SERVER_NAME_BASE;
+import static oracle.weblogic.kubernetes.TestConstants.COMPARTMENT_OCID;
import static oracle.weblogic.kubernetes.TestConstants.K8S_NODEPORT_HOST;
import static oracle.weblogic.kubernetes.TestConstants.KUBERNETES_CLI;
import static oracle.weblogic.kubernetes.TestConstants.NGINX_CHART_NAME;
@@ -40,6 +45,7 @@
import static oracle.weblogic.kubernetes.TestConstants.NGINX_RELEASE_NAME;
import static oracle.weblogic.kubernetes.TestConstants.NGINX_REPO_NAME;
import static oracle.weblogic.kubernetes.TestConstants.NGINX_REPO_URL;
+import static oracle.weblogic.kubernetes.TestConstants.OKE_CLUSTER;
import static oracle.weblogic.kubernetes.TestConstants.RESULTS_ROOT;
import static oracle.weblogic.kubernetes.TestConstants.TEST_IMAGES_REPO_SECRET_NAME;
import static oracle.weblogic.kubernetes.TestConstants.TRAEFIK_CHART_NAME;
@@ -53,10 +59,12 @@
import static oracle.weblogic.kubernetes.actions.TestActions.installTraefik;
import static oracle.weblogic.kubernetes.actions.TestActions.listIngresses;
import static oracle.weblogic.kubernetes.actions.TestActions.upgradeTraefikImage;
+import static oracle.weblogic.kubernetes.actions.impl.primitive.Command.defaultCommandParams;
import static oracle.weblogic.kubernetes.assertions.TestAssertions.isHelmReleaseDeployed;
import static oracle.weblogic.kubernetes.assertions.TestAssertions.isNginxReady;
import static oracle.weblogic.kubernetes.assertions.TestAssertions.isOCILoadBalancerReady;
import static oracle.weblogic.kubernetes.assertions.TestAssertions.isTraefikReady;
+import static oracle.weblogic.kubernetes.assertions.impl.Kubernetes.getService;
import static oracle.weblogic.kubernetes.utils.ApplicationUtils.callWebAppAndWaitTillReady;
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodReadyAndServiceExists;
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkServiceExists;
@@ -126,6 +134,14 @@ public static void installAndVerifyOCILoadBalancer(
logger,
"external IP to be generated in {0}",
namespace);
+ if (OKE_CLUSTER) {
+ testUntil(
+ assertDoesNotThrow(() -> isLoadBalancerHealthy(namespace, loadBalancerName),
+ "isLoadBalancerHealthy failed with ApiException"),
+ logger,
+ "OCI LB to be healthy in namespace {0}",
+ namespace);
+ }
}
/**
@@ -212,6 +228,22 @@ public static NginxParams installAndVerifyNginx(String nginxNamespace,
logger,
"NGINX to be ready in namespace {0}",
nginxNamespace);
+ if (OKE_CLUSTER) {
+ testUntil(
+ assertDoesNotThrow(() -> isOCILoadBalancerReady(
+ nginxHelmParams.getReleaseName() + "-ingress-nginx-controller",
+ null, nginxNamespace), "isOCILoadBalancerReady failed with ApiException"),
+ logger,
+ "external IP to be generated in {0}",
+ nginxNamespace);
+ testUntil(
+ assertDoesNotThrow(() -> isLoadBalancerHealthy(nginxNamespace,
+ nginxHelmParams.getReleaseName() + "-ingress-nginx-controller"),
+ "isLoadBalancerHealthy failed with ApiException"),
+ logger,
+ "NGINX LB to be healthy in namespace {0}",
+ nginxNamespace);
+ }
return nginxParams;
}
@@ -282,10 +314,117 @@ public static TraefikParams installAndVerifyTraefik(String traefikNamespace,
logger,
"Traefik to be ready in namespace {0}",
traefikNamespace);
+ if (OKE_CLUSTER) {
+ testUntil(
+ assertDoesNotThrow(() -> isOCILoadBalancerReady(
+ traefikHelmParams.getReleaseName(),
+ null, traefikNamespace), "isOCILoadBalancerReady failed with ApiException"),
+ logger,
+ "external IP to be generated in {0}",
+ traefikNamespace);
+ testUntil(
+ assertDoesNotThrow(() -> isLoadBalancerHealthy(traefikNamespace, traefikHelmParams.getReleaseName()),
+ "isLoadBalancerHealthy failed with ApiException"),
+ logger,
+ "Traefik to be healthy in namespace {0}",
+ traefikNamespace);
+ }
return traefikParams;
}
+ /**
+ * Check lb has healty status.
+ *
+ * @param namespace in which to check for lb controller
+ * @name service name of lb controller
+ * @return true if healthy, false otherwise
+ */
+ public static Callable isLoadBalancerHealthy(String namespace, String name) {
+ return () -> checkLoadBalancerHealthy(namespace, name);
+ }
+
+ /**
+ * Retreive external IP from OCI LoadBalancer.
+ *
+ * @param namespace - namespace
+ * @param lbName -loadbalancer service name
+ */
+ public static String getLoadBalancerIP(String namespace, String lbName) throws Exception {
+ return getLoadBalancerIP(namespace, lbName, false);
+ }
+
+ /**
+ * Retreive external IP from OCI LoadBalancer.
+ *
+ * @param namespace - namespace
+ * @param lbName -loadbalancer service name
+ * @param addLabel search service with label
+ */
+ public static String getLoadBalancerIP(String namespace, String lbName, boolean addLabel) throws Exception {
+ Map labels = null;
+ if (addLabel) {
+ labels = new HashMap<>();
+ labels.put("loadbalancer", lbName);
+ }
+ V1Service service = getService(lbName, labels, namespace);
+ assertNotNull(service, "Can't find service with name " + lbName);
+ LoggingFacade logger = getLogger();
+ logger.info("Found service with name {0} in {1} namespace ", lbName, namespace);
+ assertNotNull(service.getStatus(), "service status is null");
+ assertNotNull(service.getStatus().getLoadBalancer(), "service loadbalancer is null");
+ List ingress = service.getStatus().getLoadBalancer().getIngress();
+ if (ingress != null) {
+ logger.info("LoadBalancer Ingress " + ingress.toString());
+ V1LoadBalancerIngress lbIng =
+ ingress.stream().filter(c -> c.getIp() != null && !c.getIp().equals("pending")).findAny().orElse(null);
+ if (lbIng != null) {
+ logger.info("OCI LoadBalancer is created with external ip" + lbIng.getIp());
+ return lbIng.getIp();
+ }
+ }
+ return null;
+ }
+
+ private static boolean checkLoadBalancerHealthy(String namespace, String lbServiceName) {
+ String lbPublicIP = assertDoesNotThrow(() -> getLoadBalancerIP(namespace, lbServiceName));
+ InitializationTasks.registerLoadBalancerExternalIP(lbPublicIP);
+ LoggingFacade logger = getLogger();
+ String testcompartmentid = System.getProperty("wko.it.oci.compartment.ocid");
+ logger.info("wko.it.oci.compartment.ocid property " + testcompartmentid);
+
+ final String command = "oci lb load-balancer list --compartment-id "
+ + testcompartmentid + " --query \"data[?contains(\\\"ip-addresses\\\"[0].\\\"ip-address\\\", '"
+ + lbPublicIP + "')].id | [0]\" --raw-output --all";
+
+ logger.info("Command to retrieve Load Balancer OCID is: {0} ", command);
+
+ ExecResult result = assertDoesNotThrow(() -> exec(command, true));
+ logger.info("The command returned exit value: " + result.exitValue()
+ + " command output: " + result.stderr() + "\n" + result.stdout());
+
+ if (result == null || result.exitValue() != 0 || result.stdout() == null) {
+ return false;
+ }
+
+ // Clean up the string to extract the Load Balancer ID
+ String lbOCID = result.stdout().trim();
+
+ //check health status
+ final String command1 = "oci lb load-balancer-health get --load-balancer-id " + lbOCID;
+ logger.info("Command to retrieve Load Balancer health status is: {0} ", command1);
+ result = assertDoesNotThrow(() -> exec(command1, true));
+ logger.info("The command returned exit value: " + result.exitValue()
+ + " command output: " + result.stderr() + "\n" + result.stdout());
+
+ if (result == null || result.exitValue() != 0 || result.stdout() == null) {
+ return false;
+ }
+
+ return result.stdout().contains("OK");
+
+ }
+
/** Upgrade Traefik and wait for up to five minutes for the Traefik pod to be ready.
*
* @param traefikNamespace the namespace in which the Traefik ingress controller is installed
@@ -733,4 +872,39 @@ public static String getLbExternalIp(String lbrelname, String lbns) throws Excep
return result.stdout().trim();
}
+
+ /**
+ * Delete Load Balancer , created in OCI by using provided public IP.
+ *
+ * @param lbPublicIP public Load Balancer IP
+ */
+ public static void deleteLoadBalancer(String lbPublicIP) {
+ if (lbPublicIP != null && !lbPublicIP.isEmpty()) {
+ if (lbPublicIP.startsWith("[") && lbPublicIP.endsWith("]")) {
+ // Remove the brackets and return the content inside
+ lbPublicIP = lbPublicIP.substring(1, lbPublicIP.length() - 1);
+ }
+ LoggingFacade logger = getLogger();
+ Path deleteLBPath =
+ Paths.get(RESOURCE_DIR, "bash-scripts", "delete_loadbalancer.sh");
+ String deleteLBScript = deleteLBPath.toString();
+ String command =
+ String.format("chmod 777 %s ", deleteLBScript);
+ logger.info("Delete Load Balancer command {0}", command);
+ assertTrue(() -> Command.withParams(
+ defaultCommandParams()
+ .command(command)
+ .redirect(true))
+ .execute());
+
+ String command1 =
+ String.format("%s %s %s ", deleteLBScript, lbPublicIP, COMPARTMENT_OCID);
+ logger.info("Delete Load Balancer command {0}", command);
+ assertTrue(() -> Command.withParams(
+ defaultCommandParams()
+ .command(command1)
+ .redirect(true))
+ .execute());
+ }
+ }
}
diff --git a/integration-tests/src/test/resources/bash-scripts/delete_loadbalancer.sh b/integration-tests/src/test/resources/bash-scripts/delete_loadbalancer.sh
new file mode 100644
index 00000000000..e69de29bb2d