diff --git a/Jenkinsfile.oke b/Jenkinsfile.oke index 767f298134b..86d0d5e63c8 100644 --- a/Jenkinsfile.oke +++ b/Jenkinsfile.oke @@ -3,7 +3,9 @@ // CRON_SETTINGS = '''H 3 * * 0-4 % MAVEN_PROFILE_NAME=oke-gate;CLUSTER_NAME=seqone;PARALLEL_RUN=false + H 1 * * 0-4 % MAVEN_PROFILE_NAME=oke-sequential;CLUSTER_NAME=seqtwo;PARALLEL_RUN=false H 2 * * 0-4 % MAVEN_PROFILE_NAME=oke-parallel;CLUSTER_NAME=parone;PARALLEL_RUN=true''' + pipeline { agent { label 'large' } options { @@ -113,9 +115,10 @@ pipeline { ) choice(name: 'MAVEN_PROFILE_NAME', - description: 'Profile to use in mvn command to run the tests. Possible values are oke-gate,oke-parallel. Refer to weblogic-kubernetes-operator/integration-tests/pom.xml on the branch.', + description: 'Profile to use in mvn command to run the tests. Possible values are oke-gate,oke-parallel, oke-sequential. Refer to weblogic-kubernetes-operator/integration-tests/pom.xml on the branch.', choices: [ 'oke-gate', + 'oke-sequential', 'oke-parallel' ] ) @@ -528,6 +531,8 @@ EOF export OCI_CLI_CONFIG_FILE=${jenkins_home_directory}/.oci/config export OCI_CLI_PROFILE=${oci_profile} compartment_ocid=${compartment_id} + export COMPARTMENT_OCID=${compartment_id} + echo "COMPARTMENT_OCID : ${COMPARTMENT_OCID}" mkdir -m777 -p "${WORKSPACE}/.mvn" touch ${WORKSPACE}/.mvn/maven.config export KUBECONFIG=${kubeconfig_file} @@ -599,6 +604,7 @@ EOF echo "-Dwko.it.monitoring.exporter.branch=\"${MONITORING_EXPORTER_BRANCH}\"" >> ${WORKSPACE}/.mvn/maven.config echo "-Dwko.it.monitoring.exporter.webapp.version=\"${MONITORING_EXPORTER_WEBAPP_VERSION}\"" >> ${WORKSPACE}/.mvn/maven.config echo "-Dwko.it.prometheus.chart.version=\"${PROMETHEUS_CHART_VERSION}\"" >> ${WORKSPACE}/.mvn/maven.config + echo "-Dwko.it.oci.compartment.ocid=\"${COMPARTMENT_OCID}\"" >> ${WORKSPACE}/.mvn/maven.config echo "-Dwko.it.grafana.chart.version=\"${GRAFANA_CHART_VERSION}\"" >> ${WORKSPACE}/.mvn/maven.config echo "-Dwko.it.collect.logs.on.success=\"${COLLECT_LOGS_ON_SUCCESS}\"" >> ${WORKSPACE}/.mvn/maven.config echo "-Dwko.it.remoteconsole.version=\"${REMOTECONSOLE_VERSION}\"" >> ${WORKSPACE}/.mvn/maven.config @@ -608,6 +614,10 @@ EOF echo "${WORKSPACE}/.mvn/maven.config contents:" cat "${WORKSPACE}/.mvn/maven.config" cp "${WORKSPACE}/.mvn/maven.config" "${result_root}" + echo "${HTTP_PROXY} HTTP_PROXY settings" + echo "${HTTPS_PROXY} HTTPS_PROXY settings" + echo "${http_proxy} http_proxy settings" + echo "${https_proxy} https_proxy settings" ''' } withMaven(globalMavenSettingsConfig: 'wkt-maven-settings-xml', publisherStrategy: 'EXPLICIT') { @@ -620,12 +630,15 @@ EOF export OKE_CLUSTER_PRIVATEIP="true" export OKD="false" export KUBECONFIG=${kubeconfig_file} + export COMPARTMENT_OCID=${compartment_id} export BASE_IMAGES_REPO_USERNAME="${OCIR_USER}" export BASE_IMAGES_REPO_PASSWORD="${OCIR_PASS}" export BASE_IMAGES_REPO_EMAIL="noreply@oracle.com" export TEST_IMAGES_REPO_USERNAME="${OCIR_USER}" export TEST_IMAGES_REPO_PASSWORD="${OCIR_PASS}" export TEST_IMAGES_REPO_EMAIL="noreply@oracle.com" + export no_proxy="${NO_PROXY},localhost,127.0.0.1,.us.oracle.com,.oraclecorp.com,login.oracle.com" + export NO_PROXY="${no_proxy}" if ! mvn -pl integration-tests -P ${MAVEN_PROFILE_NAME} verify 2>&1 | tee "${result_root}/oketest.log"; then echo "integration-tests failed" @@ -670,7 +683,7 @@ EOF ${WORKSPACE}/terraform/oke.delete.sh ${OCI_PROP_FILE} ${WORKSPACE}/terraform ${AVAILABILITY_DOMAIN} fi - if [ "${MAVEN_PROFILE_NAME}" = "oke-gate" ] && [ "${BRANCH}" = "main" ]; then + if [ "${MAVEN_PROFILE_NAME}" = "oke-gate" ] && [ "${BRANCH}" = "release/4.2" ]; then compname="wkt" wkt_compartment_ocid=$(oci iam compartment list --compartment-id-in-subtree true --all | jq --arg compname "$compname" '.data[] | select(."name"==$compname)' | jq -r ."id") sec_list_id=$(oci network security-list list --compartment-id="$wkt_compartment_ocid" --display-name=Security-List-wktiso1 | jq -r '.data[] | ."id"') diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index e5180dd8bef..a36e854f998 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -48,6 +48,7 @@ ${env.GRAFANA_CHART_VERSION} ${env.ISTIO_VERSION} ${env.OKE_CLUSTER} + ${env.COMPARTMENT_OCID} ${env.ARM} ${env.OKE_CLUSTER_PRIVATEIP} ${env.NFS_SERVER} diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItAddNewDynamicClusterUsingWlst.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItAddNewDynamicClusterUsingWlst.java index a16d3a75a8a..01560558930 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItAddNewDynamicClusterUsingWlst.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItAddNewDynamicClusterUsingWlst.java @@ -57,7 +57,7 @@ @IntegrationTest @Tag("kind-parallel") @Tag("olcne-mrg") -@Tag("oke-gate") +@Tag("oke-parallel") class ItAddNewDynamicClusterUsingWlst { diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItCrossDomainTransaction.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItCrossDomainTransaction.java index d116d82a6d7..14a536e5b51 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItCrossDomainTransaction.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItCrossDomainTransaction.java @@ -107,7 +107,7 @@ */ @DisplayName("Verify cross domain transaction is successful") @IntegrationTest -@Tag("oke-gate") +@Tag("oke-sequential") @Tag("kind-parallel") @Tag("okd-wls-srg") class ItCrossDomainTransaction { @@ -666,7 +666,7 @@ private static void createNginxIngressPathRoutingRules() { ingressRules.add(ingressRule); - createIngressAndRetryIfFail(60, false, ingressName, domain1Namespace, null, ingressClassName, ingressRules, null); + createIngressAndRetryIfFail(20, false, ingressName, domain1Namespace, null, ingressClassName, ingressRules, null); // check the ingress was found in the domain namespace assertThat(assertDoesNotThrow(() -> listIngresses(domain1Namespace))) @@ -681,6 +681,6 @@ private static void createNginxIngressPathRoutingRules() { + "/weblogic/ready --write-out %{http_code} -o /dev/null"; logger.info("Executing curl command {0}", curlCmd); - assertTrue(callWebAppAndWaitTillReady(curlCmd, 60)); + assertTrue(callWebAppAndWaitTillReady(curlCmd, 20)); } } diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDynamicDomainInPV.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDynamicDomainInPV.java index bbfed633cf0..c5e405ebb89 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDynamicDomainInPV.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDynamicDomainInPV.java @@ -78,7 +78,7 @@ */ @DisplayName("Test to creat a FMW dynamic domain in persistent volume using WLST") @IntegrationTest -@Tag("oke-gate") +@Tag("oke-sequential") @Tag("kind-sequential") @Tag("okd-fmw-cert") class ItFmwDynamicDomainInPV { diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItIntrospectVersion.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItIntrospectVersion.java index 9d777584f03..83da090c4d8 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItIntrospectVersion.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItIntrospectVersion.java @@ -177,7 +177,7 @@ @Tag("olcne-srg") @Tag("kind-parallel") @Tag("okd-wls-mrg") -@Tag("oke-gate") +@Tag("oke-sequential") @Tag("oke-arm") class ItIntrospectVersion { diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItIstioMonitoringExporter.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItIstioMonitoringExporter.java index 4f6ef983c27..b407cda69dd 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItIstioMonitoringExporter.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItIstioMonitoringExporter.java @@ -77,7 +77,7 @@ @DisplayName("Test the monitoring WebLogic Domain via istio provided Prometheus") @IntegrationTest -@Tag("oke-gate") +@Tag("oke-parallel") @Tag("kind-parallel") @Tag("olcne-mrg") class ItIstioMonitoringExporter { diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItKubernetesDomainEvents.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItKubernetesDomainEvents.java index f4264063c0c..c365d4eacb0 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItKubernetesDomainEvents.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItKubernetesDomainEvents.java @@ -150,7 +150,7 @@ @Tag("oke-arm") @IntegrationTest @Tag("olcne-srg") -@Tag("oke-parallel") +@Tag("oke-sequential") class ItKubernetesDomainEvents { private static String opNamespace = null; diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java index a0056a7051a..cac5bbceac4 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java @@ -121,7 +121,7 @@ @Tag("toolkits-srg") @Tag("okd-wls-srg") @Tag("oke-arm") -@Tag("oke-gate") +@Tag("oke-parallel") class ItMiiAuxiliaryImage { private static String domainNamespace = null; diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiClusterResource.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiClusterResource.java index f3ee4ef6b26..1e0fc8b1e26 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiClusterResource.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiClusterResource.java @@ -104,7 +104,7 @@ @Tag("kind-parallel") @Tag("okd-wls-srg") @Tag("oke-arm") -@Tag("oke-gate") +@Tag("oke-sequential") class ItMiiClusterResource { private static String opNamespace = null; diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiDynamicUpdatePart1.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiDynamicUpdatePart1.java index cc0e8f7d44d..1dd9b556cda 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiDynamicUpdatePart1.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiDynamicUpdatePart1.java @@ -66,7 +66,7 @@ @Tag("kind-parallel") @Tag("toolkits-srg") @Tag("okd-wls-mrg") -@Tag("oke-gate") +@Tag("oke-sequential") @Tag("oke-arm") @Tag("olcne-srg") class ItMiiDynamicUpdatePart1 { diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiDynamicUpdatePart2.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiDynamicUpdatePart2.java index 317c27bd945..11dd73ae817 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiDynamicUpdatePart2.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiDynamicUpdatePart2.java @@ -65,7 +65,7 @@ @Tag("kind-parallel") @Tag("toolkits-srg") @Tag("okd-wls-mrg") -@Tag("oke-gate") +@Tag("oke-sequential") @Tag("oke-arm") class ItMiiDynamicUpdatePart2 { diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiDynamicUpdatePart3.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiDynamicUpdatePart3.java index 169a0ef6e1c..7d69ff6dadd 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiDynamicUpdatePart3.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiDynamicUpdatePart3.java @@ -78,7 +78,7 @@ @DisplayName("Test dynamic updates to a model in image domain, part3") @IntegrationTest @Tag("olcne-mrg") -@Tag("oke-gate") +@Tag("oke-sequential") @Tag("kind-parallel") @Tag("toolkits-srg") @Tag("okd-wls-mrg") diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiUpdateDomainConfig.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiUpdateDomainConfig.java index ca6affcebc9..1d12ea40fee 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiUpdateDomainConfig.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiUpdateDomainConfig.java @@ -119,7 +119,7 @@ @Tag("kind-parallel") @Tag("toolkits-srg") @Tag("okd-wls-srg") -@Tag("oke-gate") +@Tag("oke-sequential") class ItMiiUpdateDomainConfig { private static String opNamespace = null; diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMultiDomainModelsScale.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMultiDomainModelsScale.java index b1c61060c92..1c495e714c8 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMultiDomainModelsScale.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMultiDomainModelsScale.java @@ -121,7 +121,7 @@ + "rolling restart behavior in a multi-cluster MII domain and " + "the sample application can be accessed via NGINX ingress controller") @Tag("kind-sequential") -@Tag("oke-gate") +@Tag("oke-sequential") @IntegrationTest class ItMultiDomainModelsScale { diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItOCILoadBalancer.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItOCILoadBalancer.java index 5ea7725898a..04465a1ee48 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItOCILoadBalancer.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItOCILoadBalancer.java @@ -4,15 +4,12 @@ package oracle.weblogic.kubernetes; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; -import io.kubernetes.client.openapi.models.V1LoadBalancerIngress; -import io.kubernetes.client.openapi.models.V1Service; import oracle.weblogic.kubernetes.actions.impl.primitive.Kubernetes; import oracle.weblogic.kubernetes.annotations.IntegrationTest; import oracle.weblogic.kubernetes.annotations.Namespaces; +import oracle.weblogic.kubernetes.extensions.InitializationTasks; import oracle.weblogic.kubernetes.logging.LoggingFacade; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -25,10 +22,11 @@ import static oracle.weblogic.kubernetes.TestConstants.MII_BASIC_IMAGE_NAME; import static oracle.weblogic.kubernetes.TestConstants.MII_BASIC_IMAGE_TAG; import static oracle.weblogic.kubernetes.TestConstants.SKIP_CLEANUP; -import static oracle.weblogic.kubernetes.assertions.impl.Kubernetes.getService; import static oracle.weblogic.kubernetes.utils.ApplicationUtils.callWebAppAndCheckForServerNameInResponse; import static oracle.weblogic.kubernetes.utils.CommonMiiTestUtils.createMiiDomainAndVerify; import static oracle.weblogic.kubernetes.utils.ImageUtils.createTestRepoSecret; +import static oracle.weblogic.kubernetes.utils.LoadBalancerUtils.deleteLoadBalancer; +import static oracle.weblogic.kubernetes.utils.LoadBalancerUtils.getLoadBalancerIP; import static oracle.weblogic.kubernetes.utils.LoadBalancerUtils.installAndVerifyOCILoadBalancer; import static oracle.weblogic.kubernetes.utils.OperatorUtils.installAndVerifyOperator; import static oracle.weblogic.kubernetes.utils.ThreadSafeLogger.getLogger; @@ -45,7 +43,7 @@ + "all managed servers in the domain through OCI Load Balancer") @IntegrationTest @Tag("oke-arm") -@Tag("oke-parallel") +@Tag("oke-gate") class ItOCILoadBalancer { // domain constants private static final int replicaCount = 2; @@ -88,6 +86,7 @@ public void tearDownAll() { if (!SKIP_CLEANUP) { Kubernetes.deleteService(OCI_LB_NAME, domainNamespace); } + deleteLoadBalancer(loadBalancerIP); } /** @@ -118,37 +117,14 @@ void testOCILoadBalancer() throws Exception { assertDoesNotThrow(() -> installAndVerifyOCILoadBalancer(domainNamespace, clusterHttpPort, clusterName, domainUid, OCI_LB_NAME), "Installation of OCI Load Balancer failed"); - loadBalancerIP = getLoadBalancerIP(domainNamespace,OCI_LB_NAME); + loadBalancerIP = getLoadBalancerIP(domainNamespace,OCI_LB_NAME, true); assertNotNull(loadBalancerIP, "External IP for Load Balancer is undefined"); + InitializationTasks.registerLoadBalancerExternalIP(loadBalancerIP); + logger.info("LoadBalancer IP is " + loadBalancerIP); verifyWebAppAccessThroughOCILoadBalancer(loadBalancerIP, 2, clusterHttpPort); } - /** - * Retreive external IP from OCI LoadBalancer. - */ - private static String getLoadBalancerIP(String namespace, String lbName) throws Exception { - Map labels = new HashMap<>(); - labels.put("loadbalancer", lbName); - V1Service service = getService(lbName, labels, namespace); - assertNotNull(service, "Can't find service with name " + lbName); - logger.info("Found service with name {0} in {1} namespace ", lbName, namespace); - assertNotNull(service.getStatus(), "service status is null"); - assertNotNull(service.getStatus().getLoadBalancer(), "service loadbalancer is null"); - List ingress = service.getStatus().getLoadBalancer().getIngress(); - if (ingress != null) { - logger.info("LoadBalancer Ingress " + ingress.toString()); - V1LoadBalancerIngress lbIng = ingress.stream().filter(c -> - c.getIp() != null && !c.getIp().equals("pending") - ).findAny().orElse(null); - if (lbIng != null) { - logger.info("OCI LoadBalancer is created with external ip" + lbIng.getIp()); - return lbIng.getIp(); - } - } - return null; - } - /** * Verify the sample-app app can be accessed from all managed servers in the domain through OCI Load Balancer. */ diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/TestConstants.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/TestConstants.java index 4052b92fd48..d39edd904bb 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/TestConstants.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/TestConstants.java @@ -370,6 +370,7 @@ public interface TestConstants { Boolean.parseBoolean(getNonEmptySystemProperty("wko.it.oke.cluster.privateip", "false")); public static final String NFS_SERVER = System.getProperty("wko.it.nfs.server", ""); public static final String NODE_IP = System.getProperty("wko.it.node.ip", ""); + public static final String COMPARTMENT_OCID = System.getProperty("wko.it.oci.compartment.ocid", ""); public static final String [] FSS_DIR = System.getProperty("wko.it.fss.dir","").split(","); public static final String IMAGE_PULL_POLICY = "IfNotPresent"; diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/extensions/InitializationTasks.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/extensions/InitializationTasks.java index 405acd0f29e..186569a27a8 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/extensions/InitializationTasks.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/extensions/InitializationTasks.java @@ -11,9 +11,11 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; @@ -113,6 +115,7 @@ import static oracle.weblogic.kubernetes.utils.FileUtils.cleanupDirectory; import static oracle.weblogic.kubernetes.utils.IstioUtils.installIstio; import static oracle.weblogic.kubernetes.utils.IstioUtils.uninstallIstio; +import static oracle.weblogic.kubernetes.utils.LoadBalancerUtils.deleteLoadBalancer; import static oracle.weblogic.kubernetes.utils.LoadBalancerUtils.installAndVerifyTraefik; import static oracle.weblogic.kubernetes.utils.OperatorUtils.installAndVerifyOperator; import static oracle.weblogic.kubernetes.utils.ThreadSafeLogger.getLogger; @@ -133,6 +136,7 @@ public class InitializationTasks implements BeforeAllCallback, ExtensionContext. private static String wdtBasicImage; private static Collection pushedImages = new ArrayList<>(); + private static Set lbIPs = new HashSet<>(); private static boolean isInitializationSuccessful = false; ConditionFactory withVeryLongRetryPolicy @@ -355,6 +359,15 @@ public static void registerPushedImage(String imageName) { pushedImages.add(imageName); } + /** + * Called when load balancer is created in OCI, allowing conditional cleanup of load balancers. + * + * @param lbIP external IP of load balancer. + */ + public static void registerLoadBalancerExternalIP(String lbIP) { + lbIPs.add(lbIP); + } + @Override public void close() { LoggingFacade logger = getLogger(); @@ -395,6 +408,13 @@ public void close() { for (String image : pushedImages) { deleteImage(image); } + if (OKE_CLUSTER) { + logger.info("Cleanup created in OCI load balancers after all test suites are run"); + // delete all load balancers in OCI + for (String ip : lbIPs) { + deleteLoadBalancer(ip); + } + } } // delete images from TEST_IMAGES_REPO, if necessary diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/IstioUtils.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/IstioUtils.java index 28959823cb8..d9f6e98498c 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/IstioUtils.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/IstioUtils.java @@ -24,6 +24,7 @@ import oracle.weblogic.domain.OnlineUpdate; import oracle.weblogic.domain.ServerPod; import oracle.weblogic.kubernetes.actions.impl.primitive.Command; +import oracle.weblogic.kubernetes.extensions.InitializationTasks; import oracle.weblogic.kubernetes.logging.LoggingFacade; import oracle.weblogic.kubernetes.utils.SemanticVersion.Compatibility; @@ -37,6 +38,7 @@ import static oracle.weblogic.kubernetes.TestConstants.ISTIO_VERSION; import static oracle.weblogic.kubernetes.TestConstants.KUBERNETES_CLI; import static oracle.weblogic.kubernetes.TestConstants.OCNE; +import static oracle.weblogic.kubernetes.TestConstants.OKE_CLUSTER; import static oracle.weblogic.kubernetes.TestConstants.OKE_CLUSTER_PRIVATEIP; import static oracle.weblogic.kubernetes.TestConstants.PROMETHEUS_CONFIG_MAP_RELOAD_IMAGE_NAME; import static oracle.weblogic.kubernetes.TestConstants.PROMETHEUS_CONFIG_MAP_RELOAD_IMAGE_TAG; @@ -51,9 +53,12 @@ import static oracle.weblogic.kubernetes.utils.ClusterUtils.createClusterResourceAndAddReferenceToDomain; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkServiceExists; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.formatIPv6Host; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.getServiceExtIPAddrtOke; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.testUntil; import static oracle.weblogic.kubernetes.utils.ExecCommand.exec; import static oracle.weblogic.kubernetes.utils.FileUtils.generateFileFromTemplate; import static oracle.weblogic.kubernetes.utils.FileUtils.replaceStringInFile; +import static oracle.weblogic.kubernetes.utils.LoadBalancerUtils.isLoadBalancerHealthy; import static oracle.weblogic.kubernetes.utils.PodUtils.checkPodReady; import static oracle.weblogic.kubernetes.utils.PodUtils.setPodAntiAffinity; import static oracle.weblogic.kubernetes.utils.ThreadSafeLogger.getLogger; @@ -112,6 +117,17 @@ public static void installIstio() { .command(command) .redirect(false)) .execute()); + + if (OKE_CLUSTER) { + String loadBalancerIP = getServiceExtIPAddrtOke("istio-ingressgateway", "istio-system"); + testUntil( + assertDoesNotThrow(() -> isLoadBalancerHealthy("istio-system", "istio-ingressgateway"), + "isLoadBalancerHealthy failed with ApiException"), + logger, + "Istio LoadBalancer to be healthy in namespace {0}", + "istio-system"); + InitializationTasks.registerLoadBalancerExternalIP(loadBalancerIP); + } } /** diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/LoadBalancerUtils.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/LoadBalancerUtils.java index 00d4e2b5c39..92f52f28b64 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/LoadBalancerUtils.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/LoadBalancerUtils.java @@ -12,6 +12,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.Callable; import io.kubernetes.client.custom.IntOrString; import io.kubernetes.client.openapi.ApiException; @@ -21,6 +22,7 @@ import io.kubernetes.client.openapi.models.V1IngressRule; import io.kubernetes.client.openapi.models.V1IngressServiceBackend; import io.kubernetes.client.openapi.models.V1IngressTLS; +import io.kubernetes.client.openapi.models.V1LoadBalancerIngress; import io.kubernetes.client.openapi.models.V1ObjectMeta; import io.kubernetes.client.openapi.models.V1Service; import io.kubernetes.client.openapi.models.V1ServiceBackendPort; @@ -29,10 +31,13 @@ import oracle.weblogic.kubernetes.TestConstants; import oracle.weblogic.kubernetes.actions.impl.NginxParams; import oracle.weblogic.kubernetes.actions.impl.TraefikParams; +import oracle.weblogic.kubernetes.actions.impl.primitive.Command; import oracle.weblogic.kubernetes.actions.impl.primitive.HelmParams; +import oracle.weblogic.kubernetes.extensions.InitializationTasks; import oracle.weblogic.kubernetes.logging.LoggingFacade; import static oracle.weblogic.kubernetes.TestConstants.ADMIN_SERVER_NAME_BASE; +import static oracle.weblogic.kubernetes.TestConstants.COMPARTMENT_OCID; import static oracle.weblogic.kubernetes.TestConstants.K8S_NODEPORT_HOST; import static oracle.weblogic.kubernetes.TestConstants.KUBERNETES_CLI; import static oracle.weblogic.kubernetes.TestConstants.NGINX_CHART_NAME; @@ -40,6 +45,7 @@ import static oracle.weblogic.kubernetes.TestConstants.NGINX_RELEASE_NAME; import static oracle.weblogic.kubernetes.TestConstants.NGINX_REPO_NAME; import static oracle.weblogic.kubernetes.TestConstants.NGINX_REPO_URL; +import static oracle.weblogic.kubernetes.TestConstants.OKE_CLUSTER; import static oracle.weblogic.kubernetes.TestConstants.RESULTS_ROOT; import static oracle.weblogic.kubernetes.TestConstants.TEST_IMAGES_REPO_SECRET_NAME; import static oracle.weblogic.kubernetes.TestConstants.TRAEFIK_CHART_NAME; @@ -53,10 +59,12 @@ import static oracle.weblogic.kubernetes.actions.TestActions.installTraefik; import static oracle.weblogic.kubernetes.actions.TestActions.listIngresses; import static oracle.weblogic.kubernetes.actions.TestActions.upgradeTraefikImage; +import static oracle.weblogic.kubernetes.actions.impl.primitive.Command.defaultCommandParams; import static oracle.weblogic.kubernetes.assertions.TestAssertions.isHelmReleaseDeployed; import static oracle.weblogic.kubernetes.assertions.TestAssertions.isNginxReady; import static oracle.weblogic.kubernetes.assertions.TestAssertions.isOCILoadBalancerReady; import static oracle.weblogic.kubernetes.assertions.TestAssertions.isTraefikReady; +import static oracle.weblogic.kubernetes.assertions.impl.Kubernetes.getService; import static oracle.weblogic.kubernetes.utils.ApplicationUtils.callWebAppAndWaitTillReady; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodReadyAndServiceExists; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkServiceExists; @@ -126,6 +134,14 @@ public static void installAndVerifyOCILoadBalancer( logger, "external IP to be generated in {0}", namespace); + if (OKE_CLUSTER) { + testUntil( + assertDoesNotThrow(() -> isLoadBalancerHealthy(namespace, loadBalancerName), + "isLoadBalancerHealthy failed with ApiException"), + logger, + "OCI LB to be healthy in namespace {0}", + namespace); + } } /** @@ -212,6 +228,22 @@ public static NginxParams installAndVerifyNginx(String nginxNamespace, logger, "NGINX to be ready in namespace {0}", nginxNamespace); + if (OKE_CLUSTER) { + testUntil( + assertDoesNotThrow(() -> isOCILoadBalancerReady( + nginxHelmParams.getReleaseName() + "-ingress-nginx-controller", + null, nginxNamespace), "isOCILoadBalancerReady failed with ApiException"), + logger, + "external IP to be generated in {0}", + nginxNamespace); + testUntil( + assertDoesNotThrow(() -> isLoadBalancerHealthy(nginxNamespace, + nginxHelmParams.getReleaseName() + "-ingress-nginx-controller"), + "isLoadBalancerHealthy failed with ApiException"), + logger, + "NGINX LB to be healthy in namespace {0}", + nginxNamespace); + } return nginxParams; } @@ -282,10 +314,117 @@ public static TraefikParams installAndVerifyTraefik(String traefikNamespace, logger, "Traefik to be ready in namespace {0}", traefikNamespace); + if (OKE_CLUSTER) { + testUntil( + assertDoesNotThrow(() -> isOCILoadBalancerReady( + traefikHelmParams.getReleaseName(), + null, traefikNamespace), "isOCILoadBalancerReady failed with ApiException"), + logger, + "external IP to be generated in {0}", + traefikNamespace); + testUntil( + assertDoesNotThrow(() -> isLoadBalancerHealthy(traefikNamespace, traefikHelmParams.getReleaseName()), + "isLoadBalancerHealthy failed with ApiException"), + logger, + "Traefik to be healthy in namespace {0}", + traefikNamespace); + } return traefikParams; } + /** + * Check lb has healty status. + * + * @param namespace in which to check for lb controller + * @name service name of lb controller + * @return true if healthy, false otherwise + */ + public static Callable isLoadBalancerHealthy(String namespace, String name) { + return () -> checkLoadBalancerHealthy(namespace, name); + } + + /** + * Retreive external IP from OCI LoadBalancer. + * + * @param namespace - namespace + * @param lbName -loadbalancer service name + */ + public static String getLoadBalancerIP(String namespace, String lbName) throws Exception { + return getLoadBalancerIP(namespace, lbName, false); + } + + /** + * Retreive external IP from OCI LoadBalancer. + * + * @param namespace - namespace + * @param lbName -loadbalancer service name + * @param addLabel search service with label + */ + public static String getLoadBalancerIP(String namespace, String lbName, boolean addLabel) throws Exception { + Map labels = null; + if (addLabel) { + labels = new HashMap<>(); + labels.put("loadbalancer", lbName); + } + V1Service service = getService(lbName, labels, namespace); + assertNotNull(service, "Can't find service with name " + lbName); + LoggingFacade logger = getLogger(); + logger.info("Found service with name {0} in {1} namespace ", lbName, namespace); + assertNotNull(service.getStatus(), "service status is null"); + assertNotNull(service.getStatus().getLoadBalancer(), "service loadbalancer is null"); + List ingress = service.getStatus().getLoadBalancer().getIngress(); + if (ingress != null) { + logger.info("LoadBalancer Ingress " + ingress.toString()); + V1LoadBalancerIngress lbIng = + ingress.stream().filter(c -> c.getIp() != null && !c.getIp().equals("pending")).findAny().orElse(null); + if (lbIng != null) { + logger.info("OCI LoadBalancer is created with external ip" + lbIng.getIp()); + return lbIng.getIp(); + } + } + return null; + } + + private static boolean checkLoadBalancerHealthy(String namespace, String lbServiceName) { + String lbPublicIP = assertDoesNotThrow(() -> getLoadBalancerIP(namespace, lbServiceName)); + InitializationTasks.registerLoadBalancerExternalIP(lbPublicIP); + LoggingFacade logger = getLogger(); + String testcompartmentid = System.getProperty("wko.it.oci.compartment.ocid"); + logger.info("wko.it.oci.compartment.ocid property " + testcompartmentid); + + final String command = "oci lb load-balancer list --compartment-id " + + testcompartmentid + " --query \"data[?contains(\\\"ip-addresses\\\"[0].\\\"ip-address\\\", '" + + lbPublicIP + "')].id | [0]\" --raw-output --all"; + + logger.info("Command to retrieve Load Balancer OCID is: {0} ", command); + + ExecResult result = assertDoesNotThrow(() -> exec(command, true)); + logger.info("The command returned exit value: " + result.exitValue() + + " command output: " + result.stderr() + "\n" + result.stdout()); + + if (result == null || result.exitValue() != 0 || result.stdout() == null) { + return false; + } + + // Clean up the string to extract the Load Balancer ID + String lbOCID = result.stdout().trim(); + + //check health status + final String command1 = "oci lb load-balancer-health get --load-balancer-id " + lbOCID; + logger.info("Command to retrieve Load Balancer health status is: {0} ", command1); + result = assertDoesNotThrow(() -> exec(command1, true)); + logger.info("The command returned exit value: " + result.exitValue() + + " command output: " + result.stderr() + "\n" + result.stdout()); + + if (result == null || result.exitValue() != 0 || result.stdout() == null) { + return false; + } + + return result.stdout().contains("OK"); + + } + /** Upgrade Traefik and wait for up to five minutes for the Traefik pod to be ready. * * @param traefikNamespace the namespace in which the Traefik ingress controller is installed @@ -733,4 +872,39 @@ public static String getLbExternalIp(String lbrelname, String lbns) throws Excep return result.stdout().trim(); } + + /** + * Delete Load Balancer , created in OCI by using provided public IP. + * + * @param lbPublicIP public Load Balancer IP + */ + public static void deleteLoadBalancer(String lbPublicIP) { + if (lbPublicIP != null && !lbPublicIP.isEmpty()) { + if (lbPublicIP.startsWith("[") && lbPublicIP.endsWith("]")) { + // Remove the brackets and return the content inside + lbPublicIP = lbPublicIP.substring(1, lbPublicIP.length() - 1); + } + LoggingFacade logger = getLogger(); + Path deleteLBPath = + Paths.get(RESOURCE_DIR, "bash-scripts", "delete_loadbalancer.sh"); + String deleteLBScript = deleteLBPath.toString(); + String command = + String.format("chmod 777 %s ", deleteLBScript); + logger.info("Delete Load Balancer command {0}", command); + assertTrue(() -> Command.withParams( + defaultCommandParams() + .command(command) + .redirect(true)) + .execute()); + + String command1 = + String.format("%s %s %s ", deleteLBScript, lbPublicIP, COMPARTMENT_OCID); + logger.info("Delete Load Balancer command {0}", command); + assertTrue(() -> Command.withParams( + defaultCommandParams() + .command(command1) + .redirect(true)) + .execute()); + } + } } diff --git a/integration-tests/src/test/resources/bash-scripts/delete_loadbalancer.sh b/integration-tests/src/test/resources/bash-scripts/delete_loadbalancer.sh new file mode 100644 index 00000000000..e69de29bb2d