From 1df2aa8ceee7f436f7b01b5a1bf6d128cd6875d3 Mon Sep 17 00:00:00 2001 From: "MARINA.KOGAN" Date: Wed, 3 Jul 2024 21:45:24 +0000 Subject: [PATCH] Backport changes for Converted DBOperator tests from external to internal run in OKE --- Jenkinsfile.oke | 2 +- .../weblogic/kubernetes/ItDBOperator.java | 141 +++++++++--------- .../kubernetes/ItFmwDomainInPVUsingWDT.java | 2 +- .../kubernetes/ItFmwDomainInPVUsingWLST.java | 2 +- .../ItFmwDomainInPvUserCreateRcu.java | 2 +- .../weblogic/kubernetes/ItFmwDomainOnPV.java | 14 +- .../kubernetes/ItFmwDynamicDomainInPV.java | 41 ++++- .../kubernetes/utils/CommonMiiTestUtils.java | 33 ++-- .../kubernetes/utils/CommonTestUtils.java | 9 +- 9 files changed, 147 insertions(+), 99 deletions(-) diff --git a/Jenkinsfile.oke b/Jenkinsfile.oke index 1d195c5d0b6..b00390a1a21 100644 --- a/Jenkinsfile.oke +++ b/Jenkinsfile.oke @@ -141,7 +141,7 @@ pipeline { ) string(name: 'NODE_SHAPE', description: '', - defaultValue: "VM.Standard.E3.Flex" + defaultValue: "VM.Standard.E4.Flex" ) string(name: 'MOUNT_TARGET_OCID', description: 'only for debug runs on wko-oke-dev', diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItDBOperator.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItDBOperator.java index 144aa1b7d03..26d04cea7e3 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItDBOperator.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItDBOperator.java @@ -15,8 +15,10 @@ import io.kubernetes.client.openapi.models.V1LocalObjectReference; import oracle.weblogic.domain.ClusterResource; import oracle.weblogic.domain.DomainResource; +import oracle.weblogic.kubernetes.actions.impl.TraefikParams; import oracle.weblogic.kubernetes.actions.impl.primitive.Command; import oracle.weblogic.kubernetes.actions.impl.primitive.CommandParams; +import oracle.weblogic.kubernetes.actions.impl.primitive.HelmParams; import oracle.weblogic.kubernetes.annotations.IntegrationTest; import oracle.weblogic.kubernetes.annotations.Namespaces; import oracle.weblogic.kubernetes.logging.LoggingFacade; @@ -41,18 +43,18 @@ import static oracle.weblogic.kubernetes.TestConstants.MII_BASIC_APP_NAME; import static oracle.weblogic.kubernetes.TestConstants.MII_BASIC_IMAGE_NAME; import static oracle.weblogic.kubernetes.TestConstants.MII_BASIC_IMAGE_TAG; +import static oracle.weblogic.kubernetes.TestConstants.OKE_CLUSTER_PRIVATEIP; import static oracle.weblogic.kubernetes.TestConstants.SKIP_CLEANUP; import static oracle.weblogic.kubernetes.TestConstants.TEST_IMAGES_REPO_SECRET_NAME; -import static oracle.weblogic.kubernetes.TestConstants.TRAEFIK_INGRESS_HTTP_HOSTPORT; import static oracle.weblogic.kubernetes.actions.ActionConstants.ITTESTS_DIR; import static oracle.weblogic.kubernetes.actions.ActionConstants.MODEL_DIR; import static oracle.weblogic.kubernetes.actions.ActionConstants.RESOURCE_DIR; import static oracle.weblogic.kubernetes.actions.ActionConstants.WORK_DIR; import static oracle.weblogic.kubernetes.actions.TestActions.createDomainCustomResource; import static oracle.weblogic.kubernetes.actions.TestActions.execCommand; -import static oracle.weblogic.kubernetes.actions.TestActions.getServiceNodePort; import static oracle.weblogic.kubernetes.actions.TestActions.patchDomainCustomResource; import static oracle.weblogic.kubernetes.actions.TestActions.scaleCluster; +import static oracle.weblogic.kubernetes.actions.TestActions.uninstallTraefik; import static oracle.weblogic.kubernetes.actions.impl.primitive.Command.defaultCommandParams; import static oracle.weblogic.kubernetes.assertions.TestAssertions.domainExists; import static oracle.weblogic.kubernetes.utils.ClusterUtils.createClusterAndVerify; @@ -61,9 +63,9 @@ import static oracle.weblogic.kubernetes.utils.CommonMiiTestUtils.createDomainResourceWithLogHome; import static oracle.weblogic.kubernetes.utils.CommonMiiTestUtils.createDomainSecret; import static oracle.weblogic.kubernetes.utils.CommonMiiTestUtils.createJobToChangePermissionsOnPvHostPath; +import static oracle.weblogic.kubernetes.utils.CommonMiiTestUtils.readRuntimeResource; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodReadyAndServiceExists; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createIngressHostRouting; -import static oracle.weblogic.kubernetes.utils.CommonTestUtils.getHostAndPort; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.getUniqueName; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.runClientInsidePod; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.runJavacInsidePod; @@ -74,7 +76,6 @@ import static oracle.weblogic.kubernetes.utils.DbUtils.createRcuSchema; import static oracle.weblogic.kubernetes.utils.DbUtils.deleteOracleDB; import static oracle.weblogic.kubernetes.utils.DomainUtils.createDomainAndVerify; -import static oracle.weblogic.kubernetes.utils.ExecCommand.exec; import static oracle.weblogic.kubernetes.utils.FileUtils.copyFileToPod; import static oracle.weblogic.kubernetes.utils.FmwUtils.verifyDomainReady; import static oracle.weblogic.kubernetes.utils.FmwUtils.verifyEMconsoleAccess; @@ -82,6 +83,7 @@ import static oracle.weblogic.kubernetes.utils.ImageUtils.createMiiImageAndVerify; import static oracle.weblogic.kubernetes.utils.ImageUtils.createTestRepoSecret; import static oracle.weblogic.kubernetes.utils.ImageUtils.imageRepoLoginAndPushImageToRegistry; +import static oracle.weblogic.kubernetes.utils.LoadBalancerUtils.installAndVerifyTraefik; import static oracle.weblogic.kubernetes.utils.OKDUtils.createRouteForOKD; import static oracle.weblogic.kubernetes.utils.OperatorUtils.installAndVerifyOperator; import static oracle.weblogic.kubernetes.utils.PatchDomainUtils.patchDomainResourceServerStartPolicy; @@ -93,6 +95,7 @@ import static oracle.weblogic.kubernetes.utils.SecretUtils.createOpsswalletpasswordSecret; import static oracle.weblogic.kubernetes.utils.SecretUtils.createSecretWithUsernamePassword; import static oracle.weblogic.kubernetes.utils.ThreadSafeLogger.getLogger; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -106,7 +109,7 @@ @DisplayName("Test to create FMW model in image domain and WebLogic domain using Oracle " + "database created using Oracle Database Operator") @IntegrationTest -@Tag("oke-sequential") +@Tag("oke-gate") @Tag("kind-parallel") class ItDBOperator { @@ -114,6 +117,7 @@ class ItDBOperator { private static String opNamespace = null; private static String fmwDomainNamespace = null; private static String wlsDomainNamespace = null; + private static String traefikNamespace = null; private static String fmwMiiImage = null; private static final String RCUSCHEMAPREFIX = "FMWDOMAINMII"; @@ -124,6 +128,7 @@ class ItDBOperator { private static String dbUrl = null; private static String dbName = "my-oracle-sidb"; private static LoggingFacade logger = null; + private static HelmParams traefikHelmParams; private String fmwDomainUid = "fmwdomain-mii-db"; private String adminServerName = "admin-server"; @@ -153,6 +158,7 @@ class ItDBOperator { private final String wlsClusterResName = wlsDomainUid + "-" + clusterName; private static String hostHeader; + private static TraefikParams traefikParams; /** * Start DB service and create RCU schema. @@ -162,7 +168,7 @@ class ItDBOperator { * @param namespaces injected by JUnit */ @BeforeAll - public static void initAll(@Namespaces(4) List namespaces) { + public static void initAll(@Namespaces(5) List namespaces) { logger = getLogger(); logger.info("Assign a unique namespace for DB and RCU"); @@ -181,6 +187,18 @@ public static void initAll(@Namespaces(4) List namespaces) { assertNotNull(namespaces.get(3), "Namespace is null"); wlsDomainNamespace = namespaces.get(3); + // get a unique Traefik namespace + logger.info("Get a unique namespace for Traefik"); + assertNotNull(namespaces.get(4), "Namespace list is null"); + traefikNamespace = namespaces.get(4); + + // install and verify Traefik + if (OKE_CLUSTER_PRIVATEIP) { + traefikParams = + installAndVerifyTraefik(traefikNamespace, 0, 0); + traefikHelmParams = traefikParams.getHelmParams(); + } + // Create the repo secret to pull the image // this secret is used only for non-kind cluster createBaseRepoSecret(fmwDomainNamespace); @@ -199,6 +217,17 @@ public static void initAll(@Namespaces(4) List namespaces) { installAndVerifyOperator(opNamespace, fmwDomainNamespace, wlsDomainNamespace); } + @AfterAll + void tearDown() { + // uninstall Traefik + if (traefikHelmParams != null) { + assertThat(uninstallTraefik(traefikHelmParams)) + .as("Test uninstallTraefik returns true") + .withFailMessage("uninstallTraefik() did not return true") + .isTrue(); + } + } + /** * Create a basic FMW model in image domain using the database created by DB Operator. Verify Pod is ready and service * exists for both admin server and managed servers. Verify EM console is accessible. @@ -553,34 +582,28 @@ private void runJmsClientOnAdminPod(String action, String queue) { * @returns true if MBean is found otherwise false **/ private boolean checkJmsServerRuntime(String jmsServer, String managedServer) { - ExecResult result = null; - int adminServiceNodePort - = getServiceNodePort(wlsDomainNamespace, getExternalServicePodName(wlsAdminServerPodName), "default"); - String hostAndPort = getHostAndPort(adminSvcExtRouteHost, adminServiceNodePort); - StringBuffer curlString = new StringBuffer("status=$(curl --user " - + ADMIN_USERNAME_DEFAULT + ":" + ADMIN_PASSWORD_DEFAULT + " "); - if (TestConstants.KIND_CLUSTER - && !TestConstants.WLSIMG_BUILDER.equals(TestConstants.WLSIMG_BUILDER_DEFAULT)) { - hostAndPort = "localhost:" + TRAEFIK_INGRESS_HTTP_HOSTPORT; - curlString.append(" -H 'host: " + hostHeader + "' "); - } - curlString.append("http://" + hostAndPort) - .append("/management/weblogic/latest/domainRuntime/serverRuntimes/") - .append(managedServer) - .append("/JMSRuntime/JMSServers/") - .append(jmsServer) - .append(" --silent --show-error ") - .append(" -o /dev/null") - .append(" -w %{http_code});") - .append("echo ${status}"); - logger.info("checkJmsServerRuntime: curl command {0}", new String(curlString)); testUntil( - assertDoesNotThrow(() -> () -> exec(curlString.toString(), true).stdout().contains("200")), + assertDoesNotThrow(() -> () -> getJMSRunTimeOutput(jmsServer, + managedServer).contains("destinationsCurrentCount")), logger, "JMS Server Service to migrate"); return true; } + private String getJMSRunTimeOutput(String jmsServer, String managedServer) { + String output = readRuntimeResource( + adminSvcExtHost, + wlsDomainNamespace, + wlsAdminServerPodName, + "/management/weblogic/latest/domainRuntime/serverRuntimes/" + + managedServer + + "/JMSRuntime/JMSServers/" + + jmsServer, + "checkJmsServerRuntime"); + logger.info("Got output " + output); + return output; + } + /* * Verify the Persistent Store Runtimes through REST API. * Get the specific Persistent Store Runtime on specified managed server. @@ -589,31 +612,19 @@ private boolean checkJmsServerRuntime(String jmsServer, String managedServer) { * @returns true if MBean is found otherwise false **/ private boolean checkStoreRuntime(String storeName, String managedServer) { - ExecResult result = null; - int adminServiceNodePort - = getServiceNodePort(wlsDomainNamespace, getExternalServicePodName(wlsAdminServerPodName), "default"); - String hostAndPort = getHostAndPort(adminSvcExtRouteHost, adminServiceNodePort); - StringBuffer curlString = new StringBuffer("status=$(curl --user " - + ADMIN_USERNAME_DEFAULT + ":" + ADMIN_PASSWORD_DEFAULT + " "); - if (TestConstants.KIND_CLUSTER - && !TestConstants.WLSIMG_BUILDER.equals(TestConstants.WLSIMG_BUILDER_DEFAULT)) { - hostAndPort = "localhost:" + TRAEFIK_INGRESS_HTTP_HOSTPORT; - curlString.append(" -H 'host: " + hostHeader + "' "); - } - curlString.append("http://" + hostAndPort) - .append("/management/weblogic/latest/domainRuntime/serverRuntimes/") - .append(managedServer) - .append("/persistentStoreRuntimes/") - .append(storeName) - .append(" --silent --show-error ") - .append(" -o /dev/null") - .append(" -w %{http_code});") - .append("echo ${status}"); - logger.info("checkStoreRuntime: curl command {0}", new String(curlString)); testUntil( - assertDoesNotThrow(() -> () -> exec(curlString.toString(), true).stdout().contains("200")), + assertDoesNotThrow(() -> () -> readRuntimeResource( + adminSvcExtHost, + wlsDomainNamespace, + wlsAdminServerPodName, + "/management/weblogic/latest/domainRuntime/serverRuntimes/" + + managedServer + + "/persistentStoreRuntimes/" + + storeName, + "checkPersistentStoreRuntime").contains("PersistentStoreRuntime")), logger, "PersistentStoreRuntimes Service to migrate"); + return true; } @@ -627,28 +638,16 @@ private boolean checkStoreRuntime(String storeName, String managedServer) { * @returns true if MBean is found otherwise false **/ private boolean checkJtaRecoveryServiceRuntime(String managedServer, String recoveryService, String active) { - ExecResult result = null; - int adminServiceNodePort - = getServiceNodePort(wlsDomainNamespace, getExternalServicePodName(wlsAdminServerPodName), "default"); - String hostAndPort = getHostAndPort(adminSvcExtRouteHost, adminServiceNodePort); - StringBuffer curlString = new StringBuffer("curl --user " - + ADMIN_USERNAME_DEFAULT + ":" + ADMIN_PASSWORD_DEFAULT + " "); - if (TestConstants.KIND_CLUSTER - && !TestConstants.WLSIMG_BUILDER.equals(TestConstants.WLSIMG_BUILDER_DEFAULT)) { - hostAndPort = "localhost:" + TRAEFIK_INGRESS_HTTP_HOSTPORT; - curlString.append(" -H 'host: " + hostHeader + "' "); - } - curlString.append("\"http://" + hostAndPort) - .append("/management/weblogic/latest/domainRuntime/serverRuntimes/") - .append(managedServer) - .append("/JTARuntime/recoveryRuntimeMBeans/") - .append(recoveryService) - .append("?fields=active&links=none\"") - .append(" --show-error "); - logger.info("checkJtaRecoveryServiceRuntime: curl command {0}", new String(curlString)); testUntil( - assertDoesNotThrow(() -> () -> exec(curlString.toString(), true) - .stdout().contains("\"active\": " + active)), + assertDoesNotThrow(() -> () -> readRuntimeResource( + adminSvcExtHost, + wlsDomainNamespace, + wlsAdminServerPodName, + "/management/weblogic/latest/domainRuntime/serverRuntimes/" + + managedServer + + "/JTARuntime/recoveryRuntimeMBeans/" + + recoveryService, + "checkRecoveryServiceRuntime").contains("\"active\": " + active)), logger, "JTA Recovery Service to migrate"); return true; diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDomainInPVUsingWDT.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDomainInPVUsingWDT.java index 6fcfa5353c7..b97d6888dd5 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDomainInPVUsingWDT.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDomainInPVUsingWDT.java @@ -59,7 +59,7 @@ */ @DisplayName("Test to creat a FMW domain in persistent volume using WDT") @IntegrationTest -@Tag("oke-sequential") +@Tag("oke-gate") @Tag("kind-sequential") @Tag("okd-fmw-cert") class ItFmwDomainInPVUsingWDT { diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDomainInPVUsingWLST.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDomainInPVUsingWLST.java index d9b83d04e6b..b8616b28061 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDomainInPVUsingWLST.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDomainInPVUsingWLST.java @@ -72,7 +72,7 @@ @Tag("okd-fmw-cert") @IntegrationTest @Tag("olcne-sequential") -@Tag("oke-sequential1") +@Tag("oke-gate") class ItFmwDomainInPVUsingWLST { private static String dbNamespace = null; diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDomainInPvUserCreateRcu.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDomainInPvUserCreateRcu.java index 5c40727fe70..544e7ac413e 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDomainInPvUserCreateRcu.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDomainInPvUserCreateRcu.java @@ -99,7 +99,7 @@ @DisplayName("Test for initializeDomainOnPV when user per-creates RCU") @IntegrationTest @Tag("kind-sequential") -@Tag("oke-sequential1") +@Tag("oke-gate") public class ItFmwDomainInPvUserCreateRcu { private static String opNamespace = null; diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDomainOnPV.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDomainOnPV.java index cfec7cbbdc8..46dc2322728 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDomainOnPV.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDomainOnPV.java @@ -39,7 +39,6 @@ import static oracle.weblogic.kubernetes.TestConstants.ADMIN_USERNAME_DEFAULT; import static oracle.weblogic.kubernetes.TestConstants.BASE_IMAGES_PREFIX; import static oracle.weblogic.kubernetes.TestConstants.BASE_IMAGES_REPO_SECRET_NAME; -import static oracle.weblogic.kubernetes.TestConstants.DB_IMAGE_TO_USE_IN_SPEC; import static oracle.weblogic.kubernetes.TestConstants.DOMAIN_IMAGES_PREFIX; import static oracle.weblogic.kubernetes.TestConstants.ELASTICSEARCH_HOST; import static oracle.weblogic.kubernetes.TestConstants.FMWINFRA_IMAGE_TO_USE_IN_SPEC; @@ -64,9 +63,9 @@ import static oracle.weblogic.kubernetes.utils.CommonTestUtils.getNextFreePort; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.getUniqueName; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.testUntil; +import static oracle.weblogic.kubernetes.utils.DbUtils.createOracleDBUsingOperator; import static oracle.weblogic.kubernetes.utils.DbUtils.createRcuAccessSecret; import static oracle.weblogic.kubernetes.utils.DbUtils.createRcuSchema; -import static oracle.weblogic.kubernetes.utils.DbUtils.startOracleDB; import static oracle.weblogic.kubernetes.utils.DomainUtils.createDomainAndVerify; import static oracle.weblogic.kubernetes.utils.DomainUtils.createDomainResourceOnPv; import static oracle.weblogic.kubernetes.utils.DomainUtils.deleteDomainResource; @@ -94,7 +93,7 @@ @DisplayName("Test to create a FMW domain in persistent volume with new simplified feature") @IntegrationTest @Tag("kind-sequential") -@Tag("oke-sequential1") +@Tag("oke-gate") @Tag("okd-fmw-cert") class ItFmwDomainOnPV { @@ -144,13 +143,18 @@ public static void initAll(@Namespaces(3) List namespaces) { DOMAINHOMEPREFIX = "/shared/" + domainNamespace + "/domains/"; - // start DB + //install Oracle Database Operator + String dbName = "fmwdomainonpv1" + "my-oracle-db"; + logger.info("Create Oracle DB in namespace: {0} ", dbNamespace); + createBaseRepoSecret(dbNamespace); + dbUrl = assertDoesNotThrow(() -> createOracleDBUsingOperator(dbName, RCUSYSPASSWORD, dbNamespace)); + /* logger.info("Start DB in namespace: {0}, dbListenerPort: {1}, dbUrl: {2}, dbImage: {3}", dbNamespace, dbListenerPort, dbUrl, DB_IMAGE_TO_USE_IN_SPEC); assertDoesNotThrow(() -> startOracleDB(DB_IMAGE_TO_USE_IN_SPEC, getNextFreePort(), dbNamespace, dbListenerPort), String.format("Failed to start Oracle DB in the namespace %s with dbUrl %s, dbListenerPost %s", dbNamespace, dbUrl, dbListenerPort)); - + */ // install operator and verify its running in ready state HelmParams opHelmParams = new HelmParams().releaseName(OPERATOR_RELEASE_NAME) diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDynamicDomainInPV.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDynamicDomainInPV.java index 492fdac3b11..f0cb972f631 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDynamicDomainInPV.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDynamicDomainInPV.java @@ -9,7 +9,9 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Properties; import io.kubernetes.client.openapi.models.V1Container; @@ -25,6 +27,7 @@ import oracle.weblogic.domain.DomainResource; import oracle.weblogic.domain.DomainSpec; import oracle.weblogic.domain.ServerPod; +import oracle.weblogic.kubernetes.actions.impl.NginxParams; import oracle.weblogic.kubernetes.annotations.IntegrationTest; import oracle.weblogic.kubernetes.annotations.Namespaces; import oracle.weblogic.kubernetes.logging.LoggingFacade; @@ -40,11 +43,13 @@ import static oracle.weblogic.kubernetes.TestConstants.FMWINFRA_IMAGE_TO_USE_IN_SPEC; import static oracle.weblogic.kubernetes.TestConstants.IMAGE_PULL_POLICY; import static oracle.weblogic.kubernetes.TestConstants.K8S_NODEPORT_HOST; +import static oracle.weblogic.kubernetes.TestConstants.OKE_CLUSTER_PRIVATEIP; import static oracle.weblogic.kubernetes.TestConstants.RESULTS_TEMPFILE; import static oracle.weblogic.kubernetes.actions.ActionConstants.RESOURCE_DIR; import static oracle.weblogic.kubernetes.actions.impl.primitive.Image.getImageEnvVar; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createIngressHostRouting; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.getNextFreePort; +import static oracle.weblogic.kubernetes.utils.CommonTestUtils.getServiceExtIPAddrtOke; import static oracle.weblogic.kubernetes.utils.CommonTestUtils.getUniqueName; import static oracle.weblogic.kubernetes.utils.ConfigMapUtils.createConfigMapForDomainCreation; import static oracle.weblogic.kubernetes.utils.DbUtils.createOracleDBUsingOperator; @@ -55,6 +60,8 @@ import static oracle.weblogic.kubernetes.utils.FmwUtils.verifyEMconsoleAccess; import static oracle.weblogic.kubernetes.utils.ImageUtils.createBaseRepoSecret; import static oracle.weblogic.kubernetes.utils.JobUtils.createDomainJob; +import static oracle.weblogic.kubernetes.utils.LoadBalancerUtils.createIngressForDomainAndVerify; +import static oracle.weblogic.kubernetes.utils.LoadBalancerUtils.installAndVerifyNginx; import static oracle.weblogic.kubernetes.utils.OKDUtils.createRouteForOKD; import static oracle.weblogic.kubernetes.utils.OperatorUtils.installAndVerifyOperator; import static oracle.weblogic.kubernetes.utils.PersistentVolumeUtils.createPV; @@ -71,7 +78,7 @@ */ @DisplayName("Test to creat a FMW dynamic domain in persistent volume using WLST") @IntegrationTest -@Tag("oke-sequential") +@Tag("oke-gate") @Tag("kind-sequential") @Tag("okd-fmw-cert") class ItFmwDynamicDomainInPV { @@ -107,6 +114,13 @@ class ItFmwDynamicDomainInPV { private static String hostHeader; private static int adminPort = 7001; private static String dbName = domainUid + "my-oracle-db"; + private static String nginxNamespace = null; + private static List ingressHostList = null; + private static String ingressIP = null; + private static NginxParams nginxHelmParams = null; + + + /** * Assigns unique namespaces for DB, operator and domains. @@ -114,7 +128,7 @@ class ItFmwDynamicDomainInPV { * Pull FMW image and Oracle DB image if running tests in Kind cluster. */ @BeforeAll - public static void initAll(@Namespaces(3) List namespaces) { + public static void initAll(@Namespaces(4) List namespaces) { logger = getLogger(); logger.info("Assign a unique namespace for DB and RCU"); @@ -129,6 +143,11 @@ public static void initAll(@Namespaces(3) List namespaces) { assertNotNull(namespaces.get(2), "Namespace is null"); domainNamespace = namespaces.get(2); + // get a unique Nginx namespace + logger.info("Assign a unique namespace for Nginx"); + assertNotNull(namespaces.get(3), "Namespace list is null"); + nginxNamespace = namespaces.get(3); + logger.info("Create Oracle DB in namespace: {0} ", dbNamespace); createBaseRepoSecret(dbNamespace); dbUrl = assertDoesNotThrow(() -> createOracleDBUsingOperator(dbName, RCUSYSPASSWORD, dbNamespace)); @@ -141,6 +160,13 @@ public static void initAll(@Namespaces(3) List namespaces) { // install operator and verify its running in ready state installAndVerifyOperator(opNamespace, domainNamespace); + if (OKE_CLUSTER_PRIVATEIP) { + // install Nginx ingress controller for all test cases using Nginx + nginxHelmParams = installAndVerifyNginx(nginxNamespace, 0, 0); + String nginxServiceName = nginxHelmParams.getHelmParams().getReleaseName() + "-ingress-nginx-controller"; + ingressIP = getServiceExtIPAddrtOke(nginxServiceName, nginxNamespace); + } + } /** @@ -156,10 +182,21 @@ void testFmwDynamicDomainInPV() { verifyDomainReady(domainNamespace, domainUid, replicaCount, "nosuffix"); // Expose the admin service external node port as a route for OKD adminSvcExtHost = createRouteForOKD(getExternalServicePodName(adminServerPodName), domainNamespace); + if (OKE_CLUSTER_PRIVATEIP) { + Map clusterNameMsPortMap = new HashMap<>(); + clusterNameMsPortMap.put(clusterName, managedServerPort); + ingressHostList + = createIngressForDomainAndVerify(domainUid, domainNamespace, 0, clusterNameMsPortMap, + false, nginxHelmParams.getIngressClassName(), true, 7001); + } + if (TestConstants.KIND_CLUSTER && !TestConstants.WLSIMG_BUILDER.equals(TestConstants.WLSIMG_BUILDER_DEFAULT)) { hostHeader = createIngressHostRouting(domainNamespace, domainUid, adminServerName, adminPort); verifyEMconsoleAccess(domainNamespace, domainUid, adminSvcExtHost, hostHeader); + } else if (OKE_CLUSTER_PRIVATEIP) { + hostHeader = " -H 'host: " + ingressHostList.get(0) + "' "; + verifyEMconsoleAccess(domainNamespace, domainUid, ingressIP, hostHeader); } else { verifyEMconsoleAccess(domainNamespace, domainUid, adminSvcExtHost); } diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonMiiTestUtils.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonMiiTestUtils.java index 3a846053a2a..0455d203c53 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonMiiTestUtils.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonMiiTestUtils.java @@ -1065,10 +1065,20 @@ public static boolean checkApplicationRuntime( expectedStatusCode); } - private static String readRuntimeResource(String adminSvcExtHost, String domainNamespace, - String adminServerPodName, String resourcePath, String callerName) { + /** + * Use REST APIs to check the application runtime mbean from the WebLogic server. + * + * @param adminSvcExtHost Used only in OKD env - this is the route host created for AS external service + * @param domainNamespace Kubernetes namespace that the domain is hosted + * @param adminServerPodName Name of the admin server pod to which the REST requests should be sent to + * @param resourcePath resource path + * @param callerName caller name + * @return the expected response + */ + public static String readRuntimeResource(String adminSvcExtHost, String domainNamespace, + String adminServerPodName, String resourcePath, String callerName) { LoggingFacade logger = getLogger(); - String returnString = null; + String returnString = ""; if (TestConstants.KIND_CLUSTER && !TestConstants.WLSIMG_BUILDER.equals(TestConstants.WLSIMG_BUILDER_DEFAULT)) { @@ -1095,10 +1105,12 @@ private static String readRuntimeResource(String adminSvcExtHost, String domainN HttpResponse response; try { response = OracleHttpClient.get(url, headers, true); - assertEquals(200, response.statusCode()); returnString = response.body(); + assertEquals(200, response.statusCode()); } catch (Exception ex) { ex.printStackTrace(); + } catch (AssertionError e) { + e.printStackTrace(); } } else { String curlString; @@ -1116,10 +1128,7 @@ private static String readRuntimeResource(String adminSvcExtHost, String domainN } else { int adminServiceNodePort = getServiceNodePort(domainNamespace, getExternalServicePodName(adminServerPodName), "default"); - String host = K8S_NODEPORT_HOST; - if (host.contains(":")) { - host = "[" + host + "]"; - } + String host = formatIPv6Host(K8S_NODEPORT_HOST); String hostAndPort = (OKD) ? adminSvcExtHost : host + ":" + adminServiceNodePort; logger.info("hostAndPort = {0} ", hostAndPort); @@ -1128,9 +1137,9 @@ private static String readRuntimeResource(String adminSvcExtHost, String domainN + ":" + ADMIN_PASSWORD_DEFAULT + " http://%s%s/ --silent --show-error ", hostAndPort, resourcePath); - } - logger.info(callerName + ": curl command {0}", curlString); + logger.info(callerName + ": curl command {0}", curlString); + } try { String result = exec(curlString, true).stdout(); logger.info(callerName + ": exec curl command {0} got: {1}", curlString, result); @@ -1139,7 +1148,6 @@ private static String readRuntimeResource(String adminSvcExtHost, String domainN logger.info(callerName + ": caught unexpected exception {0}", ex); } } - return returnString; } @@ -1237,8 +1245,7 @@ public static boolean checkWeblogicMBean(String adminSvcExtHost, curlString = new StringBuffer("status=$(curl -g --user weblogic:welcome1 http://"); } - String host = K8S_NODEPORT_HOST; - formatIPv6Host(host); + String host = formatIPv6Host(K8S_NODEPORT_HOST); String hostAndPort = (OKD) ? adminSvcExtHost : host + ":" + adminServiceNodePort; logger.info("hostAndPort = {0} ", hostAndPort); diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java index fa30c49b4a9..1c49d9c1003 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java @@ -69,6 +69,7 @@ import static oracle.weblogic.kubernetes.TestConstants.NO_PROXY; import static oracle.weblogic.kubernetes.TestConstants.OKD; import static oracle.weblogic.kubernetes.TestConstants.OKE_CLUSTER; +import static oracle.weblogic.kubernetes.TestConstants.OKE_CLUSTER_PRIVATEIP; import static oracle.weblogic.kubernetes.TestConstants.RESULTS_ROOT; import static oracle.weblogic.kubernetes.TestConstants.RESULTS_TEMPFILE; import static oracle.weblogic.kubernetes.TestConstants.TRAEFIK_INGRESS_HTTP_HOSTPORT; @@ -1165,10 +1166,7 @@ public static void verifyServerCommunication(String curlRequest, List ma logger.severe(ex.getMessage()); } - String response = ""; - if (result.stdout() != null) { - response = result.stdout().trim(); - } + String response = result != null ? result.stdout().trim() : "result is null"; logger.info(response); for (var managedServer : managedServers.entrySet()) { boolean connectToOthers = true; @@ -1340,6 +1338,9 @@ public static String getHostAndPort(String hostName, int servicePort) { host = formatIPv6Host(host); String hostAndPort = ((OKD) ? hostName : host + ":" + servicePort); logger.info("hostAndPort = {0} ", hostAndPort); + if (OKE_CLUSTER_PRIVATEIP) { + hostAndPort = hostName; + } return hostAndPort; } catch (UnknownHostException e) { throw new RuntimeException(e);