diff --git a/common/src/main/java/io/brokerqe/claire/Constants.java b/common/src/main/java/io/brokerqe/claire/Constants.java index 4c9f150c..0e301663 100644 --- a/common/src/main/java/io/brokerqe/claire/Constants.java +++ b/common/src/main/java/io/brokerqe/claire/Constants.java @@ -21,6 +21,7 @@ public interface Constants { String TAG_TLS = "tls"; // Environment Variables + String EV_KUBE_CONTEXT = "KUBE_CONTEXT"; String EV_ARTEMIS_VERSION = "ARTEMIS_VERSION"; String EV_ARTEMIS_TEST_VERSION = "ARTEMIS_TEST_VERSION"; String EV_DISABLE_RANDOM_NAMESPACES = "DISABLE_RANDOM_NAMESPACES"; diff --git a/common/src/main/java/io/brokerqe/claire/TestDataCollector.java b/common/src/main/java/io/brokerqe/claire/TestDataCollector.java index aedfba38..1d768c16 100644 --- a/common/src/main/java/io/brokerqe/claire/TestDataCollector.java +++ b/common/src/main/java/io/brokerqe/claire/TestDataCollector.java @@ -25,7 +25,6 @@ public abstract class TestDataCollector implements TestWatcher, TestExecutionExc @Override public void handleTestExecutionException(ExtensionContext extensionContext, Throwable throwable) throws Throwable { testClass = extensionContext.getRequiredTestClass().getName(); -// .replaceAll(Constants.CLAIRE_TEST_PKG_REGEX, ""); testMethod = extensionContext.getRequiredTestMethod().getName(); testInstance = extensionContext.getRequiredTestInstance(); environment = Environment.get(); @@ -36,24 +35,31 @@ public void handleTestExecutionException(ExtensionContext extensionContext, Thro String classDir = TestUtils.getClassName(extensionContext); String testDir = TestUtils.getTestName(extensionContext); - archiveDir = environment.getLogsDirLocation() + Constants.FILE_SEPARATOR + testDir; + archiveDir = environment.getLogsDirLocation() + Constants.FILE_SEPARATOR + testDir.replaceFirst("io.brokerqe.claire.", ""); String certificatesDir = Environment.get().getCertificatesLocation() + Constants.FILE_SEPARATOR + testDir; String certificatesDirClass = Environment.get().getCertificatesLocation() + Constants.FILE_SEPARATOR + classDir; TestUtils.createDirectory(archiveDir); String certificatesArchiveDirectory = archiveDir + Constants.FILE_SEPARATOR + "certificates"; + String certificatesArchiveDirectoryClass = archiveDir + Constants.FILE_SEPARATOR + "class_certificates"; if (!TestUtils.isEmptyDirectory(certificatesDirClass) || !TestUtils.isEmptyDirectory(certificatesDir)) { if (TestUtils.directoryExists(certificatesArchiveDirectory)) { LOGGER.warn("[TDC] Skipping duplicated copying of certificates into {}", certificatesArchiveDirectory); } else { TestUtils.copyDirectoryFlat(certificatesDir, certificatesArchiveDirectory); - TestUtils.copyDirectoryFlat(certificatesDirClass, certificatesArchiveDirectory); + TestUtils.copyDirectoryFlat(certificatesDirClass, certificatesArchiveDirectoryClass); } } collectTestData(); throw throwable; } + /** + * Method is currently not used, but might be useful in future (getting variable from class by name) + * @param testInstance + * @param fieldName + * @return + */ public Object getTestInstanceDeclaredField(Object testInstance, String fieldName) { Field field = null; Class clazz = testInstance.getClass(); diff --git a/common/src/main/java/io/brokerqe/claire/junit/TestSeparator.java b/common/src/main/java/io/brokerqe/claire/junit/TestSeparator.java index 6ba3dfe9..41451526 100644 --- a/common/src/main/java/io/brokerqe/claire/junit/TestSeparator.java +++ b/common/src/main/java/io/brokerqe/claire/junit/TestSeparator.java @@ -50,5 +50,6 @@ static void afterAllTests(ExtensionContext testContext) { LOGGER.info((char) 27 + "[33m" + String.format("Finished Class: %s", testContext.getRequiredTestClass().getName()) + (char) 27 + "[0m"); LOGGER.info((char) 27 + "[34m" + String.join("", Collections.nCopies(76, SEPARATOR_CHAR)) + (char) 27 + "[0m"); TestUtils.deleteEmptyDirectories(Environment.get().getCertificatesLocation()); + TestUtils.deleteEmptyDirectories(Environment.get().getTmpDirLocation()); } } diff --git a/common/src/main/java/io/brokerqe/claire/security/CertificateManager.java b/common/src/main/java/io/brokerqe/claire/security/CertificateManager.java index 170d2ead..c8fec265 100644 --- a/common/src/main/java/io/brokerqe/claire/security/CertificateManager.java +++ b/common/src/main/java/io/brokerqe/claire/security/CertificateManager.java @@ -272,8 +272,8 @@ public static X500PrivateCredential createPrivateCredential(X509Certificate cert */ public static Map createEntityKeystores(CertificateData certificateData, String keystorePassword) { Map keystores = new HashMap<>(); - String keyStoreFileName = currentTestDirectory + certificateData.getAlias() + "_keystore.jks"; - String trustStoreFileName = currentTestDirectory + certificateData.getAlias() + "_truststore.jks"; + String keyStoreFileName = getCurrentTestDirectory() + certificateData.getAlias() + "_keystore.jks"; + String trustStoreFileName = getCurrentTestDirectory() + certificateData.getAlias() + "_truststore.jks"; String keyStoreDataName = certificateData.getAlias() + ".ks"; String trustStoreDataName = certificateData.getAlias() + ".ts"; @@ -305,10 +305,10 @@ public static Map createEntityKeystores(CertificateData ce public static Map createKeystores(CertificateData brokerCertificateData, CertificateData clientCertificateData, String brokerAlias, String brokerPassword, String clientAlias, String clientPassword) { Map keystores = new HashMap<>(); - String brokerKeyStoreFileName = currentTestDirectory + brokerAlias + "_keystore.jks"; - String brokerTrustStoreFileName = currentTestDirectory + brokerAlias + "_truststore.jks"; - String clientKeyStoreFileName = currentTestDirectory + clientAlias + "_keystore.jks"; - String clientTrustStoreFileName = currentTestDirectory + clientAlias + "_truststore.jks"; + String brokerKeyStoreFileName = getCurrentTestDirectory() + brokerAlias + "_keystore.jks"; + String brokerTrustStoreFileName = getCurrentTestDirectory() + brokerAlias + "_truststore.jks"; + String clientKeyStoreFileName = getCurrentTestDirectory() + clientAlias + "_keystore.jks"; + String clientTrustStoreFileName = getCurrentTestDirectory() + clientAlias + "_truststore.jks"; try { LOGGER.info("[TLS] Creating Broker keystore"); @@ -385,17 +385,20 @@ public static void writeCertificateToFile(X509Certificate certificate, String fi } public static Map generateDefaultCertificateKeystores(String brokerDN, String clientDN, List extensions, CertificateData issuer) { + return generateCertificateKeystores(brokerDN, DEFAULT_BROKER_ALIAS, clientDN, DEFAULT_CLIENT_ALIAS, extensions, issuer); + } + + public static Map generateCertificateKeystores(String brokerDN, String brokerAlias, String clientDN, String clientAlias, List extensions, CertificateData issuer) { LOGGER.info("[TLS] Generating Broker KeyPair, Certificates"); - CertificateData brokerCertData = new CertificateData(DEFAULT_BROKER_ALIAS, brokerDN, extensions, 30, issuer); - writeCertificateToFile(brokerCertData.getCertificate(), currentTestDirectory + DEFAULT_BROKER_ALIAS + ".crt"); + CertificateData brokerCertData = new CertificateData(brokerAlias, brokerDN, extensions, 30, issuer); + writeCertificateToFile(brokerCertData.getCertificate(), getCurrentTestDirectory() + brokerAlias + ".crt"); // Client cert + keypair LOGGER.info("[TLS] Generating Client KeyPair, Certificates"); - CertificateData clientCertData = new CertificateData(DEFAULT_CLIENT_ALIAS, clientDN); - writeCertificateToFile(clientCertData.getCertificate(), currentTestDirectory + DEFAULT_CLIENT_ALIAS + ".crt"); + CertificateData clientCertData = new CertificateData(clientAlias, clientDN); + writeCertificateToFile(clientCertData.getCertificate(), getCurrentTestDirectory() + clientAlias + ".crt"); - return CertificateManager.createKeystores(brokerCertData, clientCertData, - DEFAULT_BROKER_ALIAS, DEFAULT_BROKER_PASSWORD, DEFAULT_CLIENT_ALIAS, DEFAULT_CLIENT_PASSWORD); + return CertificateManager.createKeystores(brokerCertData, clientCertData, brokerAlias, DEFAULT_BROKER_PASSWORD, clientAlias, DEFAULT_CLIENT_PASSWORD); } public static String readCertificateFromFile(String certificatePath) { @@ -485,11 +488,11 @@ public static void addToTruststore(KeyStoreData keyStoreData, String stringTlsCe */ public static Map reuseDefaultGeneratedKeystoresFromFiles() { Map keystores = new HashMap<>(); - if (TestUtils.directoryExists(currentTestDirectory)) { - String brokerKeyStoreFileName = currentTestDirectory + DEFAULT_BROKER_ALIAS + "_keystore.jks"; - String brokerTrustStoreFileName = currentTestDirectory + DEFAULT_BROKER_ALIAS + "_truststore.jks"; - String clientKeyStoreFileName = currentTestDirectory + DEFAULT_CLIENT_ALIAS + "_keystore.jks"; - String clientTrustStoreFileName = currentTestDirectory + DEFAULT_CLIENT_ALIAS + "_truststore.jks"; + if (TestUtils.directoryExists(getCurrentTestDirectory())) { + String brokerKeyStoreFileName = getCurrentTestDirectory() + DEFAULT_BROKER_ALIAS + "_keystore.jks"; + String brokerTrustStoreFileName = getCurrentTestDirectory() + DEFAULT_BROKER_ALIAS + "_truststore.jks"; + String clientKeyStoreFileName = getCurrentTestDirectory() + DEFAULT_CLIENT_ALIAS + "_keystore.jks"; + String clientTrustStoreFileName = getCurrentTestDirectory() + DEFAULT_CLIENT_ALIAS + "_truststore.jks"; try { Security.addProvider(new BouncyCastleProvider()); @@ -514,7 +517,7 @@ public static Map reuseDefaultGeneratedKeystoresFromFiles( throw new RuntimeException(e); } } else { - LOGGER.error("[TLS] {} does not exist! Can not reuse it!", currentTestDirectory); + LOGGER.error("[TLS] {} does not exist! Can not reuse it!", getCurrentTestDirectory()); throw new RuntimeException("Can not find expected directory and load certificates!"); } } diff --git a/operator-suite/README.md b/operator-suite/README.md index 898d05e3..48c4aa58 100644 --- a/operator-suite/README.md +++ b/operator-suite/README.md @@ -34,29 +34,30 @@ If you would want to try your own OLM operator, you would need to specify only f ## List of available Environment Variables -| Name | Description | Default | Possible values | -|---------------------------|---------------------------------------------------------------------|-----------------------------|--------------------------------------------------| -| ARTEMIS_VERSION | ArtemisCloud Version to be used (Makefile) | 7.10.2 | \.\.\ | -| ARTEMIS_TEST_VERSION | ArtemisCloud Version to be used by tests | not set | \.\ | -| OPERATOR_IMAGE | ArtemisCloud Operator image url | not set | \ | -| BROKER_IMAGE | Broker image url | not set | \ | -| BROKER_INIT_IMAGE | Broker init image url | not set | \ | -| BUNDLE_IMAGE | Bundle image url | not set | \ | -| OLM | Whether to install latest available Operator | false | `true`, `false` | -| OLM_LTS | Whether to install lts or latest available operator (tied to `OLM`) | false | `true`, `false` | -| OLM_IIB | OLM Index Image Bundle to use | not set | \ | -| OLM_CHANNEL | OLM channel to use with Subscription | not set | \ | -| DISABLE_RANDOM_NAMESPACES | Whether to use random string suffices | not set (`false`) | `true`, `false` | -| LOGS_LOCATION | Location where to generate collected logs | `test-logs` | \ | +| Name | Description | Default | Possible values | +|---------------------------|---------------------------------------------------------------------|-----------------------------|-------------------------------------------------| +| ARTEMIS_VERSION | ArtemisCloud Version to be used (Makefile) | 7.10.2 | \.\.\ | +| ARTEMIS_TEST_VERSION | ArtemisCloud Version to be used by tests | not set | \.\ | +| OPERATOR_IMAGE | ArtemisCloud Operator image url | not set | \ | +| BROKER_IMAGE | Broker image url | not set | \ | +| BROKER_INIT_IMAGE | Broker init image url | not set | \ | +| BUNDLE_IMAGE | Bundle image url | not set | \ | +| OLM | Whether to install latest available Operator | false | `true`, `false` | +| OLM_LTS | Whether to install lts or latest available operator (tied to `OLM`) | false | `true`, `false` | +| OLM_IIB | OLM Index Image Bundle to use | not set | \ | +| OLM_CHANNEL | OLM channel to use with Subscription | not set | \ | +| DISABLE_RANDOM_NAMESPACES | Whether to use random string suffices | not set (`false`) | `true`, `false` | +| LOGS_LOCATION | Location where to generate collected logs | `test-logs` | \ | | TEST_LOG_LEVEL | Set logging level of test suite | `INFO` set in `logback.xml` | `TRACE`, `DEBUG`, `INFO`, `WARN`, `ERROR`, `OFF` | -| CLUSTER_OPERATOR_MANAGED | Whether test suite manages CO or not (Makefile) | `true` | `false` | -| COLLECT_TEST_DATA | Whether to gather test data on error or not | `true` | `true`, `false` | -| CUSTOM_EXTRA_DELAY | Prolonged all internal waitFor calls (seconds) | `0` | \ | -| OPERATOR_INSTALL_ZIP | Url to zip file with install/examples (Makefile) | 7.10.2 url | \ | -| OPERATOR_VERSION_UPSTREAM | Version/branch of repository (Makefile) | main | \ | -| DUMP_ENABLED | Enabled serialization of deployed resources | `false` | `true`, `false` | -| DUMP_FORMAT | Format of serialized deployed resources | `yaml` | `yaml`, `json` | -| DUMP_LOCATION | Location to dump serialized deployed resources | `serialization-dump` | \ | +| CLUSTER_OPERATOR_MANAGED | Whether test suite manages CO or not (Makefile) | `true` | `false` | +| COLLECT_TEST_DATA | Whether to gather test data on error or not | `true` | `true`, `false` | +| CUSTOM_EXTRA_DELAY | Prolonged all internal waitFor calls (seconds) | `0` | \ | +| OPERATOR_INSTALL_ZIP | Url to zip file with install/examples (Makefile) | 7.10.2 url | \ | +| OPERATOR_VERSION_UPSTREAM | Version/branch of repository (Makefile) | main | \ | +| DUMP_ENABLED | Enabled serialization of deployed resources | `false` | `true`, `false` | +| DUMP_FORMAT | Format of serialized deployed resources | `yaml` | `yaml`, `json` | +| DUMP_LOCATION | Location to dump serialized deployed resources | `serialization-dump` | \ | +| KUBE_CONTEXT | Provide comma separated context(s) for kubernetes client | `default/null` | null, \ | ## Setting log level Currently, there is supported `TEST_LOG_LEVEL` environment variable, which can set desired logging level of test suite. diff --git a/operator-suite/src/main/java/io/brokerqe/claire/EnvironmentOperator.java b/operator-suite/src/main/java/io/brokerqe/claire/EnvironmentOperator.java index ee8e84a5..bf868574 100644 --- a/operator-suite/src/main/java/io/brokerqe/claire/EnvironmentOperator.java +++ b/operator-suite/src/main/java/io/brokerqe/claire/EnvironmentOperator.java @@ -16,6 +16,9 @@ import java.io.FileInputStream; import java.io.IOException; import java.nio.file.Path; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; import java.util.Properties; public class EnvironmentOperator extends Environment { @@ -29,6 +32,7 @@ public class EnvironmentOperator extends Environment { private final boolean olmInstallation; private final String artemisOperatorName; private final String artemisOperatorType; + private List kubeContexts; private String artemisVersion; private final ArtemisVersion artemisTestVersion; private final String brokerImage; @@ -41,7 +45,7 @@ public class EnvironmentOperator extends Environment { private final String tmpDirLocation; private final String keycloakVersion; static final Logger LOGGER = LoggerFactory.getLogger(Environment.class); - private final KubeClient kubeClient; + private Map kubeClients; private final boolean collectTestData; private final int customExtraDelay; private final boolean serializationEnabled; @@ -51,6 +55,7 @@ public class EnvironmentOperator extends Environment { public EnvironmentOperator() { this.set(this); String initialTimestamp = TestUtils.generateTimestamp(); + initializeKubeContexts(System.getenv().getOrDefault(Constants.EV_KUBE_CONTEXT, null)); artemisVersion = System.getenv(Constants.EV_ARTEMIS_VERSION); testLogLevel = System.getenv(Constants.EV_TEST_LOG_LEVEL); logsDirLocation = System.getenv().getOrDefault(Constants.EV_LOGS_LOCATION, Constants.LOGS_DEFAULT_DIR) + Constants.FILE_SEPARATOR + initialTimestamp; @@ -60,7 +65,6 @@ public EnvironmentOperator() { serializationDirectory = System.getenv().getOrDefault(Constants.EV_DUMP_LOCATION, Constants.DUMP_DEFAULT_DIR) + Constants.FILE_SEPARATOR + initialTimestamp; serializationFormat = System.getenv().getOrDefault(Constants.EV_DUMP_FORMAT, Constants.DUMP_DEFAULT_TYPE); - kubeClient = new KubeClient("default"); disabledRandomNs = Boolean.parseBoolean(System.getenv(Constants.EV_DISABLE_RANDOM_NAMESPACES)); customExtraDelay = Integer.parseInt(System.getenv().getOrDefault(Constants.EV_CUSTOM_EXTRA_DELAY, "0")); @@ -102,6 +106,7 @@ public EnvironmentOperator() { @SuppressWarnings("checkstyle:NPathComplexity") private void printAllUsedTestVariables() { StringBuilder envVarsSB = new StringBuilder("List of all used Claire related variables:").append(Constants.LINE_SEPARATOR); + envVarsSB.append(Constants.EV_KUBE_CONTEXT).append("=").append(String.join(" ", kubeContexts)).append(Constants.LINE_SEPARATOR); envVarsSB.append(Constants.EV_DISABLE_RANDOM_NAMESPACES).append("=").append(disabledRandomNs).append(Constants.LINE_SEPARATOR); envVarsSB.append(Constants.EV_CLUSTER_OPERATOR_MANAGED).append("=").append(projectManagedClusterOperator).append(Constants.LINE_SEPARATOR); envVarsSB.append(Constants.EV_COLLECT_TEST_DATA).append("=").append(collectTestData).append(Constants.LINE_SEPARATOR); @@ -212,6 +217,39 @@ public Database getDatabase() { return null; } + private void initializeKubeContexts(String contexts) { + kubeClients = new LinkedHashMap<>(1); + if (contexts == null) { + // return default + kubeContexts = List.of("default"); + kubeClients.put("default", new KubeClient(null)); + return; + } + if (!contexts.contains(",")) { + // single context + kubeContexts = List.of(contexts); + } else { + // multiple contexts + kubeContexts = List.of(contexts.split(",")); + } + kubeClients = new LinkedHashMap<>(kubeContexts.size()); + for (String context : kubeContexts) { + kubeClients.put(context, new KubeClient(context)); + } + } + + public KubeClient getDefaultKubeClient() { + return kubeClients.get(kubeContexts.get(0)); + } + + public KubeClient getKubeClient(String context) { + return kubeClients.get(context); + } + + public Map getKubeClients() { + return kubeClients; + } + public void checkSetProvidedImages() { Path operatorFile = ArtemisFileProvider.getOperatorInstallFile(); @@ -274,10 +312,6 @@ private String getDefaultKeycloakVersion() { } } - public KubeClient getKubeClient() { - return kubeClient; - } - public String getOlmIndexImageBundle() { return olmIndexImageBundle; } diff --git a/operator-suite/src/main/java/io/brokerqe/claire/KubeClient.java b/operator-suite/src/main/java/io/brokerqe/claire/KubeClient.java index f7bb66c3..74331e0f 100644 --- a/operator-suite/src/main/java/io/brokerqe/claire/KubeClient.java +++ b/operator-suite/src/main/java/io/brokerqe/claire/KubeClient.java @@ -41,6 +41,7 @@ import io.fabric8.openshift.client.OpenShiftClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.slf4j.MDC; import java.io.ByteArrayInputStream; import java.nio.file.Path; @@ -62,19 +63,31 @@ import static org.hamcrest.Matchers.is; public class KubeClient { - protected final KubernetesClient client; + private final KubernetesClient client; private final KubernetesPlatform platform; protected String namespace; + private final String kubeContext; private static final Logger LOGGER = LoggerFactory.getLogger(KubeClient.class); // ============================ // ---------> CLIENT <--------- // ============================ + public KubeClient(String context) { + this(context, "default"); + } - public KubeClient(String namespace) { - LOGGER.debug("Creating client in namespace: {}", namespace); - Config config = Config.autoConfigure(System.getenv().getOrDefault("KUBE_CONTEXT", null)); + public KubeClient(String context, String namespace) { + Config config = Config.autoConfigure(context); + String contextMessage = "Using context {}"; + if (context == null) { + contextMessage = "Using context null/default - as none was provided"; + kubeContext = "default"; + } else { + kubeContext = context; + } + LOGGER.debug(contextMessage, context); + LOGGER.debug("[{}] Creating client in namespace: {}", context, namespace); config.setConnectionTimeout(60000); // default 10000ms config.setRequestTimeout(30000); // default 10000ms KubernetesClient tmpClient = new KubernetesClientBuilder().withConfig(config).build().adapt(OpenShiftClient.class); @@ -95,6 +108,7 @@ public KubeClient(String namespace) { } public KubernetesClient getKubernetesClient() { + MDC.put("kubecontext", kubeContext); return client; } diff --git a/operator-suite/src/main/java/io/brokerqe/claire/OperatorTestDataCollector.java b/operator-suite/src/main/java/io/brokerqe/claire/OperatorTestDataCollector.java index 01f4f7d6..a7517d1d 100644 --- a/operator-suite/src/main/java/io/brokerqe/claire/OperatorTestDataCollector.java +++ b/operator-suite/src/main/java/io/brokerqe/claire/OperatorTestDataCollector.java @@ -47,42 +47,55 @@ private ArtemisCloudClusterOperator getOperatorDifferentNamespace() { @Override protected void collectTestData() { List testNamespaces = getTestNamespaces(); - LOGGER.info("Error detected will gather! data from namespace: {}", String.join(" ", testNamespaces)); - kubeClient = (KubeClient) getTestInstanceDeclaredField(testInstance, "client"); - for (String testNamespace : testNamespaces) { - String archiveDirTmp = archiveDir + Constants.FILE_SEPARATOR + testNamespace; - TestUtils.createDirectory(archiveDirTmp); - LOGGER.debug("[{}] Gathering debug data for failed {}#{} into {}", testNamespace, testClass, testMethod, archiveDirTmp); - - List deployments = kubeClient.getKubernetesClient().apps().deployments().inNamespace(testNamespace).list().getItems(); - List statefulSets = kubeClient.getKubernetesClient().apps().statefulSets().inNamespace(testNamespace).list().getItems(); - List replicaSets = kubeClient.getKubernetesClient().apps().replicaSets().inNamespace(testNamespace).list().getItems(); - List configMaps = kubeClient.getKubernetesClient().configMaps().inNamespace(testNamespace).list().getItems(); - List persistentVolumeClaims = kubeClient.getKubernetesClient().persistentVolumeClaims().inNamespace(testNamespace).list().getItems(); - List persistentVolumes = kubeClient.getKubernetesClient().persistentVolumes().list().getItems(); - List services = kubeClient.getKubernetesClient().services().inNamespace(testNamespace).list().getItems(); - List secrets = kubeClient.getKubernetesClient().secrets().inNamespace(testNamespace).list().getItems(); - List events = kubeClient.getKubernetesClient().v1().events().inNamespace(testNamespace).list().getItems(); - List pods = kubeClient.getKubernetesClient().pods().inNamespace(testNamespace).list().getItems(); - List artemises = ResourceManager.getArtemisClient().inNamespace(testNamespace).list().getItems(); - List artemisAddresses = ResourceManager.getArtemisAddressClient().inNamespace(testNamespace).list().getItems(); - List artemisSecurities = ResourceManager.getArtemisSecurityClient().inNamespace(testNamespace).list().getItems(); - - writeHasMetadataObject(deployments, archiveDirTmp); - writeHasMetadataObject(statefulSets, archiveDirTmp); - writeHasMetadataObject(replicaSets, archiveDirTmp); - writeHasMetadataObject(configMaps, archiveDirTmp); - writeHasMetadataObject(persistentVolumeClaims, archiveDirTmp); - writeHasMetadataObject(persistentVolumes, archiveDirTmp); - writeHasMetadataObject(services, archiveDirTmp); - writeHasMetadataObject(secrets, archiveDirTmp); - writeHasMetadataObject(pods, archiveDirTmp); - writeHasMetadataObject(artemises, archiveDirTmp); - writeHasMetadataObject(artemisAddresses, archiveDirTmp); - writeHasMetadataObject(artemisSecurities, archiveDirTmp); - writeEvents(events, archiveDirTmp); - collectPodLogs(pods, archiveDirTmp); - collectBrokerPodFiles(pods, archiveDirTmp); + boolean useClusterDir = ResourceManager.getKubeClients().size() > 1; + for (KubeClient kubeClientTmp : ResourceManager.getKubeClients()) { + this.kubeClient = kubeClientTmp; + ResourceManager.setKubeClient(kubeClientTmp); + String archiveDirTmp = archiveDir + Constants.FILE_SEPARATOR; +// kubeClient = (KubeClient) getTestInstanceDeclaredField(testInstance, "client"); + if (useClusterDir) { + String clusterName = kubeClient.getKubernetesClient().getMasterUrl().getHost(); + archiveDirTmp += clusterName; + LOGGER.info("[{}] Error detected! Will gather data from namespace: {}", clusterName, String.join(" ", testNamespaces)); + } else { + LOGGER.info("Error detected! Will gather data from namespace: {}", String.join(" ", testNamespaces)); + } + + for (String testNamespace : testNamespaces) { + archiveDirTmp += Constants.FILE_SEPARATOR + testNamespace; + TestUtils.createDirectory(archiveDirTmp); + LOGGER.debug("[{}] Gathering debug data for failed {}#{} into {}", testNamespace, testClass, testMethod, archiveDirTmp); + + List deployments = kubeClient.getKubernetesClient().apps().deployments().inNamespace(testNamespace).list().getItems(); + List statefulSets = kubeClient.getKubernetesClient().apps().statefulSets().inNamespace(testNamespace).list().getItems(); + List replicaSets = kubeClient.getKubernetesClient().apps().replicaSets().inNamespace(testNamespace).list().getItems(); + List configMaps = kubeClient.getKubernetesClient().configMaps().inNamespace(testNamespace).list().getItems(); + List persistentVolumeClaims = kubeClient.getKubernetesClient().persistentVolumeClaims().inNamespace(testNamespace).list().getItems(); + List persistentVolumes = kubeClient.getKubernetesClient().persistentVolumes().list().getItems(); + List services = kubeClient.getKubernetesClient().services().inNamespace(testNamespace).list().getItems(); + List secrets = kubeClient.getKubernetesClient().secrets().inNamespace(testNamespace).list().getItems(); + List events = kubeClient.getKubernetesClient().v1().events().inNamespace(testNamespace).list().getItems(); + List pods = kubeClient.getKubernetesClient().pods().inNamespace(testNamespace).list().getItems(); + List artemises = ResourceManager.getArtemisClient().inNamespace(testNamespace).list().getItems(); + List artemisAddresses = ResourceManager.getArtemisAddressClient().inNamespace(testNamespace).list().getItems(); + List artemisSecurities = ResourceManager.getArtemisSecurityClient().inNamespace(testNamespace).list().getItems(); + + writeHasMetadataObject(deployments, archiveDirTmp); + writeHasMetadataObject(statefulSets, archiveDirTmp); + writeHasMetadataObject(replicaSets, archiveDirTmp); + writeHasMetadataObject(configMaps, archiveDirTmp); + writeHasMetadataObject(persistentVolumeClaims, archiveDirTmp); + writeHasMetadataObject(persistentVolumes, archiveDirTmp); + writeHasMetadataObject(services, archiveDirTmp); + writeHasMetadataObject(secrets, archiveDirTmp); + writeHasMetadataObject(pods, archiveDirTmp); + writeHasMetadataObject(artemises, archiveDirTmp); + writeHasMetadataObject(artemisAddresses, archiveDirTmp); + writeHasMetadataObject(artemisSecurities, archiveDirTmp); + writeEvents(events, archiveDirTmp); + collectPodLogs(pods, archiveDirTmp); + collectBrokerPodFiles(pods, archiveDirTmp); + } } } diff --git a/operator-suite/src/main/java/io/brokerqe/claire/ResourceManager.java b/operator-suite/src/main/java/io/brokerqe/claire/ResourceManager.java index 83af8469..e312e49e 100644 --- a/operator-suite/src/main/java/io/brokerqe/claire/ResourceManager.java +++ b/operator-suite/src/main/java/io/brokerqe/claire/ResourceManager.java @@ -64,6 +64,7 @@ import java.time.Duration; import java.time.ZonedDateTime; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -73,12 +74,6 @@ public class ResourceManager { final static Logger LOGGER = LoggerFactory.getLogger(ResourceManager.class); - - private static MixedOperation, Resource> artemisClient; - private static MixedOperation, Resource> artemisAddressClient; - private static MixedOperation, Resource> artemisSecurityClient; - private static MixedOperation, Resource> artemisScaledownClient; - private static List deployedOperators = new ArrayList<>(); private static Map deployedContainers = new HashMap<>(); private static List deployedNamespaces = new ArrayList<>(); @@ -92,11 +87,7 @@ public class ResourceManager { private static TestInfo testInfo; private ResourceManager(EnvironmentOperator env) { - kubeClient = env.getKubeClient(); - artemisClient = kubeClient.getKubernetesClient().resources(ActiveMQArtemis.class); - artemisAddressClient = kubeClient.getKubernetesClient().resources(ActiveMQArtemisAddress.class); - artemisSecurityClient = kubeClient.getKubernetesClient().resources(ActiveMQArtemisSecurity.class); - artemisScaledownClient = kubeClient.getKubernetesClient().resources(ActiveMQArtemisScaledown.class); + kubeClient = env.getDefaultKubeClient(); projectCODeploy = env.isProjectManagedClusterOperator(); environmentOperator = env; } @@ -111,20 +102,40 @@ public static KubeClient getKubeClient() { return kubeClient; } + public static KubeClient getKubeClient(String context) { + return environmentOperator.getKubeClient(context); + } + + public static KubeClient getKubeClient(int position) { + return environmentOperator.getKubeClients().values().stream().toList().get(position); + } + + public static Collection getKubeClients() { + return environmentOperator.getKubeClients().values(); + } + + public static void setKubeClient(String context) { + kubeClient = getKubeClient(context); + } + + public static void setKubeClient(KubeClient client) { + kubeClient = client; + } + public static MixedOperation, Resource> getArtemisClient() { - return artemisClient; + return getKubeClient().getKubernetesClient().resources(ActiveMQArtemis.class); } public static MixedOperation, Resource> getArtemisAddressClient() { - return artemisAddressClient; + return getKubeClient().getKubernetesClient().resources(ActiveMQArtemisAddress.class); } public static MixedOperation, Resource> getArtemisSecurityClient() { - return artemisSecurityClient; + return getKubeClient().getKubernetesClient().resources(ActiveMQArtemisSecurity.class); } public static MixedOperation, Resource> getArtemisScaledownClient() { - return artemisScaledownClient; + return getKubeClient().getKubernetesClient().resources(ActiveMQArtemisScaledown.class); } // Artemis ClusterOperator diff --git a/operator-suite/src/main/java/io/brokerqe/claire/clients/BundledClientDeployment.java b/operator-suite/src/main/java/io/brokerqe/claire/clients/BundledClientDeployment.java index 08246a30..26b658f7 100644 --- a/operator-suite/src/main/java/io/brokerqe/claire/clients/BundledClientDeployment.java +++ b/operator-suite/src/main/java/io/brokerqe/claire/clients/BundledClientDeployment.java @@ -4,7 +4,6 @@ */ package io.brokerqe.claire.clients; -import io.brokerqe.claire.KubeClient; import io.brokerqe.claire.ResourceManager; import io.brokerqe.claire.exception.ClaireNotImplementedException; import io.brokerqe.claire.executor.Executor; @@ -20,7 +19,6 @@ public class BundledClientDeployment implements KubernetesDeployableClient { private static final Logger LOGGER = LoggerFactory.getLogger(BundledClientDeployment.class); private final String namespace; - static KubeClient kubeClient = ResourceManager.getKubeClient(); private Pod pod; public BundledClientDeployment(String namespace) { @@ -36,7 +34,7 @@ public BundledClientDeployment(String namespace, Pod pod) { public Pod getContainer() { if (pod == null) { LOGGER.debug("[{}] [BundledClient] Using default first found artemis pod", namespace); - pod = kubeClient.getArtemisPodByLabel(namespace); + pod = ResourceManager.getKubeClient().getArtemisPodByLabel(namespace); } return pod; } @@ -91,7 +89,7 @@ public String getNamespace() { @Override public Deployment getDeployment() { // TODO ??! - return kubeClient.getDeployment(namespace, pod.getMetadata().getGenerateName()); + return ResourceManager.getKubeClient().getDeployment(namespace, pod.getMetadata().getGenerateName()); // throw new UnsupportedOperationException("Statefulset of broker pod is not needed for clients pod"); } } diff --git a/operator-suite/src/main/java/io/brokerqe/claire/clients/StClientDeployment.java b/operator-suite/src/main/java/io/brokerqe/claire/clients/StClientDeployment.java index 61cdc928..71bcd15a 100644 --- a/operator-suite/src/main/java/io/brokerqe/claire/clients/StClientDeployment.java +++ b/operator-suite/src/main/java/io/brokerqe/claire/clients/StClientDeployment.java @@ -4,7 +4,6 @@ */ package io.brokerqe.claire.clients; -import io.brokerqe.claire.KubeClient; import io.brokerqe.claire.ResourceManager; import io.brokerqe.claire.exception.ClaireNotImplementedException; import io.brokerqe.claire.executor.Executor; @@ -27,7 +26,6 @@ public abstract class StClientDeployment implements KubernetesDeployableClient { private static final Logger LOGGER = LoggerFactory.getLogger(StClientDeployment.class); private final String namespace; - static KubeClient kubeClient = ResourceManager.getKubeClient(); private Pod pod; private Deployment deployment; @@ -41,7 +39,7 @@ public StClientDeployment(String namespace) { @Override public Pod getContainer() { if (pod == null) { - pod = kubeClient.getFirstPodByPrefixName(namespace, getPodName()); + pod = ResourceManager.getKubeClient().getFirstPodByPrefixName(namespace, getPodName()); LOGGER.debug("[{}] [STClient] Using first found container {}", namespace, pod.getMetadata().getName()); } return pod; @@ -73,7 +71,7 @@ public Deployment deployContainer() { @Override public void undeployContainer() { - kubeClient.getKubernetesClient().apps().deployments().inNamespace(namespace).resource(deployment).delete(); + ResourceManager.getKubeClient().getKubernetesClient().apps().deployments().inNamespace(namespace).resource(deployment).delete(); } @Override @@ -105,10 +103,10 @@ public Deployment deployContainer(boolean secured, List secretNames) { } } - deployment = kubeClient.getKubernetesClient().apps().deployments().inNamespace(namespace).resource( + deployment = ResourceManager.getKubeClient().getKubernetesClient().apps().deployments().inNamespace(namespace).resource( createSystemTestsDeployment(namespace, secured, secretVolumes, secretVolumeMounts)).createOrReplace(); LOGGER.debug("[{}] Wait 30s for systemtest-clients deployment to be ready", namespace); - kubeClient.getKubernetesClient().resource(deployment).inNamespace(namespace).waitUntilReady(30, TimeUnit.SECONDS); + ResourceManager.getKubeClient().getKubernetesClient().resource(deployment).inNamespace(namespace).waitUntilReady(30, TimeUnit.SECONDS); this.deployment = deployment; return deployment; } @@ -208,7 +206,7 @@ private Deployment createSystemTestsDeployment(String namespace, boolean secured .endSpec() .build(); } - if (kubeClient.isKubernetesPlatform()) { + if (ResourceManager.getKubeClient().isKubernetesPlatform()) { // add userId systemtestClients.getSpec().getTemplate().getSpec().getSecurityContext().setRunAsUser(defaultUserId); } diff --git a/operator-suite/src/main/java/io/brokerqe/claire/junit/TestMinimumKubernetesCount.java b/operator-suite/src/main/java/io/brokerqe/claire/junit/TestMinimumKubernetesCount.java new file mode 100644 index 00000000..3e87c68c --- /dev/null +++ b/operator-suite/src/main/java/io/brokerqe/claire/junit/TestMinimumKubernetesCount.java @@ -0,0 +1,63 @@ +/* + * Copyright Broker QE authors. + * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html). + */ +package io.brokerqe.claire.junit; + +import io.brokerqe.claire.KubeClient; +import io.brokerqe.claire.ResourceManager; +import org.junit.jupiter.api.extension.ConditionEvaluationResult; +import org.junit.jupiter.api.extension.ExecutionCondition; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.platform.commons.PreconditionViolationException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import java.lang.reflect.AnnotatedElement; +import java.util.Collection; + +import static org.junit.platform.commons.util.AnnotationUtils.findAnnotation; + +/** + * This annotation executes tests only on specified platform. + * Supported values are "Openshift" and "Kubernetes". + */ + +@Target({ElementType.TYPE, ElementType.METHOD}) +@Retention(RetentionPolicy.RUNTIME) +@ExtendWith(TestMinimumKubernetesCount.MinimumKubernetesCountCondition.class) +public @interface TestMinimumKubernetesCount { + int value(); + + class MinimumKubernetesCountCondition implements ExecutionCondition { + private final static Logger LOGGER = LoggerFactory.getLogger(MinimumKubernetesCountCondition.class); + private static final ConditionEvaluationResult ENABLED = ConditionEvaluationResult.enabled("@TestMinimumKubernetesCount is not present"); + + @Override + public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext context) { + AnnotatedElement element = context.getElement().orElse(null); + String testname = context.getRequiredTestClass().getName(); + try { + testname += context.getRequiredTestMethod().getName(); + } catch (PreconditionViolationException ignored) { } + String finalTestname = testname; + ConditionEvaluationResult result = findAnnotation(element, TestMinimumKubernetesCount.class).map(annotation -> toResult(element, annotation, finalTestname)).orElse(ENABLED); + return result; + } + + private ConditionEvaluationResult toResult(AnnotatedElement element, TestMinimumKubernetesCount annotation, String testName) { + Collection kubeClients = ResourceManager.getEnvironment().getKubeClients().values(); + if (kubeClients.size() >= annotation.value()) { + return ConditionEvaluationResult.enabled("Test enabled on provided number of Kubernetes platforms"); + } else { + LOGGER.warn("[TEST][{}] Disabling test - not enough Kubernetes platforms provided. Needed {}, got {}.", testName, annotation.value(), kubeClients.size()); + return ConditionEvaluationResult.disabled("[TEST] Skipped: Unsupported platform count for this test."); + } + } + } +} diff --git a/operator-suite/src/main/java/io/brokerqe/claire/junit/TestSupportedPlatform.java b/operator-suite/src/main/java/io/brokerqe/claire/junit/TestSupportedPlatform.java index 7f01044c..094c9111 100644 --- a/operator-suite/src/main/java/io/brokerqe/claire/junit/TestSupportedPlatform.java +++ b/operator-suite/src/main/java/io/brokerqe/claire/junit/TestSupportedPlatform.java @@ -4,12 +4,14 @@ */ package io.brokerqe.claire.junit; -import io.brokerqe.claire.KubernetesPlatform; import io.brokerqe.claire.KubeClient; +import io.brokerqe.claire.KubernetesPlatform; +import io.brokerqe.claire.ResourceManager; import org.junit.jupiter.api.extension.ConditionEvaluationResult; import org.junit.jupiter.api.extension.ExecutionCondition; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.platform.commons.PreconditionViolationException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -18,6 +20,9 @@ import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; import java.lang.reflect.AnnotatedElement; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; import static org.junit.platform.commons.util.AnnotationUtils.findAnnotation; @@ -33,31 +38,45 @@ KubernetesPlatform value(); class SupportedPlatformTestCondition implements ExecutionCondition { - - KubeClient kubeClient = new KubeClient("default"); private final static Logger LOGGER = LoggerFactory.getLogger(SupportedPlatformTestCondition.class); private static final ConditionEvaluationResult ENABLED = ConditionEvaluationResult.enabled("@PlatformTest is not present"); @Override public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext context) { AnnotatedElement element = context.getElement().orElse(null); - ConditionEvaluationResult result = findAnnotation(element, TestSupportedPlatform.class).map(annotation -> toResult(element, annotation)).orElse(ENABLED); - kubeClient.getKubernetesClient().close(); + String testname = context.getRequiredTestClass().getName(); + try { + testname += context.getRequiredTestMethod().getName(); + } catch (PreconditionViolationException ignored) { } + String finalTestname = testname; + ConditionEvaluationResult result = findAnnotation(element, TestSupportedPlatform.class).map(annotation -> toResult(element, annotation, finalTestname)).orElse(ENABLED); return result; } - private ConditionEvaluationResult toResult(AnnotatedElement element, TestSupportedPlatform annotation) { - KubernetesPlatform platformValue = annotation.value(); - if (kubeClient.isKubernetesPlatform() && platformValue.equals(KubernetesPlatform.KUBERNETES)) { - return ConditionEvaluationResult.enabled("Test enabled on Kubernetes-like platforms"); - } + private ConditionEvaluationResult toResult(AnnotatedElement element, TestSupportedPlatform annotation, String testName) { + Collection kubeClients = ResourceManager.getEnvironment().getKubeClients().values(); + List resultList = new ArrayList<>(); + for (KubeClient kubeClient : kubeClients) { + KubernetesPlatform platformValue = annotation.value(); + if (kubeClient.isKubernetesPlatform() && platformValue.equals(KubernetesPlatform.KUBERNETES)) { + resultList.add(ConditionEvaluationResult.enabled("Test enabled on Kubernetes-like platforms")); + continue; + } - if (kubeClient.isOpenshiftPlatform() && platformValue.equals(KubernetesPlatform.OPENSHIFT)) { - return ConditionEvaluationResult.enabled("Test enabled on Openshift"); - } + if (kubeClient.isOpenshiftPlatform() && platformValue.equals(KubernetesPlatform.OPENSHIFT)) { + resultList.add(ConditionEvaluationResult.enabled("Test enabled on Openshift")); + continue; + } - LOGGER.info("[TEST] Skipped: Test/class does not meet TestValidSince criteria."); - return ConditionEvaluationResult.disabled("[TEST] Skipped: Unsupported platform for this test."); + LOGGER.info("[TEST][{}] Skipped: Test/class does not meet TestValidSince criteria.", testName); + return ConditionEvaluationResult.disabled("[TEST] Skipped: Unsupported platform for this test." + kubeClient.getKubernetesPlatform()); + } + if (resultList.stream().allMatch(resultList.get(0)::equals)) { + return resultList.get(0); + } else { + LOGGER.warn("[TEST][{}]Disabling test - Detected mixture of provided platforms. This is not supported.", testName); + return ConditionEvaluationResult.disabled("[TEST] Skipped: Unsupported platform for this test."); + } } } } diff --git a/operator-suite/src/main/java/io/brokerqe/claire/operator/ArtemisCloudClusterOperatorFile.java b/operator-suite/src/main/java/io/brokerqe/claire/operator/ArtemisCloudClusterOperatorFile.java index 06d39208..551f3d93 100644 --- a/operator-suite/src/main/java/io/brokerqe/claire/operator/ArtemisCloudClusterOperatorFile.java +++ b/operator-suite/src/main/java/io/brokerqe/claire/operator/ArtemisCloudClusterOperatorFile.java @@ -5,6 +5,7 @@ package io.brokerqe.claire.operator; import io.brokerqe.claire.ArtemisConstants; +import io.brokerqe.claire.KubeClient; import io.brokerqe.claire.ResourceManager; import io.brokerqe.claire.TestUtils; import io.brokerqe.claire.helpers.DataStorer; @@ -65,32 +66,36 @@ public ArtemisCloudClusterOperatorFile(String deploymentNamespace, boolean isNam } public static void deployOperatorCRDs() { - DEFAULT_OPERATOR_INSTALL_CRD_FILES.forEach(fileName -> { - try { - ArtemisCloudClusterOperator.LOGGER.debug("[Operator] Deploying CRD file {}", fileName); - List resources = ResourceManager.getKubeClient().getKubernetesClient().load(new FileInputStream(fileName.toFile())).createOrReplace(); - DataStorer.dumpResourceToFile(resources); - } catch (FileNotFoundException e) { - throw new RuntimeException(e); - } - }); - ArtemisCloudClusterOperator.LOGGER.info("[Operator] Deployed Cluster operator CRDs"); + for (KubeClient kubeclient : ResourceManager.getKubeClients()) { + DEFAULT_OPERATOR_INSTALL_CRD_FILES.forEach(fileName -> { + try { + ArtemisCloudClusterOperator.LOGGER.debug("[Operator] Deploying CRD file {}", fileName); + List resources = kubeclient.getKubernetesClient().load(new FileInputStream(fileName.toFile())).createOrReplace(); + DataStorer.dumpResourceToFile(resources); + } catch (FileNotFoundException e) { + throw new RuntimeException(e); + } + }); + ArtemisCloudClusterOperator.LOGGER.info("[Operator] Deployed Cluster operator CRDs"); + } } public static void undeployOperatorCRDs(boolean waitForUndeployment) { - DEFAULT_OPERATOR_INSTALL_CRD_FILES.forEach(fileName -> { - try { - LOGGER.debug("[Operator] Undeploying CRD file {}", fileName); - List result = ResourceManager.getKubeClient().getKubernetesClient().load(new FileInputStream(fileName.toFile())).delete(); - } catch (FileNotFoundException e) { - throw new RuntimeException(e); + for (KubeClient kubeclient : ResourceManager.getKubeClients()) { + DEFAULT_OPERATOR_INSTALL_CRD_FILES.forEach(fileName -> { + try { + LOGGER.debug("[Operator] Undeploying CRD file {}", fileName); + List result = kubeclient.getKubernetesClient().load(new FileInputStream(fileName.toFile())).delete(); + } catch (FileNotFoundException e) { + throw new RuntimeException(e); + } + }); + if (waitForUndeployment) { + // todo + LOGGER.warn("!!wait for undeployment not implemented yet!!"); } - }); - if (waitForUndeployment) { - // todo - LOGGER.warn("!!wait for undeployment not implemented yet!!"); + LOGGER.info("[Operator] Undeployed Cluster operator CRDs"); } - LOGGER.info("[Operator] Undeployed Cluster operator CRDs"); } @Override diff --git a/operator-suite/src/main/resources/logback.xml b/operator-suite/src/main/resources/logback.xml index 4029151a..042799cf 100644 --- a/operator-suite/src/main/resources/logback.xml +++ b/operator-suite/src/main/resources/logback.xml @@ -1,7 +1,7 @@ - %d{yyyy-MM-dd'T'HH:mm:ss.SSS'Z',GMT} %highlight(%-5p) [%c{0}:%L] %m%n + %d{yyyy-MM-dd'T'HH:mm:ss.SSS'Z',GMT} %highlight(%-5p) [%cyan(%X{kubecontext:-})][%c{0}:%L] %m%n @@ -9,7 +9,7 @@ test-logs/operator-suite.log false - %d{yyyy-MM-dd'T'HH:mm:ss.SSS'Z',GMT} %highlight(%-5p) [%c{0}:%L] %m%n + %d{yyyy-MM-dd'T'HH:mm:ss.SSS'Z',GMT} %highlight(%-5p) [%cyan(%X{kubecontext:-})][%c{0}:%L] %m%n diff --git a/operator-suite/src/test/java/io/brokerqe/claire/AbstractSystemTests.java b/operator-suite/src/test/java/io/brokerqe/claire/AbstractSystemTests.java index f858303f..09b90211 100644 --- a/operator-suite/src/test/java/io/brokerqe/claire/AbstractSystemTests.java +++ b/operator-suite/src/test/java/io/brokerqe/claire/AbstractSystemTests.java @@ -13,15 +13,22 @@ import io.brokerqe.claire.clients.MessagingClient; import io.brokerqe.claire.clients.bundled.ArtemisCommand; import io.brokerqe.claire.clients.bundled.BundledArtemisClient; +import io.brokerqe.claire.exception.ClaireNotImplementedException; import io.brokerqe.claire.exception.ClaireRuntimeException; import io.brokerqe.claire.junit.TestSeparator; import io.brokerqe.claire.operator.ArtemisCloudClusterOperator; import io.brokerqe.claire.security.CertificateManager; +import io.fabric8.kubernetes.api.model.HasMetadata; import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.api.model.Service; import io.fabric8.kubernetes.api.model.ServicePort; import io.fabric8.kubernetes.api.model.apps.Deployment; import io.fabric8.kubernetes.client.KubernetesClient; +import io.fabric8.kubernetes.client.dsl.base.PatchContext; +import io.fabric8.kubernetes.client.dsl.base.PatchType; +import io.fabric8.openshift.api.model.Route; +import io.fabric8.openshift.api.model.TLSConfigBuilder; +import io.fabric8.openshift.client.OpenShiftClient; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.TestInfo; @@ -30,17 +37,23 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URLConnection; import java.nio.file.Path; import java.time.Duration; import java.time.Instant; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Scanner; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -98,7 +111,6 @@ protected void cleanResourcesAfterTest(String namespace) { } public KubeClient getClient() { - // client? if (ResourceManager.getKubeClient() == null) { ResourceManager.getInstance(testEnvironmentOperator); } @@ -106,6 +118,11 @@ public KubeClient getClient() { return client; } + public void setClient(KubeClient kubeClient) { + ResourceManager.setKubeClient(kubeClient); + client = kubeClient; + } + public KubernetesClient getKubernetesClient() { return this.client.getKubernetesClient(); } @@ -301,6 +318,38 @@ protected String maybeStripBrokerName(String testBrokerName, String testNamespac return testBrokerName; } + public void checkHttpResponse(URLConnection connection, int expectedCode, String expectedString) { + InputStream response = null; + try { + response = connection.getInputStream(); + assertThat(((HttpURLConnection) connection).getResponseCode(), equalTo(expectedCode)); + Scanner scanner = new Scanner(response); + String responseBody = scanner.useDelimiter("\\A").next(); + assertThat(responseBody, containsString(expectedString)); + response.close(); + } catch (IOException e) { + // carry on with execution, we've got expected exception + if (e.getMessage().contains(String.valueOf(expectedCode))) { + assertThat(e.getMessage(), containsString(String.valueOf(expectedCode))); + } else { + throw new RuntimeException(e); + } + } + } + + protected void patchRouteTls(HasMetadata service, String edgeTerminationPolicy, String termination) { + if (getClient().getKubernetesPlatform().equals(KubernetesPlatform.OPENSHIFT)) { + Route route = (Route) service; + route.getSpec().setTls(new TLSConfigBuilder().withInsecureEdgeTerminationPolicy(edgeTerminationPolicy).withTermination(termination).build()); + route.getMetadata().setManagedFields(null); + ((OpenShiftClient) getKubernetesClient()).routes().withName(route.getMetadata().getName()).patch(PatchContext.of(PatchType.SERVER_SIDE_APPLY), route); + TestUtils.threadSleep(Constants.DURATION_5_SECONDS); + } else { + // Once supported, remove Openshift limitation + throw new ClaireNotImplementedException("Ingress is not supported yet!"); + } + } + // Messaging methods public Map> checkMessageCount(String namespace, Pod brokerPod) { return checkMessageCount(namespace, brokerPod, null); @@ -357,14 +406,14 @@ public void testMessaging(ClientType clientType, String namespace, Pod brokerPod } } - public void testTlsMessaging(String namespace, Pod brokerPod, ActiveMQArtemisAddress address, + public void testTlsMessaging(String namespace, ActiveMQArtemisAddress address, String externalBrokerUri, String saslMechanism, String secretName, String clientKeyStore, String clientKeyStorePassword, String clientTrustStore, String clientTrustStorePassword) { - testTlsMessaging(namespace, brokerPod, address, externalBrokerUri, saslMechanism, secretName, + testTlsMessaging(namespace, address, externalBrokerUri, saslMechanism, secretName, null, clientKeyStore, clientKeyStorePassword, clientTrustStore, clientTrustStorePassword); } - public void testTlsMessaging(String namespace, Pod brokerPod, ActiveMQArtemisAddress address, + public void testTlsMessaging(String namespace, ActiveMQArtemisAddress address, String externalBrokerUri, String saslMechanism, String secretName, Pod clientsPod, String clientKeyStore, String clientKeyStorePassword, String clientTrustStore, String clientTrustStorePassword) { Deployment clients = null; diff --git a/operator-suite/src/test/java/io/brokerqe/claire/multicluster/CrossPlatformTests.java b/operator-suite/src/test/java/io/brokerqe/claire/multicluster/CrossPlatformTests.java new file mode 100644 index 00000000..658b9520 --- /dev/null +++ b/operator-suite/src/test/java/io/brokerqe/claire/multicluster/CrossPlatformTests.java @@ -0,0 +1,226 @@ +/* + * Copyright Broker QE authors. + * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html). + */ +package io.brokerqe.claire.multicluster; + +import io.amq.broker.v1beta1.ActiveMQArtemis; +import io.amq.broker.v1beta1.ActiveMQArtemisAddress; +import io.amq.broker.v1beta1.ActiveMQArtemisBuilder; +import io.amq.broker.v1beta1.activemqartemisspec.Acceptors; +import io.brokerqe.claire.AbstractSystemTests; +import io.brokerqe.claire.Constants; +import io.brokerqe.claire.KubeClient; +import io.brokerqe.claire.ResourceManager; +import io.brokerqe.claire.TestUtils; +import io.brokerqe.claire.junit.TestMinimumKubernetesCount; +import io.brokerqe.claire.operator.ArtemisCloudClusterOperator; +import io.brokerqe.claire.operator.ArtemisFileProvider; +import io.brokerqe.claire.scalability.ScalabilityTests; +import io.brokerqe.claire.security.CertificateManager; +import io.brokerqe.claire.security.KeyStoreData; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +@TestMinimumKubernetesCount(2) +public class CrossPlatformTests extends AbstractSystemTests { + + private static final Logger LOGGER = LoggerFactory.getLogger(ScalabilityTests.class); + private final String testNamespace = getRandomNamespaceName("cross-tests", 3); + private KubeClient kubeclient0; + private KubeClient kubeclient1; + private List operators = new ArrayList<>(); + + @BeforeAll + void setupClusterOperator() { + kubeclient0 = ResourceManager.getKubeClient(0); + kubeclient1 = ResourceManager.getKubeClient(1); + setupMulticlusterOperator(kubeclient0, kubeclient1, testNamespace); + } + + private void setupMulticlusterOperator(KubeClient kc0, KubeClient kc1, String namespace) { + for (KubeClient kubeclient : List.of(kubeclient0, kubeclient1)) { + LOGGER.info("[{}] Deploying Operator", namespace); + setClient(kubeclient); + getClient().createNamespace(testNamespace, true); + operators.add(ResourceManager.deployArtemisClusterOperator(testNamespace)); + } + } + + @AfterAll + void teardownClusterOperator() { + for (KubeClient kubeclient : List.of(kubeclient0, kubeclient1)) { + LOGGER.info("[{}] Teardown Operator", testNamespace); + setClient(kubeclient); + for (ArtemisCloudClusterOperator tmpOperator : operators) { + this.operator = tmpOperator; + teardownDefaultClusterOperator(testNamespace); + } + getClient().deleteNamespace(testNamespace); + } + } + + @Test + void externalMessagingTest() { + setClient(kubeclient0); + String brokerSecretName = "broker-tls-secret"; + String clientSecret = "client-tls-secret"; + String amqpAcceptorName = "my-amqp"; + String owireAcceptorName = "my-owire"; + + ActiveMQArtemis broker = ResourceManager.createArtemis(testNamespace, "tls-broker"); + ActiveMQArtemisAddress tlsAddress = ResourceManager.createArtemisAddress(testNamespace, ArtemisFileProvider.getAddressQueueExampleFile()); + Acceptors amqpAcceptors = createAcceptor(amqpAcceptorName, "amqp", 5672, true, true, brokerSecretName, true); + Acceptors owireAcceptors = createAcceptor(owireAcceptorName, "openwire", 61618, true, true, brokerSecretName, true); + Map keystores = CertificateManager.generateDefaultCertificateKeystores( + ResourceManager.generateDefaultBrokerDN(), + ResourceManager.generateDefaultClientDN(), + List.of(ResourceManager.generateSanDnsNames(broker, List.of(amqpAcceptorName, owireAcceptorName))), + null + ); + + // One Way TLS + getClient().createSecretEncodedData(testNamespace, brokerSecretName, CertificateManager.createBrokerKeystoreSecret(keystores)); + + broker = addAcceptorsWaitForPodReload(testNamespace, List.of(amqpAcceptors, owireAcceptors), broker); + String brokerName = broker.getMetadata().getName(); + List brokerUris = getClient().getExternalAccessServiceUrlPrefixName(testNamespace, brokerName + "-" + amqpAcceptorName); + LOGGER.info("[{}] Broker {} is up and running with TLS", testNamespace, brokerName); + + setClient(kubeclient1); + // Two Way - Mutual Authentication (Clients TLS secret) + getClient().createSecretEncodedData(testNamespace, clientSecret, CertificateManager.createClientKeystoreSecret(keystores)); + // TLS Authentication for netty, but for Artemis as Guest due to JAAS settings + testTlsMessaging(testNamespace, tlsAddress, brokerUris.get(0), null, clientSecret, + Constants.CLIENT_KEYSTORE_ID, keystores.get(Constants.CLIENT_KEYSTORE_ID).getPassword(), + Constants.CLIENT_TRUSTSTORE_ID, keystores.get(Constants.CLIENT_TRUSTSTORE_ID).getPassword()); + + getClient().deleteSecret(testNamespace, clientSecret); + setClient(kubeclient0); + ResourceManager.deleteArtemis(testNamespace, broker); + ResourceManager.deleteArtemisAddress(testNamespace, tlsAddress); + getClient().deleteSecret(testNamespace, brokerSecretName); + } + + @Test + void simpleCrossPlatformTest() { + String broker0Name = "artemis0"; + String broker0SecretName = "broker0-tls-secret-" + TestUtils.getRandomString(3); + String broker1Name = "artemis1"; + String broker1SecretName = "broker1-tls-secret-" + TestUtils.getRandomString(3); + String clientSecret = "client-tls-secret-" + TestUtils.getRandomString(3); + String acceptorName = "all-acceptor"; + Acceptors allAcceptors0 = createAcceptor(acceptorName, "all", 61618, true, true, broker0SecretName, true); + Acceptors allAcceptors1 = createAcceptor(acceptorName, "all", 33333, true, true, broker1SecretName, true); + + // Deployment on kubernetes0 + setClient(kubeclient0); + ActiveMQArtemisAddress tlsAddress0 = ResourceManager.createArtemisAddress(testNamespace, "testq", "testq"); + ActiveMQArtemis broker0 = new ActiveMQArtemisBuilder() + .editOrNewMetadata() + .withName(broker0Name) + .withNamespace(testNamespace) + .endMetadata() + .editOrNewSpec() + .editOrNewDeploymentPlan() + .withSize(1) + .withImage("placeholder") + // POSSIBLE BUG? Issue with TLS login +// .withRequireLogin() + .endDeploymentPlan() + .withAcceptors(allAcceptors0) + .editOrNewConsole() + .withExpose(true) + .withSslEnabled(false) + .endConsole() + .endSpec() + .build(); + + Map keystores0 = CertificateManager.generateCertificateKeystores( + ResourceManager.generateDefaultBrokerDN(), + broker0Name, + ResourceManager.generateDefaultClientDN(), + "client0", + List.of(ResourceManager.generateSanDnsNames(broker0, List.of(acceptorName))), + null + ); + getClient().createSecretEncodedData(testNamespace, broker0SecretName, CertificateManager.createBrokerKeystoreSecret(keystores0)); + broker0 = ResourceManager.createArtemis(testNamespace, broker0); + List brokerUris0 = getClient().getExternalAccessServiceUrlPrefixName(testNamespace, broker0Name + "-" + acceptorName); + LOGGER.info("[{}] Broker {} is up and running with enabled TLS {}", testNamespace, broker0Name, brokerUris0); + + // Deployment on kubernetes1 + setClient(kubeclient1); + ActiveMQArtemisAddress tlsAddress1 = ResourceManager.createArtemisAddress(testNamespace, "testq", "testq"); + ActiveMQArtemis broker1 = new ActiveMQArtemisBuilder() + .editOrNewMetadata() + .withName(broker1Name) + .withNamespace(testNamespace) + .endMetadata() + .editOrNewSpec() + .editOrNewDeploymentPlan() + .withSize(1) + .withImage("placeholder") + .endDeploymentPlan() + .withAcceptors(allAcceptors1) + .editOrNewConsole() + .withExpose(true) + .withSslEnabled(false) + .endConsole() + .endSpec() + .build(); + + Map keystores1 = CertificateManager.generateCertificateKeystores( + ResourceManager.generateDefaultBrokerDN(), + broker1Name, + ResourceManager.generateDefaultClientDN(), + "client1", + List.of(ResourceManager.generateSanDnsNames(broker1, List.of(acceptorName))), + null + ); + + getClient().createSecretEncodedData(testNamespace, broker1SecretName, CertificateManager.createBrokerKeystoreSecret(keystores1)); + broker1 = ResourceManager.createArtemis(testNamespace, broker1); + List brokerUris1 = getClient().getExternalAccessServiceUrlPrefixName(testNamespace, broker1Name + "-" + acceptorName); + LOGGER.info("[{}] Broker {} is up and running with enabled TLS {}", testNamespace, broker1Name, brokerUris1); + + // ======== test cross platform messaging + // setClient(kubeclient1); + LOGGER.info("[{}] Test Cross platform messaging kubeclient1 -> broker0", testNamespace); + getClient().createSecretEncodedData(testNamespace, clientSecret, CertificateManager.createClientKeystoreSecret(keystores0)); + LOGGER.info("[{}] Starting SystemTest clients AMQP subscriber - publisher test", testNamespace); + testTlsMessaging(testNamespace, tlsAddress0, brokerUris0.get(0), null, clientSecret, + Constants.CLIENT_KEYSTORE_ID, keystores0.get(Constants.CLIENT_KEYSTORE_ID).getPassword(), + Constants.CLIENT_TRUSTSTORE_ID, keystores0.get(Constants.CLIENT_TRUSTSTORE_ID).getPassword()); + + + LOGGER.info("[{}] Test Cross platform messaging kubeclient0 -> broker1", testNamespace); + setClient(kubeclient0); + getClient().createSecretEncodedData(testNamespace, clientSecret, CertificateManager.createClientKeystoreSecret(keystores1)); + LOGGER.info("[{}] Starting SystemTest clients AMQP subscriber - publisher test", testNamespace); + testTlsMessaging(testNamespace, tlsAddress1, brokerUris1.get(0), null, clientSecret, + Constants.CLIENT_KEYSTORE_ID, keystores1.get(Constants.CLIENT_KEYSTORE_ID).getPassword(), + Constants.CLIENT_TRUSTSTORE_ID, keystores1.get(Constants.CLIENT_TRUSTSTORE_ID).getPassword()); + + LOGGER.info("[{}] Cross Messaging tests finished successfully. Tearing down environment", testNamespace); + setClient(kubeclient1); + ResourceManager.deleteArtemis(testNamespace, broker1); + ResourceManager.deleteArtemisAddress(testNamespace, tlsAddress1); + getClient().deleteSecret(testNamespace, clientSecret); + getClient().deleteSecret(testNamespace, broker1SecretName); + + setClient(kubeclient0); + ResourceManager.deleteArtemis(testNamespace, broker0); + ResourceManager.deleteArtemisAddress(testNamespace, tlsAddress0); + getClient().deleteSecret(testNamespace, clientSecret); + getClient().deleteSecret(testNamespace, broker0SecretName); + } + +} diff --git a/operator-suite/src/test/java/io/brokerqe/claire/security/TLSProviderTests.java b/operator-suite/src/test/java/io/brokerqe/claire/security/TLSProviderTests.java index e4a50c73..fc7b1737 100644 --- a/operator-suite/src/test/java/io/brokerqe/claire/security/TLSProviderTests.java +++ b/operator-suite/src/test/java/io/brokerqe/claire/security/TLSProviderTests.java @@ -132,7 +132,7 @@ void testTLSProviders(String sslProvider) { assertBrokerConfigHasSslProvider(brokerPod, sslProvider); Deployment clients = ResourceManager.deploySecuredClientsContainer(testNamespace, List.of(clientSecret)); Pod clientsPod = getClient().getFirstPodByPrefixName(testNamespace, Constants.PREFIX_SYSTEMTESTS_CLIENTS); - testTlsMessaging(testNamespace, brokerPod, tlsAddress, brokerUris.get(0), null, clientSecret, clientsPod, + testTlsMessaging(testNamespace, tlsAddress, brokerUris.get(0), null, clientSecret, clientsPod, Constants.CLIENT_KEYSTORE_ID, keystores.get(Constants.CLIENT_KEYSTORE_ID).getPassword(), Constants.CLIENT_TRUSTSTORE_ID, keystores.get(Constants.CLIENT_TRUSTSTORE_ID).getPassword()); ResourceManager.undeployClientsContainer(testNamespace, clients); diff --git a/operator-suite/src/test/java/io/brokerqe/claire/security/TLSSecurityTests.java b/operator-suite/src/test/java/io/brokerqe/claire/security/TLSSecurityTests.java index 8325fd38..30075800 100644 --- a/operator-suite/src/test/java/io/brokerqe/claire/security/TLSSecurityTests.java +++ b/operator-suite/src/test/java/io/brokerqe/claire/security/TLSSecurityTests.java @@ -14,7 +14,6 @@ import io.brokerqe.claire.junit.TestValidSince; import io.brokerqe.claire.junit.TestValidUntil; import io.brokerqe.claire.operator.ArtemisFileProvider; -import io.fabric8.kubernetes.api.model.Pod; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -90,12 +89,11 @@ public void doTestMutualAuthentication(boolean singleSecret) { broker = addAcceptorsWaitForPodReload(testNamespace, List.of(amqpAcceptors, owireAcceptors), broker); String brokerName = broker.getMetadata().getName(); - Pod brokerPod = getClient().getFirstPodByPrefixName(testNamespace, brokerName); List brokerUris = getClient().getExternalAccessServiceUrlPrefixName(testNamespace, brokerName + "-" + amqpAcceptorName); LOGGER.info("[{}] Broker {} is up and running with TLS", testNamespace, brokerName); // TLS Authentication for netty, but for Artemis as Guest due to JAAS settings - testTlsMessaging(testNamespace, brokerPod, tlsAddress, brokerUris.get(0), null, clientSecret, + testTlsMessaging(testNamespace, tlsAddress, brokerUris.get(0), null, clientSecret, Constants.CLIENT_KEYSTORE_ID, keystores.get(Constants.CLIENT_KEYSTORE_ID).getPassword(), Constants.CLIENT_TRUSTSTORE_ID, keystores.get(Constants.CLIENT_TRUSTSTORE_ID).getPassword()); diff --git a/operator-suite/src/test/java/io/brokerqe/claire/smoke/SmokeTests.java b/operator-suite/src/test/java/io/brokerqe/claire/smoke/SmokeTests.java index 12232109..e4716bf2 100644 --- a/operator-suite/src/test/java/io/brokerqe/claire/smoke/SmokeTests.java +++ b/operator-suite/src/test/java/io/brokerqe/claire/smoke/SmokeTests.java @@ -13,6 +13,7 @@ import io.brokerqe.claire.ResourceManager; import io.brokerqe.claire.clients.ClientType; import io.brokerqe.claire.clients.MessagingClient; +import io.brokerqe.claire.exception.ClaireRuntimeException; import io.brokerqe.claire.operator.ArtemisFileProvider; import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.api.model.Service; @@ -59,7 +60,7 @@ void brokerErrorTest() { Pod brokerPod = getClient().getFirstPodByPrefixName(testNamespace, broker.getMetadata().getName()); broker = ResourceManager.getArtemisClient().inNamespace(testNamespace).resource(broker).createOrReplace(); ResourceManager.waitForBrokerDeployment(testNamespace, broker, false, brokerPod); - throw new RuntimeException("Throwing random exception, to trigger TestDataCollection."); + throw new ClaireRuntimeException("Throwing random exception, to trigger TestDataCollection."); } @Test diff --git a/operator-suite/src/test/java/io/brokerqe/claire/webconsole/WebConsoleTests.java b/operator-suite/src/test/java/io/brokerqe/claire/webconsole/WebConsoleTests.java index 763a99cf..57eeb39b 100644 --- a/operator-suite/src/test/java/io/brokerqe/claire/webconsole/WebConsoleTests.java +++ b/operator-suite/src/test/java/io/brokerqe/claire/webconsole/WebConsoleTests.java @@ -9,13 +9,10 @@ import io.brokerqe.claire.AbstractSystemTests; import io.brokerqe.claire.ArtemisConstants; import io.brokerqe.claire.ArtemisVersion; -import io.brokerqe.claire.Constants; import io.brokerqe.claire.KubernetesPlatform; -import io.brokerqe.claire.exception.ClaireNotImplementedException; -import io.brokerqe.claire.junit.TestSupportedPlatform; import io.brokerqe.claire.ResourceManager; import io.brokerqe.claire.TestUtils; - +import io.brokerqe.claire.junit.TestSupportedPlatform; import io.brokerqe.claire.junit.TestValidSince; import io.brokerqe.claire.junit.TestValidUntil; import io.brokerqe.claire.security.CertificateManager; @@ -23,29 +20,20 @@ import io.fabric8.kubernetes.api.model.HasMetadata; import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.api.model.Secret; -import io.fabric8.kubernetes.client.dsl.base.PatchContext; -import io.fabric8.kubernetes.client.dsl.base.PatchType; import io.fabric8.openshift.api.model.Route; -import io.fabric8.openshift.api.model.TLSConfigBuilder; -import io.fabric8.openshift.client.OpenShiftClient; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.io.InputStream; import java.net.HttpURLConnection; -import java.net.URLConnection; import java.util.List; import java.util.Map; -import java.util.Scanner; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.not; public class WebConsoleTests extends AbstractSystemTests { @@ -62,26 +50,6 @@ void teardownClusterOperator() { teardownDefaultClusterOperator(testNamespace); } - - public void checkHttpResponse(URLConnection connection, int expectedCode, String expectedString) { - InputStream response = null; - try { - response = connection.getInputStream(); - assertThat(((HttpURLConnection) connection).getResponseCode(), equalTo(expectedCode)); - Scanner scanner = new Scanner(response); - String responseBody = scanner.useDelimiter("\\A").next(); - assertThat(responseBody, containsString(expectedString)); - response.close(); - } catch (IOException e) { - // carry on with execution, we've got expected exception - if (e.getMessage().contains(String.valueOf(expectedCode))) { - assertThat(e.getMessage(), containsString(String.valueOf(expectedCode))); - } else { - throw new RuntimeException(e); - } - } - } - @Test @TestSupportedPlatform(KubernetesPlatform.OPENSHIFT) public void unsecuredConsoleTLSExternalAccessTest() { @@ -118,16 +86,7 @@ public void unsecuredConsoleTLSExternalAccessTest() { LOGGER.info("[{}] Probing https request on console should fail.", testNamespace); checkHttpResponse(TestUtils.makeInsecureHttpsRequest(url), HttpURLConnection.HTTP_UNAVAILABLE, "Application is not available"); - if (getClient().getKubernetesPlatform().equals(KubernetesPlatform.OPENSHIFT)) { - Route route = (Route) service; - route.getSpec().setTls(new TLSConfigBuilder().withInsecureEdgeTerminationPolicy("Redirect").withTermination("edge").build()); - route.getMetadata().setManagedFields(null); - ((OpenShiftClient) getKubernetesClient()).routes().withName(route.getMetadata().getName()).patch(PatchContext.of(PatchType.SERVER_SIDE_APPLY), route); - TestUtils.threadSleep(Constants.DURATION_5_SECONDS); - } else { - // Once supported, remove Openshift limitation - throw new ClaireNotImplementedException("Ingress is not supported yet!"); - } + patchRouteTls(service, "Redirect", "edge"); LOGGER.info("[{}] Probing https request on console should pass", testNamespace); checkHttpResponse(TestUtils.makeInsecureHttpsRequest(url), HttpURLConnection.HTTP_OK, "hawtio-login"); } @@ -182,8 +141,13 @@ public void doConnectSecurelyConsoleTest(boolean checkLogForSecret) { Pod operatorPod = getClient().getFirstPodByPrefixName(testNamespace, operator.getOperatorName()); ResourceManager.createArtemis(testNamespace, artemis, true); - List webserviceUrl = getClient().getExternalAccessServicePrefixName(testNamespace, brokerName + "-" + ArtemisConstants.WEBCONSOLE_URI_PREFIX); - LOGGER.info("[{}] webservice url {}", testNamespace, webserviceUrl); + HasMetadata webservice = getClient().getExternalAccessServicePrefixName(testNamespace, brokerName + "-" + ArtemisConstants.WEBCONSOLE_URI_PREFIX).get(0); + if (getClient().getKubernetesPlatform().equals(KubernetesPlatform.OPENSHIFT)) { + Route route = (Route) webservice; + LOGGER.debug("[{}] webservice url https://{}", testNamespace, route.getSpec().getHost()); + } else { + LOGGER.debug("[{}] webservice url in details of {}", testNamespace, webservice.getMetadata().getName()); + } if (checkLogForSecret) { LOGGER.info("[{}] Checking for 'Failed to create new Secret error' in CO log", testNamespace);