From 0af5e2b5b97ca5e22d6504ef70244d289dab3de9 Mon Sep 17 00:00:00 2001 From: Anthony Lai Date: Tue, 13 Feb 2018 10:23:25 -0800 Subject: [PATCH 001/186] WebLogic dynamic cluster support --- .../operator/helpers/PodHelper.java | 16 +- .../kubernetes/operator/http/HttpClient.java | 38 +-- .../operator/logging/MessageKeys.java | 6 +- .../operator/rest/RestBackendImpl.java | 4 +- .../operator/wlsconfig/MacroSubstitutor.java | 131 ++++++++ .../operator/wlsconfig/WlsClusterConfig.java | 171 ++++++++++- .../wlsconfig/WlsConfigRetriever.java | 242 +++++++++++---- .../operator/wlsconfig/WlsDomainConfig.java | 176 ++++++++--- .../wlsconfig/WlsDynamicServerConfig.java | 105 +++++++ .../wlsconfig/WlsDynamicServersConfig.java | 200 ++++++++++++ .../operator/wlsconfig/WlsServerConfig.java | 196 ++++++++++-- src/main/resources/Operator.properties | 5 + .../operator/ExternalChannelTest.java | 4 +- .../operator/helpers/IngressHelperTest.java | 6 +- .../wlsconfig/MacroSubstitutorTest.java | 102 +++++++ .../wlsconfig/WlsClusterConfigTest.java | 151 ++++++++- .../wlsconfig/WlsConfigRetreiverTest.java | 8 +- .../wlsconfig/WlsDomainConfigTest.java | 286 ++++++++++++++++-- .../wlsconfig/WlsDynamicServerConfigTest.java | 95 ++++++ 19 files changed, 1744 insertions(+), 198 deletions(-) create mode 100644 src/main/java/oracle/kubernetes/operator/wlsconfig/MacroSubstitutor.java create mode 100644 src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDynamicServerConfig.java create mode 100644 src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDynamicServersConfig.java create mode 100644 src/test/java/oracle/kubernetes/operator/wlsconfig/MacroSubstitutorTest.java create mode 100644 src/test/java/oracle/kubernetes/operator/wlsconfig/WlsDynamicServerConfigTest.java diff --git a/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java b/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java index aaefb07dba5..bd8490462f1 100644 --- a/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java +++ b/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java @@ -149,7 +149,9 @@ public NextAction apply(Packet packet) { volumeMountSecret.setMountPath("/weblogic-operator/secrets"); container.addVolumeMountsItem(volumeMountSecret); - container.addCommandItem("/shared/domain/" + weblogicDomainName + "/servers/" + spec.getAsName() + "/nodemgr_home/startServer.sh"); + container.addCommandItem("/shared/domain/" + weblogicDomainName + "/nodemgr_home/startServer.sh"); + container.addCommandItem(weblogicDomainUID); + container.addCommandItem(spec.getAsName()); V1Probe readinessProbe = new V1Probe(); V1HTTPGetAction httpGet = new V1HTTPGetAction(); @@ -163,7 +165,8 @@ public NextAction apply(Packet packet) { V1Probe livenessProbe = new V1Probe(); V1ExecAction livenessExecAction = new V1ExecAction(); - livenessExecAction.addCommandItem("/shared/domain/" + weblogicDomainName + "/servers/" + spec.getAsName() + "/nodemgr_home/livenessProbe.sh"); + livenessExecAction.addCommandItem("/shared/domain/" + weblogicDomainName + "/nodemgr_home/livenessProbe.sh"); + livenessExecAction.addCommandItem(spec.getAsName()); livenessProbe.exec(livenessExecAction); livenessProbe.setInitialDelaySeconds(10); livenessProbe.setPeriodSeconds(1); @@ -493,7 +496,11 @@ public NextAction apply(Packet packet) { volumeMountSecret.setMountPath("/weblogic-operator/secrets"); container.addVolumeMountsItem(volumeMountSecret); - container.addCommandItem("/shared/domain/" + weblogicDomainName + "/servers/" + weblogicServerName + "/nodemgr_home/startServer.sh"); + container.addCommandItem("/shared/domain/" + weblogicDomainName + "/nodemgr_home/startServer.sh"); + container.addCommandItem(weblogicDomainUID); + container.addCommandItem(weblogicServerName); + container.addCommandItem(spec.getAsName()); + container.addCommandItem(String.valueOf(spec.getAsPort())); V1Probe readinessProbe = new V1Probe(); V1HTTPGetAction httpGet = new V1HTTPGetAction(); @@ -507,7 +514,8 @@ public NextAction apply(Packet packet) { V1Probe livenessProbe = new V1Probe(); V1ExecAction execAction = new V1ExecAction(); - execAction.addCommandItem("/shared/domain/" + weblogicDomainName + "/servers/" + weblogicServerName + "/nodemgr_home/livenessProbe.sh"); + execAction.addCommandItem("/shared/domain/" + weblogicDomainName + "/nodemgr_home/livenessProbe.sh"); + execAction.addCommandItem(weblogicServerName); livenessProbe.exec(execAction); livenessProbe.setInitialDelaySeconds(10); livenessProbe.setPeriodSeconds(1); diff --git a/src/main/java/oracle/kubernetes/operator/http/HttpClient.java b/src/main/java/oracle/kubernetes/operator/http/HttpClient.java index 507dd0c1d53..cb48c7382d1 100644 --- a/src/main/java/oracle/kubernetes/operator/http/HttpClient.java +++ b/src/main/java/oracle/kubernetes/operator/http/HttpClient.java @@ -34,7 +34,6 @@ public class HttpClient { private static final LoggingFacade LOGGER = LoggingFactory.getLogger("Operator", "Operator"); private Client httpClient; - private String principal; private String encodedCredentials; private static final String HTTP_PROTOCOL = "http://"; @@ -42,14 +41,13 @@ public class HttpClient { // for debugging private static final String SERVICE_URL = System.getProperty("oracle.kubernetes.operator.http.HttpClient.SERVICE_URL"); - private HttpClient(Client httpClient, String principal, String encodedCredentials) { + private HttpClient(Client httpClient, String encodedCredentials) { this.httpClient = httpClient; - this.principal = principal; this.encodedCredentials = encodedCredentials; } public String executeGetOnServiceClusterIP(String requestUrl, ClientHolder client, String serviceName, String namespace) { - String serviceURL = SERVICE_URL == null ? getServiceURL(client, principal, serviceName, namespace) : SERVICE_URL; + String serviceURL = SERVICE_URL == null ? getServiceURL(client, serviceName, namespace) : SERVICE_URL; String url = serviceURL + requestUrl; WebTarget target = httpClient.target(url); Invocation.Builder invocationBuilder = target.request().accept("application/json").header("Authorization", "Basic " + encodedCredentials); @@ -65,16 +63,16 @@ public String executeGetOnServiceClusterIP(String requestUrl, ClientHolder clien } public String executePostUrlOnServiceClusterIP(String requestUrl, ClientHolder client, String serviceName, String namespace, String payload) { - String serviceURL = SERVICE_URL == null ? getServiceURL(client, principal, serviceName, namespace) : SERVICE_URL; - return executePostUrlOnServiceClusterIP(requestUrl, serviceURL, serviceName, namespace, payload); + String serviceURL = SERVICE_URL == null ? getServiceURL(client, serviceName, namespace) : SERVICE_URL; + return executePostUrlOnServiceClusterIP(requestUrl, serviceURL, payload); } - public String executePostUrlOnServiceClusterIP(String requestUrl, String serviceURL, String serviceName, String namespace, String payload) { + public String executePostUrlOnServiceClusterIP(String requestUrl, String serviceURL, String payload) { String url = serviceURL + requestUrl; WebTarget target = httpClient.target(url); Invocation.Builder invocationBuilder = target.request().accept("application/json") .header("Authorization", "Basic " + encodedCredentials) - .header("X-Requested-By", "MyClient"); + .header("X-Requested-By", "Weblogic Operator"); Response response = invocationBuilder.post(Entity.json(payload)); LOGGER.finer("Response is " + response.getStatusInfo()); if (response.getStatusInfo().getFamily() == Response.Status.Family.SUCCESSFUL) { @@ -95,8 +93,8 @@ public String executePostUrlOnServiceClusterIP(String requestUrl, String service * @param next Next processing step * @return step to create client */ - public static Step createAuthenticatedClientForAdminServer(String principal, String namespace, String adminSecretName, Step next) { - return new AuthenticatedClientForAdminServerStep(namespace, adminSecretName, new WithSecretDataStep(principal, next)); + public static Step createAuthenticatedClientForAdminServer(String namespace, String adminSecretName, Step next) { + return new AuthenticatedClientForAdminServerStep(namespace, adminSecretName, new WithSecretDataStep(next)); } private static class AuthenticatedClientForAdminServerStep extends Step { @@ -117,11 +115,9 @@ public NextAction apply(Packet packet) { } private static class WithSecretDataStep extends Step { - private final String principal; - public WithSecretDataStep(String principal, Step next) { + public WithSecretDataStep(Step next) { super(next); - this.principal = principal; } @Override @@ -134,7 +130,7 @@ public NextAction apply(Packet packet) { username = secretData.get(SecretHelper.ADMIN_SERVER_CREDENTIALS_USERNAME); password = secretData.get(SecretHelper.ADMIN_SERVER_CREDENTIALS_PASSWORD); } - packet.put(KEY, createAuthenticatedClient(principal, username, password)); + packet.put(KEY, createAuthenticatedClient(username, password)); Arrays.fill(username, (byte) 0); Arrays.fill(password, (byte) 0); @@ -145,12 +141,11 @@ public NextAction apply(Packet packet) { /** * Create authenticated client specifically targeted at an admin server * @param client Client holder - * @param principal Principal * @param namespace Namespace * @param adminSecretName Admin secret name * @return authenticated client */ - public static HttpClient createAuthenticatedClientForAdminServer(ClientHolder client, String principal, String namespace, String adminSecretName) { + public static HttpClient createAuthenticatedClientForAdminServer(ClientHolder client, String namespace, String adminSecretName) { SecretHelper secretHelper = new SecretHelper(client, namespace); Map secretData = secretHelper.getSecretData(SecretHelper.SecretType.AdminServerCredentials, adminSecretName); @@ -161,18 +156,16 @@ public static HttpClient createAuthenticatedClientForAdminServer(ClientHolder cl username = secretData.get(SecretHelper.ADMIN_SERVER_CREDENTIALS_USERNAME); password = secretData.get(SecretHelper.ADMIN_SERVER_CREDENTIALS_PASSWORD); } - return createAuthenticatedClient(principal, username, password); + return createAuthenticatedClient(username, password); } /** * Create authenticated HTTP client - * @param principal Principal * @param username Username * @param password Password * @return authenticated client */ - public static HttpClient createAuthenticatedClient(String principal, - final byte[] username, + public static HttpClient createAuthenticatedClient(final byte[] username, final byte[] password) { // build client with authentication information. Client client = ClientBuilder.newClient(); @@ -184,19 +177,18 @@ public static HttpClient createAuthenticatedClient(String principal, System.arraycopy(password, 0, usernameAndPassword, username.length + 1, password.length); encodedCredentials = java.util.Base64.getEncoder().encodeToString(usernameAndPassword); } - return new HttpClient(client, principal, encodedCredentials); + return new HttpClient(client, encodedCredentials); } /** * Returns the URL to access the service; using the service clusterIP and port. * * @param client The ClientHolder that will be used to obtain the Kubernetes API client. - * @param principal The principal that will be used to call the Kubernetes API. * @param name The name of the Service that you want the URL for. * @param namespace The Namespace in which the Service you want the URL for is defined. * @return The URL of the Service, or null if it is not found or principal does not have sufficient permissions. */ - public static String getServiceURL(ClientHolder client, String principal, String name, String namespace) { + public static String getServiceURL(ClientHolder client, String name, String namespace) { try { return getServiceURL(client.callBuilder().readService(name, namespace)); } catch (ApiException e) { diff --git a/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java b/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java index 2b7fc62ef03..22e01ff070e 100644 --- a/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java +++ b/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java @@ -125,5 +125,9 @@ private MessageKeys() { public static final String ROLLING_SERVERS = "WLSKO-0109"; public static final String REMOVING_INGRESS = "WLSKO-0110"; public static final String LIST_INGRESS_FOR_DOMAIN = "WLSKO-0111"; - + public static final String WLS_UPDATE_CLUSTER_SIZE_FAILED = "WLSKO-0112"; + public static final String WLS_UPDATE_CLUSTER_SIZE_TIMED_OUT = "WLSKO-0113"; + public static final String WLS_UPDATE_CLUSTER_SIZE_INVALID_CLUSTER = "WLSKO-0114"; + public static final String WLS_CLUSTER_SIZE_UPDATED = "WLSKO-0115"; + public static final String WLS_SERVER_TEMPLATE_NOT_FOUND = "WLSKO-0116"; } diff --git a/src/main/java/oracle/kubernetes/operator/rest/RestBackendImpl.java b/src/main/java/oracle/kubernetes/operator/rest/RestBackendImpl.java index 671a78833a4..4a1e4d6d39c 100644 --- a/src/main/java/oracle/kubernetes/operator/rest/RestBackendImpl.java +++ b/src/main/java/oracle/kubernetes/operator/rest/RestBackendImpl.java @@ -401,14 +401,14 @@ private ClusterStartup getClusterStartup(Domain domain, String cluster) { private int getWLSConfiguredClusterSize(ClientHolder client, String adminServerServiceName, String cluster, String namespace, String adminSecretName) { WlsConfigRetriever wlsConfigRetriever = WlsConfigRetriever.create(client.getHelper(), namespace, adminServerServiceName, adminSecretName); - WlsDomainConfig wlsDomainConfig = wlsConfigRetriever.readConfig(principal); + WlsDomainConfig wlsDomainConfig = wlsConfigRetriever.readConfig(); WlsClusterConfig wlsClusterConfig = wlsDomainConfig.getClusterConfig(cluster); return wlsClusterConfig.getClusterSize(); } private Map getWLSConfiguredClusters(ClientHolder client, String namespace, String adminServerServiceName, String adminSecretName) { WlsConfigRetriever wlsConfigRetriever = WlsConfigRetriever.create(client.getHelper(), namespace, adminServerServiceName, adminSecretName); - WlsDomainConfig wlsDomainConfig = wlsConfigRetriever.readConfig(principal); + WlsDomainConfig wlsDomainConfig = wlsConfigRetriever.readConfig(); return wlsDomainConfig.getClusterConfigs(); } diff --git a/src/main/java/oracle/kubernetes/operator/wlsconfig/MacroSubstitutor.java b/src/main/java/oracle/kubernetes/operator/wlsconfig/MacroSubstitutor.java new file mode 100644 index 00000000000..f2437aa13d7 --- /dev/null +++ b/src/main/java/oracle/kubernetes/operator/wlsconfig/MacroSubstitutor.java @@ -0,0 +1,131 @@ +// Copyright 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.wlsconfig; + +/** + * + * Substitute macro specified in WLS server template. Behavior of this class should mimic the behavior of the + * macro substitution logic in WebLogic + * + * From WebLogic documentation at https://docs.oracle.com/middleware/12213/wls/DOMCF/server_templates.htm#DOMCF-GUID-EA003F89-C8E4-4CE1-81DF-6FF25F92D21B + * + * You can define a macro for any string attribute in a server template. + * Macros cannot be used for integers or references to other configuration elements. + * The valid macros available for use in server templates are: + * + *
${id}: Instance ID of the dynamically created server; this ID starts at 1 + * + *
${serverName}: The name of the server to which this element belongs + * + *
${clusterName}: The name of the cluster to which this element belongs + * + *
${domainName}: The name of the domain to which this element belongs + * + *
${system-property-name}: If this is not one of the predefined macro names listed previously, then it will be evaluated as a system property, and the value will be returned. If the system property does not exist, then an empty string will be substituted. + * + *
${machineName}: The name of the machine to which this element belongs (Note: This is missing from the documentation) + * + */ +public class MacroSubstitutor { + + private static final String START_MACRO = "${"; + private static final String END_MACRO = "}"; + + private final int id; + private final String serverName; + private final String clusterName; + private final String domainName; + private final String machineName; + + /** + * Creates a MacroSubstitutor with the given macro values that will be used in macro substitution + * + * @param id Value for replacing values in ${id} macro + * @param serverName Value for replacing values in ${serverName} macro + * @param clusterName Value for replacing values in ${clusterName} macro + * @param domainName Value for replacing values in ${domainName} macro + * @param machineName Value for replacing values in ${machineName} macro + */ + public MacroSubstitutor(int id, String serverName, String clusterName, String domainName, String machineName) { + this.id = id; + this.serverName = serverName; + this.clusterName = clusterName; + this.domainName = domainName; + this.machineName = machineName; + } + + /** + * Perform macro substitution. Extracts the macro name and resolves its macro value using values specified + * in this MacroSubstitutor. + * + * @param inputValue String containing macros + * @return String with values substituted for macros + */ + String substituteMacro(String inputValue) { + + if (inputValue == null) { + return inputValue; + } + + int start = 0; + int idx = inputValue.indexOf(START_MACRO); + if (idx == -1) { + return inputValue; + } + + StringBuilder retStr = new StringBuilder(); + + while (idx != -1) { + retStr.append(inputValue.substring(start, idx)); + + int macroIdx = idx; + int end = inputValue.indexOf(END_MACRO, macroIdx); + if (end == -1) { + start = idx; + idx = -1; + continue; + } + + String macro = inputValue.substring(macroIdx + 2, end); + String macroVal = resolveMacroValue(macro); + + if (macroVal != null) { + retStr.append(macroVal); + } + + start = end + 1; + idx = inputValue.indexOf(START_MACRO, start); + } + + retStr.append(inputValue.substring(start)); + return retStr.toString(); + } + + /** + * Return value for the given macro + * + * @param macro Macro to be substituted + * @return Value for the macro + */ + private String resolveMacroValue(String macro) { + if (macro == null || macro.isEmpty()) { + return ""; + } + if (macro.equals("domainName")) { + return domainName; + } else if (macro.equals("serverName")) { + return serverName; + } else if (macro.equals("clusterName")) { + return clusterName; + } else if (macro.equals("machineName")) { + return machineName; + } else if (macro.equals("id")) { + return "" + id; + } + + // Look for macro in ConfigurationProperty or as system property + return System.getProperty(macro); + } + +} diff --git a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfig.java b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfig.java index 2e4d1ff0ea0..47fc1006571 100644 --- a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfig.java +++ b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfig.java @@ -10,6 +10,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; /** * Contains configuration of a WLS cluster @@ -21,24 +22,65 @@ public class WlsClusterConfig { private final String clusterName; private List serverConfigs = new ArrayList<>(); + private final WlsDynamicServersConfig dynamicServersConfig; - public static WlsClusterConfig create(String clusterName) { - return new WlsClusterConfig(clusterName); + /** + * Constructor for a static cluster when Json result is not available + * + * @param clusterName Name of the WLS cluster + */ + public WlsClusterConfig(String clusterName) { + this.clusterName = clusterName; + this.dynamicServersConfig = null; } - public WlsClusterConfig(String clusterName) { + /** + * Constructor for a dynamic cluster + * * + * @param clusterName Name of the WLS cluster + * @param dynamicServersConfig A WlsDynamicServersConfig object containing the dynamic servers configuration for this + * cluster + */ + public WlsClusterConfig(String clusterName, WlsDynamicServersConfig dynamicServersConfig) { this.clusterName = clusterName; + this.dynamicServersConfig = dynamicServersConfig; } + /** + * Creates a WlsClusterConfig object using an "clusters" item parsed from JSON result from WLS REST call + * + * @param clusterConfigMap Map containing "cluster" item parsed from JSON result from WLS REST call + * @param serverTemplates Map containing all server templates configuration read from the WLS domain + * @param domainName Name of the WLS domain that this WLS cluster belongs to + * + * @return A new WlsClusterConfig object created based on the JSON result + */ + static WlsClusterConfig create(Map clusterConfigMap, Map serverTemplates, String domainName) { + String clusterName = (String) clusterConfigMap.get("name"); + WlsDynamicServersConfig dynamicServersConfig = + WlsDynamicServersConfig.create((Map) clusterConfigMap.get("dynamicServers"), serverTemplates, clusterName, domainName); + // set dynamicServersConfig only if the cluster contains dynamic servers, i.e., its dynamic servers configuration + // contains non-null server template name + if (dynamicServersConfig.getServerTemplate() == null) { + dynamicServersConfig = null; + } + return new WlsClusterConfig(clusterName, dynamicServersConfig); + } + /** + * Add a statically configured WLS server to this cluster + * + * @param wlsServerConfig A WlsServerConfig object containing the configuration of the statically configured WLS server + * that belongs to this cluster + */ synchronized void addServerConfig(WlsServerConfig wlsServerConfig) { serverConfigs.add(wlsServerConfig); } /** - * Returns the number of servers configured in this cluster + * Returns the number of servers that are statically configured in this cluster * - * @return The number of servers configured in this cluster + * @return The number of servers that are statically configured in this cluster */ public synchronized int getClusterSize() { return serverConfigs.size(); @@ -54,14 +96,56 @@ public String getClusterName() { } /** - * Returns a list of server configurations for servers that belong to this cluster. + * Returns a list of server configurations for servers that belong to this cluster, which includes + * both statically configured servers and dynamic servers * * @return A list of WlsServerConfig containing configurations of servers that belong to this cluster */ public synchronized List getServerConfigs() { + if (dynamicServersConfig != null) { + List result = new ArrayList<>(dynamicServersConfig.getDynamicClusterSize() + serverConfigs.size()); + result.addAll(dynamicServersConfig.getServerConfigs()); + result.addAll(serverConfigs); + return result; + } return serverConfigs; } + /** + * Whether the cluster contains any statically configured servers + * @return True if the cluster contains any statically configured servers + */ + public synchronized boolean hasStaticServers() { + return !serverConfigs.isEmpty(); + } + + /** + * Whether the cluster contains any dynamic servers + * @return True if the cluster contains any dynamic servers + */ + public boolean hasDynamicServers() { + return dynamicServersConfig != null; + } + + /** + * Returns the current size of the dynamic cluster (the number of dynamic server instances allowed + * to be created) + * + * @return the current size of the dynamic cluster, or -1 if there is no dynamic servers in this cluster + */ + public int getDynamicClusterSize() { + return dynamicServersConfig != null? dynamicServersConfig.getDynamicClusterSize(): -1; + } + + /** + * Returns the maximum size of the dynamic cluster + * + * @return the maximum size of the dynamic cluster, or -1 if there is no dynamic servers in this cluster + */ + public int getMaxDynamicClusterSize() { + return dynamicServersConfig != null? dynamicServersConfig.getMaxDynamicClusterSize(): -1; + } + /** * Validate the clusterStartup configured should be consistent with this configured WLS cluster. The method * also logs warning if inconsistent WLS configurations are found. @@ -78,7 +162,7 @@ public boolean validateClusterStartup(ClusterStartup clusterStartup) { boolean modified = false; // log warning if no servers are configured in the cluster - if (getClusterSize() == 0) { + if (getClusterSize() == 0 && !hasDynamicServers()) { LOGGER.warning(MessageKeys.NO_WLS_SERVER_IN_CLUSTER, clusterName); } @@ -90,17 +174,82 @@ public boolean validateClusterStartup(ClusterStartup clusterStartup) { return modified; } + + /** + * Validate the configured replicas value in the kubernetes weblogic domain spec against the + * configured size of this cluster. Log warning if any inconsistencies are found. + * + * @param replicas The configured replicas value for this cluster in the kubernetes weblogic domain spec + * for this cluster + * @param source The name of the section in the domain spec where the replicas is specified, + * for logging purposes + */ public void validateReplicas(Integer replicas, String source) { - if (replicas != null && replicas > getClusterSize()) { + if (replicas != null && replicas > getClusterSize() && !hasDynamicServers()) { LOGGER.warning(MessageKeys.REPLICA_MORE_THAN_WLS_SERVERS, source, clusterName, replicas, getClusterSize()); } } + /** + * Return the list of configuration attributes to be retrieved from the REST search request to the + * WLS admin server. The value would be used for constructing the REST POST request. + * + * @return The list of configuration attributes to be retrieved from the REST search request + * to the WLS admin server. The value would be used for constructing the REST POST request. + */ + static String getSearchPayload() { + return " fields: [ " + getSearchFields() + " ], " + + " links: [], " + + " children: { " + + " dynamicServers: { " + + " fields: [ " + WlsDynamicServersConfig.getSearchFields() + " ], " + + " links: [] " + + " }" + + " } "; + } + + /** + * Return the fields from cluster WLS configuration that should be retrieved from the WLS REST + * request. + * + * @return A string containing cluster configuration fields that should be retrieved from the WLS REST + * request, in a format that can be used in the REST request payload + */ + private static String getSearchFields() { + return "'name' "; + } + + /** + * Return the URL path of REST request for updating dynamic cluster size + * + * @return The REST URL path for updating cluster size of dynamic servers for this cluster + */ + public String getUpdateDynamicClusterSizeUrl() { + return "/management/weblogic/latest/edit/clusters/" + clusterName + "/dynamicServers"; + } + + /** + * Return the payload used in the REST request for updating the dynamic cluster size. It will + * be used to update the cluster size and if necessary, the max cluster size of the dynamic servers + * of this cluster. + * + * @param clusterSize Desired dynamic cluster size + * @return A string containing the payload to be used in the REST request for updating the dynamic + * cluster size to the specified value. + */ + public String getUpdateDynamicClusterSizePayload(final int clusterSize) { + return "{ dynamicClusterSize: " + clusterSize + ", " + + " maxDynamicClusterSize: " + (clusterSize > getMaxDynamicClusterSize()? clusterSize: getMaxDynamicClusterSize()) + + " }"; + } + @Override public String toString() { return "WlsClusterConfig{" + - "clusterName='" + clusterName + '\'' + - ", serverConfigs=" + serverConfigs + - '}'; + "clusterName='" + clusterName + '\'' + + ", serverConfigs=" + serverConfigs + + ", dynamicServersConfig=" + dynamicServersConfig + + '}'; } + } diff --git a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetriever.java b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetriever.java index f7dcc904d27..2c0606f486c 100644 --- a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetriever.java +++ b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetriever.java @@ -3,7 +3,6 @@ package oracle.kubernetes.operator.wlsconfig; -import oracle.kubernetes.operator.ProcessingConstants; import oracle.kubernetes.operator.domain.model.oracle.kubernetes.weblogic.domain.v1.Domain; import oracle.kubernetes.operator.domain.model.oracle.kubernetes.weblogic.domain.v1.DomainSpec; import oracle.kubernetes.operator.helpers.CallBuilder; @@ -35,7 +34,7 @@ */ public class WlsConfigRetriever { public static final String KEY = "wlsDomainConfig"; - + private ClientHelper clientHelper; private String namespace; private HttpClient httpClient; @@ -47,15 +46,18 @@ public class WlsConfigRetriever { // timeout for reading server configured for the cluster from admin server - default is 180s private static final int READ_CONFIG_TIMEOUT_MILLIS = Integer.getInteger("read.config.timeout.ms", 180000); + // timeout for updating server configuration - default is 60s + private static final int UPDATE_CONFIG_TIMEOUT_MILLIS = Integer.getInteger("update.config.timeout.ms", 60000); + // wait time before retrying to read server configured for the cluster from admin server - default is 1s private static final int READ_CONFIG_RETRY_MILLIS = Integer.getInteger("read.config.retry.ms", 1000); /** * Constructor. * - * @param clientHelper The ClientHelper to be used to obtain the Kubernetes API Client. - * @param namespace The Namespace in which the target Domain is located. - * @param asServiceName The name of the Kubernetes Service which provides access to the Admin Server. + * @param clientHelper The ClientHelper to be used to obtain the Kubernetes API Client. + * @param namespace The Namespace in which the target Domain is located. + * @param asServiceName The name of the Kubernetes Service which provides access to the Admin Server. * @param adminSecretName The name of the Kubernetes Secret which contains the credentials to authenticate to the Admin Server. * @return The WlsConfigRetriever object for the specified inputs. */ @@ -66,9 +68,9 @@ public static WlsConfigRetriever create(ClientHelper clientHelper, String namesp /** * Constructor. * - * @param clientHelper The ClientHelper to be used to obtain the Kubernetes API Client. - * @param namespace The Namespace in which the target Domain is located. - * @param asServiceName The name of the Kubernetes Service which provides access to the Admin Server. + * @param clientHelper The ClientHelper to be used to obtain the Kubernetes API Client. + * @param namespace The Namespace in which the target Domain is located. + * @param asServiceName The name of the Kubernetes Service which provides access to the Admin Server. * @param adminSecretName The name of the Kubernetes Secret which contains the credentials to authenticate to the Admin Server. */ public WlsConfigRetriever(ClientHelper clientHelper, String namespace, String asServiceName, String adminSecretName) { @@ -77,22 +79,23 @@ public WlsConfigRetriever(ClientHelper clientHelper, String namespace, String as this.asServiceName = asServiceName; this.adminSecretName = adminSecretName; } - + /** * Creates asynchronous {@link Step} to read configuration from an admin server + * * @param next Next processing step * @return asynchronous step */ public static Step readConfigStep(Step next) { return new ReadConfigStep(next); } - + private static final String START_TIME = "WlsConfigRetriever-startTime"; private static final String RETRY_COUNT = "WlsConfigRetriever-retryCount"; private static final Random R = new Random(); private static final int HIGH = 1000; private static final int LOW = 100; - + private static final class ReadConfigStep extends Step { public ReadConfigStep(Step next) { super(next); @@ -102,32 +105,31 @@ public ReadConfigStep(Step next) { public NextAction apply(Packet packet) { try { DomainPresenceInfo info = packet.getSPI(DomainPresenceInfo.class); - String principal = (String) packet.get(ProcessingConstants.PRINCIPAL); - + packet.putIfAbsent(START_TIME, Long.valueOf(System.currentTimeMillis())); - + Domain dom = info.getDomain(); V1ObjectMeta meta = dom.getMetadata(); DomainSpec spec = dom.getSpec(); String namespace = meta.getNamespace(); - + String domainUID = spec.getDomainUID(); - + String name = CallBuilder.toDNS1123LegalName(domainUID + "-" + spec.getAsName()); - + String adminSecretName = spec.getAdminSecret() == null ? null : spec.getAdminSecret().getName(); String adminServerServiceName = name; - + Step getClient = HttpClient.createAuthenticatedClientForAdminServer( - principal, namespace, adminSecretName, new WithHttpClientStep(namespace, adminServerServiceName, next)); + namespace, adminSecretName, new WithHttpClientStep(namespace, adminServerServiceName, next)); packet.remove(RETRY_COUNT); return doNext(getClient, packet); } catch (Throwable t) { LOGGER.info(MessageKeys.EXCEPTION, t); - + // Not clear why we should have a maximum time to read the domain configuration. We already know that the // admin server Pod is READY and failing to read the config just means failure, so might as well keep trying. - + // exponential back-off Integer retryCount = (Integer) packet.get(RETRY_COUNT); if (retryCount == null) { @@ -139,7 +141,7 @@ public NextAction apply(Packet packet) { } } } - + private static final class WithHttpClientStep extends Step { private final String namespace; private final String adminServerServiceName; @@ -156,27 +158,27 @@ public NextAction apply(Packet packet) { HttpClient httpClient = (HttpClient) packet.get(HttpClient.KEY); DomainPresenceInfo info = packet.getSPI(DomainPresenceInfo.class); Domain dom = info.getDomain(); - + String serviceURL = HttpClient.getServiceURL(info.getAdmin().getService()); - + WlsDomainConfig wlsDomainConfig = null; - String jsonResult = httpClient.executePostUrlOnServiceClusterIP(WlsDomainConfig.getRetrieveServersSearchUrl(), serviceURL, adminServerServiceName, namespace, WlsDomainConfig.getRetrieveServersSearchPayload()); + String jsonResult = httpClient.executePostUrlOnServiceClusterIP(WlsDomainConfig.getRetrieveServersSearchUrl(), serviceURL, WlsDomainConfig.getRetrieveServersSearchPayload()); if (jsonResult != null) { - wlsDomainConfig = WlsDomainConfig.create().load(jsonResult); + wlsDomainConfig = WlsDomainConfig.create(jsonResult); } - + // validate domain spec against WLS configuration. Currently this only logs warning messages. wlsDomainConfig.updateDomainSpecAsNeeded(dom.getSpec()); - + info.setScan(wlsDomainConfig); info.setLastScanTime(new DateTime()); - + LOGGER.info(MessageKeys.WLS_CONFIGURATION_READ, (System.currentTimeMillis() - ((Long) packet.get(START_TIME))), wlsDomainConfig); return doNext(packet); } catch (Throwable t) { LOGGER.warning(MessageKeys.WLS_CONFIGURATION_READ_FAILED, t); - + // exponential back-off Integer retryCount = (Integer) packet.get(RETRY_COUNT); if (retryCount == null) { @@ -194,20 +196,18 @@ public NextAction apply(Packet packet) { * would repeatedly try to connect to the admin server to retrieve the configuration until the configured timeout * occurs. * - * @param principal The principal that should be used to retrieve the configuration. - * * @return A WlsClusterConfig object containing selected server configurations of all WLS servers configured in the * domain that belongs to a cluster. This method returns an empty configuration object even if it fails to retrieve * WLS configuration from the admin server. */ - public WlsDomainConfig readConfig(String principal) { + public WlsDomainConfig readConfig() { LOGGER.entering(); - long timeout = READ_CONFIG_TIMEOUT_MILLIS; + final long timeout = READ_CONFIG_TIMEOUT_MILLIS; ExecutorService executorService = Executors.newSingleThreadExecutor(); long startTime = System.currentTimeMillis(); - Future future = executorService.submit(() -> getWlsDomainConfig(principal, timeout)); + Future future = executorService.submit(() -> getWlsDomainConfig(timeout)); executorService.shutdown(); WlsDomainConfig wlsConfig = null; try { @@ -219,7 +219,7 @@ public WlsDomainConfig readConfig(String principal) { LOGGER.warning(MessageKeys.WLS_CONFIGURATION_READ_TIMED_OUT, timeout); } if (wlsConfig == null) { - wlsConfig = WlsDomainConfig.create(); + wlsConfig = new WlsDomainConfig(null); } LOGGER.exiting(wlsConfig); @@ -230,52 +230,169 @@ public WlsDomainConfig readConfig(String principal) { * Method called by the Callable that is submitted from the readConfig method for reading the WLS server * configurations * - * @param principal The Service Account or User to use when accessing WebLogic. - * @param timeout Maximum amount of time in millis to try to read configuration from the admin server before giving up + * @param timeoutMillis Maximum amount of time in milliseconds to try to read configuration from the admin server + * before giving up + * * @return A WlsClusterConfig object containing selected server configurations of all WLS servers configured in the * cluster. Method returns empty configuration object if timeout occurs before the configuration could be read from * the admin server. * @throws Exception if an exception occurs in the attempt just prior to the timeout */ - private WlsDomainConfig getWlsDomainConfig(String principal, long timeout) throws Exception { + private WlsDomainConfig getWlsDomainConfig(final long timeoutMillis) throws Exception { LOGGER.entering(); - long stopTime = System.currentTimeMillis() + timeout; - Exception exception = null; WlsDomainConfig result = null; + String jsonResult = executePostUrlWithRetry( + WlsDomainConfig.getRetrieveServersSearchUrl(), + WlsDomainConfig.getRetrieveServersSearchPayload(), + timeoutMillis); + if (jsonResult != null) { + result = WlsDomainConfig.create(jsonResult); + } + LOGGER.exiting(result); + return result; + } + + /** + * Invokes a HTTP POST request using the provided URL and payload, and retry the request until it + * succeeded or the specified timeout period has reached. + * + * @param url The URL of the HTTP post request to be invoked + * @param payload The payload of the HTTP Post request to be invoked + * @param timeoutMillis Timeout in milliseconds + * @return The Json string returned from the HTTP POST request + * @throws Exception Any exception thrown while trying to invoke the HTTP POST request + */ + private String executePostUrlWithRetry(final String url, final String payload, final long timeoutMillis) + throws Exception { + LOGGER.entering(); + + long stopTime = System.currentTimeMillis() + timeoutMillis; + Exception exception = null; + String jsonResult = null; long timeRemaining = stopTime - System.currentTimeMillis(); // keep trying and ignore exceptions until timeout. - while (timeRemaining > 0 && result == null) { + while (timeRemaining > 0 && jsonResult == null) { LOGGER.info(MessageKeys.WLS_CONFIGURATION_READ_TRYING, timeRemaining); exception = null; - ClientHolder client = null; try { - client = clientHelper.take(); - - connectAdminServer(client, principal); - String jsonResult = httpClient.executePostUrlOnServiceClusterIP(WlsDomainConfig.getRetrieveServersSearchUrl(), client, asServiceName, namespace, WlsDomainConfig.getRetrieveServersSearchPayload()); - if (jsonResult != null) { - result = WlsDomainConfig.create().load(jsonResult); - } + jsonResult = executePostUrl(url, payload); } catch (Exception e) { exception = e; LOGGER.info(MessageKeys.WLS_CONFIGURATION_READ_RETRY, e, READ_CONFIG_RETRY_MILLIS); - } finally { - if (client != null) - clientHelper.recycle(client); } - try { - // sleep before retrying - Thread.sleep(READ_CONFIG_RETRY_MILLIS); - } catch (InterruptedException ex) { - // ignore + if (jsonResult == null) { + try { + // sleep before retrying + Thread.sleep(READ_CONFIG_RETRY_MILLIS); + } catch (InterruptedException ex) { + // ignore + } } timeRemaining = stopTime - System.currentTimeMillis(); } - if (result == null && exception != null) { + if (jsonResult == null && exception != null) { LOGGER.throwing(exception); throw exception; } + LOGGER.exiting(jsonResult); + return jsonResult; + } + + /** + * Invokes a HTTP POST request using the provided URL and payload + * + * @param url The URL of the HTTP post request to be invoked + * @param payload The payload of the HTTP Post request to be invoked + * + * @return The Json string returned from the HTTP POST request + * @throws Exception Any exception thrown while invoking the HTTP POST request + */ + private String executePostUrl(final String url, final String payload) + throws Exception { + LOGGER.entering(); + + String jsonResult = null; + ClientHolder client = null; + try { + client = clientHelper.take(); + + connectAdminServer(client); + jsonResult = httpClient.executePostUrlOnServiceClusterIP(url, client, asServiceName, namespace, payload); + } finally { + if (client != null) + clientHelper.recycle(client); + } + LOGGER.exiting(jsonResult); + return jsonResult; + } + /** + * Update the dynamic cluster size of the WLS cluster configuration. + * + * @param wlsClusterConfig The WlsClusterConfig object of the WLS cluster whose cluster size needs to be updated + * @param clusterSize The desire dynamic cluster size + * @return true if the request to update the cluster size is successful, false if it was not successful within the + * time period, or the cluster is not a dynamic cluster + */ + public boolean updateDynamicClusterSize(final WlsClusterConfig wlsClusterConfig, int clusterSize) { + + LOGGER.entering(); + + final long timeout = UPDATE_CONFIG_TIMEOUT_MILLIS; + + String clusterName = wlsClusterConfig == null? "null": wlsClusterConfig.getClusterName(); + + if (wlsClusterConfig == null || !wlsClusterConfig.hasDynamicServers()) { + LOGGER.warning(MessageKeys.WLS_UPDATE_CLUSTER_SIZE_INVALID_CLUSTER, clusterName); + return false; + } + + ExecutorService executorService = Executors.newSingleThreadExecutor(); + long startTime = System.currentTimeMillis(); + Future future = executorService.submit(() -> doUpdateDynamicClusterSize(wlsClusterConfig, clusterSize)); + executorService.shutdown(); + boolean result = false; + try { + result = future.get(timeout, TimeUnit.MILLISECONDS); + if (result) { + LOGGER.info(MessageKeys.WLS_CLUSTER_SIZE_UPDATED, clusterName, clusterSize, (System.currentTimeMillis() - startTime)); + } else { + LOGGER.warning(MessageKeys.WLS_UPDATE_CLUSTER_SIZE_FAILED, clusterName, null); + } + } catch (InterruptedException | ExecutionException e) { + LOGGER.warning(MessageKeys.WLS_UPDATE_CLUSTER_SIZE_FAILED, clusterName, e); + } catch (TimeoutException e) { + LOGGER.warning(MessageKeys.WLS_UPDATE_CLUSTER_SIZE_TIMED_OUT, clusterName, timeout); + } + LOGGER.exiting(result); + return result; + } + + /** + * Method called by the Callable that is submitted from the updateDynamicClusterSize method for updating the + * WLS dynamic cluster size configuration. + * + * @param wlsClusterConfig The WlsClusterConfig object of the WLS cluster whose cluster size needs to be updated. The + * caller should make sure that the cluster is a dynamic cluster. + * @param clusterSize The desire dynamic cluster size + * @return true if the request to update the cluster size is successful, false if it was not successful + */ + + private boolean doUpdateDynamicClusterSize(final WlsClusterConfig wlsClusterConfig, + final int clusterSize) throws Exception { + LOGGER.entering(); + + final String EXPECTED_RESULT = "{}"; + + String jsonResult = executePostUrl( + wlsClusterConfig.getUpdateDynamicClusterSizeUrl(), + wlsClusterConfig.getUpdateDynamicClusterSizePayload(clusterSize)); + boolean result = false; + if (EXPECTED_RESULT.equals(jsonResult)) { + result = true; + } else { + LOGGER.warning(MessageKeys.WLS_UPDATE_CLUSTER_SIZE_FAILED, wlsClusterConfig.getClusterName(), jsonResult); + } LOGGER.exiting(result); return result; } @@ -284,12 +401,11 @@ private WlsDomainConfig getWlsDomainConfig(String principal, long timeout) throw * Connect to the WebLogic Administration Server. * * @param clientHolder The ClientHolder from which to get the Kubernetes API client. - * @param principal The principal that should be used to connect to the Admin Server. */ - public void connectAdminServer(ClientHolder clientHolder, String principal) { + public void connectAdminServer(ClientHolder clientHolder) { if (httpClient == null) { - httpClient = HttpClient.createAuthenticatedClientForAdminServer(clientHolder, principal, namespace, adminSecretName); + httpClient = HttpClient.createAuthenticatedClientForAdminServer(clientHolder, namespace, adminSecretName); } } -} +} \ No newline at end of file diff --git a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfig.java b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfig.java index 4a90b459c10..33b1e08a1e5 100644 --- a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfig.java +++ b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfig.java @@ -21,11 +21,61 @@ public class WlsDomainConfig { private static final LoggingFacade LOGGER = LoggingFactory.getLogger("Operator", "Operator"); + // Contains all configured WLS clusters in the WLS domain private Map wlsClusterConfigs = new HashMap<>(); + // Contains all statically configured WLS servers in the WLS domain private Map wlsServerConfigs = new HashMap<>(); + // Contains all configured server templates in the WLS domain + private Map wlsServerTemplates = new HashMap<>(); - static WlsDomainConfig create() { - return new WlsDomainConfig(); + // Name of this WLS domain (This is NOT the domain UID in the weblogic domain kubernetes CRD) + private final String name; + + /** + * Create a new WlsDomainConfig object using the json result from the WLS REST call + * + * @param jsonResult A String containing the JSON response from the WLS REST call + * + * @return A new WlsDomainConfig object created with information from the JSON response + */ + public static WlsDomainConfig create(String jsonResult) { + ParsedJson parsedResult = parseJson(jsonResult); + return WlsDomainConfig.create(parsedResult); + } + + /** + * Constructor when no JSON response is available + * + * @param name Name of the WLS domain + */ + public WlsDomainConfig(String name) { + this.name = name; + } + + /** + * Constructor + * + * @param name Name of this WLS domain + * @param wlsClusterConfigs A Map containing clusters configured in this WLS domain + * @param wlsServerConfigs A Map containing servers configured in the WLS domain + * @param wlsServerTemplates A Map containing server templates configued in this WLS domain + */ + WlsDomainConfig(String name, Map wlsClusterConfigs, + Map wlsServerConfigs, + Map wlsServerTemplates) { + this.wlsClusterConfigs = wlsClusterConfigs; + this.wlsServerConfigs = wlsServerConfigs; + this.wlsServerTemplates = wlsServerTemplates; + this.name = name; + } + + /** + * Return the name of the WLS domain + * + * @return Name of the WLS domain + */ + public String getName() { + return name; } /** @@ -38,8 +88,9 @@ public synchronized Map getClusterConfigs() { } /** - * Returns configuration of all servers found in the WLS domain, including admin server, standalone managed servers - * that do not belong to any cluster, and managed servers that belong to a cluster. + * Returns configuration of servers found in the WLS domain, including admin server, standalone managed servers + * that do not belong to any cluster, and statically configured managed servers that belong to a cluster. + * It does not include dynamic servers configured in dynamic clusters. * * @return A Map of WlsServerConfig, keyed by name, for each server configured the WLS domain */ @@ -61,13 +112,14 @@ public synchronized WlsClusterConfig getClusterConfig(String clusterName) { } if (result == null) { // create an empty WlsClusterConfig, but do not add to wlsClusterConfigs - result = WlsClusterConfig.create(clusterName); + result = new WlsClusterConfig(clusterName); } return result; } /** - * Returns the configuration for the WLS server with the given name + * Returns the configuration for the WLS server with the given name. Note that this method would not return + * dynamic server. * * @param serverName name of the WLS server * @return The WlsServerConfig object containing configuration of the WLS server with the given name. This methods @@ -81,34 +133,50 @@ public synchronized WlsServerConfig getServerConfig(String serverName) { return result; } - public synchronized WlsDomainConfig load(String jsonResult) { - List> serversMap = parseJson(jsonResult); - if (serversMap != null) { - for (Map thisServer : serversMap) { - WlsServerConfig wlsServerConfig = new WlsServerConfig(thisServer); + /** + * Create a new WlsDomainConfig object based on the parsed JSON result from WLS admin server + * + * @param parsedResult ParsedJson object containing the parsed JSON result + * @return A new WlsDomainConfig object based on the provided parsed JSON result + */ + private static WlsDomainConfig create(ParsedJson parsedResult) { + if (parsedResult == null) { + // return empty WlsDomainConfig if no parsedResult is provided + return new WlsDomainConfig(null); + } + + String name = parsedResult.domainName; + Map wlsClusterConfigs = new HashMap<>(); + Map wlsServerConfigs = new HashMap<>(); + Map wlsServerTemplates = new HashMap<>(); + + // process list of server templates + if (parsedResult.serverTemplates != null) { + for (Map thisServerTemplate : parsedResult.serverTemplates) { + WlsServerConfig wlsServerTemplate = WlsServerConfig.create(thisServerTemplate); + wlsServerTemplates.put(wlsServerTemplate.getName(), wlsServerTemplate); + } + } + // process list of clusters + if (parsedResult.clusters != null) { + for (Map clusterConfig : parsedResult.clusters) { + WlsClusterConfig wlsClusterConfig = WlsClusterConfig.create(clusterConfig, wlsServerTemplates, name); + wlsClusterConfigs.put(wlsClusterConfig.getClusterName(), wlsClusterConfig); + } + } + // process list of statically configured servers + if (parsedResult.servers != null) { + for (Map thisServer : parsedResult.servers) { + WlsServerConfig wlsServerConfig = WlsServerConfig.create(thisServer); wlsServerConfigs.put(wlsServerConfig.getName(), wlsServerConfig); - String clusterName = getClusterNameForServer(thisServer); + String clusterName = WlsServerConfig.getClusterNameFromJsonMap(thisServer); if (clusterName != null) { - WlsClusterConfig wlsClusterConfig = wlsClusterConfigs.computeIfAbsent(clusterName, WlsClusterConfig::create); + WlsClusterConfig wlsClusterConfig = wlsClusterConfigs.computeIfAbsent(clusterName, WlsClusterConfig::new); wlsClusterConfig.addServerConfig(wlsServerConfig); } } } - return this; - } - - private String getClusterNameForServer(Map serverMap) { - // serverMap contains a "cluster" entry from the REST call which is in the form: "cluster": ["clusters", "DockerCluster"] - List clusterList = (List) serverMap.get("cluster"); - if (clusterList != null) { - for (Object value : clusterList) { - // the first entry that is not "clusters" is assumed to be the cluster name - if (!"clusters".equals(value)) { - return (String) value; - } - } - } - return null; + return new WlsDomainConfig(name, wlsClusterConfigs, wlsServerConfigs, wlsServerTemplates); } public static String getRetrieveServersSearchUrl() { @@ -116,23 +184,20 @@ public static String getRetrieveServersSearchUrl() { } public static String getRetrieveServersSearchPayload() { - return "{ fields: [], " + + return "{ fields: [ " + getSearchFields() + " ], " + " links: [], " + " children: { " + - " servers: { " + - " fields: [ " + WlsServerConfig.getSearchFields() + " ], " + - " links: [], " + - " children: { " + - " networkAccessPoints: { " + - " fields: [ " + NetworkAccessPoint.getSearchFields() + " ], " + - " links: [] " + - " } " + - " } " + - " } " + + " servers: { " + WlsServerConfig.getSearchPayload() + " }, " + + " serverTemplates: { " + WlsServerConfig.getSearchPayload() + " }, " + + " clusters: { " + WlsClusterConfig.getSearchPayload() + " } " + " } " + "}"; } + private static String getSearchFields() { + return "'name' "; + } + /** * Parse the json string containing WLS configuration and return a list containing a map of (server * attribute name, attribute value). @@ -142,14 +207,25 @@ public static String getRetrieveServersSearchPayload() { * (server attribute name, attribute value) */ @SuppressWarnings("unchecked") - private List> parseJson(String jsonString) { + private static ParsedJson parseJson(String jsonString) { ObjectMapper mapper = new ObjectMapper(); try { + ParsedJson parsedJson = new ParsedJson(); Map result = mapper.readValue(jsonString, Map.class); + parsedJson.domainName = (String) result.get("name"); Map servers = (Map) result.get("servers"); if (servers != null) { - return (List>) servers.get("items"); + parsedJson.servers = (List>) servers.get("items"); + } + Map serverTemplates = (Map) result.get("serverTemplates"); + if (serverTemplates != null) { + parsedJson.serverTemplates = (List>) serverTemplates.get("items"); + } + Map clusters = (Map) result.get("clusters"); + if (clusters != null) { + parsedJson.clusters = (List>) clusters.get("items"); } + return parsedJson; } catch (Exception e) { LOGGER.warning(MessageKeys.JSON_PARSING_FAILED, jsonString, e.getMessage()); } @@ -201,10 +277,24 @@ public boolean updateDomainSpecAsNeeded(DomainSpec domainSpec) { return updated; } + /** + * Object used by the {@link #parseJson(String)} method to return multiple parsed objects + */ + static class ParsedJson { + String domainName; + List> servers; + List> serverTemplates; + List> clusters; + } + @Override public String toString() { return "WlsDomainConfig{" + - "wlsClusterConfigs=" + wlsClusterConfigs + - '}'; + "wlsClusterConfigs=" + wlsClusterConfigs + + ", wlsServerConfigs=" + wlsServerConfigs + + ", wlsServerTemplates=" + wlsServerTemplates + + ", name='" + name + '\'' + + '}'; } + } diff --git a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDynamicServerConfig.java b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDynamicServerConfig.java new file mode 100644 index 00000000000..3ed13000254 --- /dev/null +++ b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDynamicServerConfig.java @@ -0,0 +1,105 @@ +// Copyright 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.wlsconfig; + +import java.util.ArrayList; +import java.util.List; + +/** + * Contains configuration of a WLS server that belongs to a dynamic cluster + *

+ */ +public class WlsDynamicServerConfig extends WlsServerConfig { + + // default listen ports per WebLogic DynamicServersMBean + final static int DEFAULT_LISTEN_PORT_RANGE_BASE = 7100; + final static int DEFAULT_SSL_LISTEN_PORT_RANGE_BASE = 8100; + final static int DEFAULT_NAP_LISTEN_PORT_RANGE_BASE = 9100; + + /** + * Create a dynamic server config using server template and index number of this server + * + * @param name Name of the server + * @param index index of this server within the cluster, for example, the index of dserver-2 would be 2 + * @param clusterName name of the WLS cluster that this server belongs to + * @param domainName name of the WLS domain that this server belongs to + * @param calculatedListenPorts whether listen ports are calculated according to configuration in the dynamic cluster + * @param serverTemplate server template used for servers in the dynamic cluster + * + * @return a dynamic server configuration object containing configuration of this dynamic server + */ + static WlsDynamicServerConfig create(String name, int index, + String clusterName, String domainName, + boolean calculatedListenPorts, + WlsServerConfig serverTemplate) { + Integer listenPort = serverTemplate.getListenPort(); + Integer sslListenPort = serverTemplate.getSslListenPort(); + List networkAccessPoints = new ArrayList<>(); + if (serverTemplate.getNetworkAccessPoints() != null) { + for (NetworkAccessPoint networkAccessPoint: serverTemplate.getNetworkAccessPoints()) { + Integer networkAccessPointListenPort = networkAccessPoint.getListenPort(); + if (calculatedListenPorts) { + networkAccessPointListenPort = networkAccessPointListenPort == null? + (DEFAULT_NAP_LISTEN_PORT_RANGE_BASE + index): networkAccessPointListenPort + index; + } + networkAccessPoints.add(new NetworkAccessPoint(networkAccessPoint.getName(), networkAccessPoint.getProtocol(), + networkAccessPointListenPort, networkAccessPoint.getPublicPort())); + } + } + // calculate listen ports if configured to do so + if (calculatedListenPorts) { + listenPort = (listenPort == null) ? (DEFAULT_LISTEN_PORT_RANGE_BASE + index): (listenPort + index); + sslListenPort = (sslListenPort == null) ? (DEFAULT_SSL_LISTEN_PORT_RANGE_BASE + index): (sslListenPort + index); + } + MacroSubstitutor macroSubstitutor = new MacroSubstitutor(index, name, clusterName, domainName, + serverTemplate.getMachineName()); + return new WlsDynamicServerConfig(name, listenPort, + macroSubstitutor.substituteMacro(serverTemplate.getListenAddress()), + sslListenPort, + serverTemplate.isSslPortEnabled(), + macroSubstitutor.substituteMacro(serverTemplate.getMachineName()), + networkAccessPoints); + } + + /** + * private constructor. Use {@link #create(String, int, String, String, boolean, WlsServerConfig)} for creating + * an instance of WlsDynamicServerConfig instead. + * + * @param name Name of the dynamic server + * @param listenPort list port of the dynamic server + * @param listenAddress listen address of the dynamic server + * @param sslListenPort SSL listen port of the dynamic server + * @param sslPortEnabled boolean indicating whether SSL listen port is enabled + * @param machineName machine name of the dynamic server + * @param networkAccessPoints network access points or channels configured for this dynamic server + */ + private WlsDynamicServerConfig(String name, Integer listenPort, String listenAddress, + Integer sslListenPort, boolean sslPortEnabled, + String machineName, List networkAccessPoints) { + super(name, listenPort, listenAddress, sslListenPort, sslPortEnabled, machineName, networkAccessPoints); + } + + /** + * Whether this server is a dynamic server, ie, not statically configured + * @return True if this server is a dynamic server, false if this server is configured statically + */ + @Override + public boolean isDynamicServer() { + return true; + } + + @Override + public String toString() { + return "WlsDynamicServerConfig{" + + "name='" + name + '\'' + + ", listenPort=" + listenPort + + ", listenAddress='" + listenAddress + '\'' + + ", sslListenPort=" + sslListenPort + + ", sslPortEnabled=" + sslPortEnabled + + ", machineName='" + machineName + '\'' + + ", networkAccessPoints=" + networkAccessPoints + + '}'; + } + +} diff --git a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDynamicServersConfig.java b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDynamicServersConfig.java new file mode 100644 index 00000000000..ac9565a2212 --- /dev/null +++ b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDynamicServersConfig.java @@ -0,0 +1,200 @@ +// Copyright 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.wlsconfig; + +import oracle.kubernetes.operator.logging.LoggingFacade; +import oracle.kubernetes.operator.logging.LoggingFactory; +import oracle.kubernetes.operator.logging.MessageKeys; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Contains values from a WLS dynamic servers configuration, which configures a WLS dynamic cluster + *

+ */ +public class WlsDynamicServersConfig { + + private static final LoggingFacade LOGGER = LoggingFactory.getLogger("Operator", "Operator"); + + Integer dynamicClusterSize; + Integer maxDynamicClusterSize; + final String serverNamePrefix; + final boolean calculatedListenPorts; + final WlsServerConfig serverTemplate; + List serverConfigs; + + /** + * + * Creates a WlsDynamicServersConfig object using an "dynamicServers" item parsed from JSON result from WLS REST call + * + * @param dynamicServerConfig Map containing "dynamicServers" item parsed from JSON result from WLS REST call + * @param serverTemplates Map containing all server templates configuration read from the WLS domain + * @param clusterName Name of the WLS cluster that this dynamic servers configuration belongs to + * @param domainName Name of the WLS domain that this WLS cluster belongs to + * + * @return A new WlsDynamicServersConfig object created based on the JSON result + */ + static WlsDynamicServersConfig create(Map dynamicServerConfig, Map serverTemplates, + String clusterName, String domainName) { + Integer dynamicClusterSize = null; + Integer maxDynamicClusterSize = null; + String serverNamePrefix = null; + boolean calculatedListenPorts = false; + WlsServerConfig serverTemplate = null; + List serverConfigs = null; + if (dynamicServerConfig != null) { + dynamicClusterSize = (Integer) dynamicServerConfig.get("dynamicClusterSize"); + maxDynamicClusterSize = (Integer) dynamicServerConfig.get("maxDynamicClusterSize"); + serverNamePrefix = (String) dynamicServerConfig.get("serverNamePrefix"); + calculatedListenPorts = (boolean) dynamicServerConfig.get("calculatedListenPorts"); + String serverTemplateName = getServerTemplateNameFromConfig(dynamicServerConfig); + + if (serverTemplateName != null) { + serverTemplate = serverTemplates.get(serverTemplateName); + if (serverTemplate == null) { + LOGGER.warning(MessageKeys.WLS_SERVER_TEMPLATE_NOT_FOUND, serverTemplateName, clusterName); + } else { + serverConfigs = createServerConfigsFromTemplate((List) dynamicServerConfig.get("dynamicServerNames"), + serverTemplate, clusterName, domainName, calculatedListenPorts); + } + } + } + return new WlsDynamicServersConfig(dynamicClusterSize, maxDynamicClusterSize, serverNamePrefix, + calculatedListenPorts, serverTemplate, serverConfigs); + } + + /** + * Constructor + * + * @param dynamicClusterSize current size of the dynamic cluster + * @param maxDynamicClusterSize maximum size of the dynamic cluster + * @param serverNamePrefix prefix for names of servers in this dynamic cluster + * @param calculatedListenPorts whether listen ports are fixed or calculated based on server index + * @param serverTemplate template of servers in the dynamic cluster + * @param serverConfigs List of WlsServerConfig containing configurations of dynamic servers that corresponds to the + * current cluster size + */ + public WlsDynamicServersConfig(Integer dynamicClusterSize, Integer maxDynamicClusterSize, String serverNamePrefix, + boolean calculatedListenPorts, WlsServerConfig serverTemplate, + List serverConfigs) { + this.dynamicClusterSize = dynamicClusterSize; + this.maxDynamicClusterSize = maxDynamicClusterSize; + this.serverNamePrefix = serverNamePrefix; + this.calculatedListenPorts = calculatedListenPorts; + this.serverTemplate = serverTemplate; + this.serverConfigs = serverConfigs; + } + + /** + * Create a list of WlsServerConfig objects for dynamic servers that corresponds to the current cluster size + * + * @param serverNames Names of the servers corresponding to the current cluster size + * @param serverTemplate WlsServerConfig object containing template used for creating dynamic servers in this cluster + * @param clusterName Name of the WLS cluster that this dynamic servers configuration belongs to + * @param domainName Name of the WLS domain that this WLS cluster belongs to + * @param calculatedListenPorts whether listen ports are fixed or calculated based on server index + * + * @return A list of WlsServerConfig objects for dynamic servers + */ + static List createServerConfigsFromTemplate(List serverNames, WlsServerConfig serverTemplate, + String clusterName, String domainName, + boolean calculatedListenPorts) { + List serverConfigs = null; + if (serverNames != null && !serverNames.isEmpty()) { + serverConfigs = new ArrayList<>(serverNames.size()); + int index = 0; + int startingServerIndex = 1; // hard coded to 1 for the time being. This will be configurable in later version of WLS + for (String serverName : serverNames) { + serverConfigs.add( + WlsDynamicServerConfig.create(serverName, index + startingServerIndex, + clusterName, domainName, calculatedListenPorts, serverTemplate)); + index++; + } + } + return serverConfigs; + } + + /** + * Return current size of the dynamic cluster + * @return current size of the dynamic cluster + */ + public Integer getDynamicClusterSize() { + return dynamicClusterSize; + } + + /** + * Return maximum size of the dynamic cluster + * @return maximum size of the dynamic cluster + */ + public Integer getMaxDynamicClusterSize() { + return maxDynamicClusterSize; + } + + /** + * Return list of WlsServerConfig objects containing configurations of WLS dynamic server that can be started under + * the current cluster size + * + * @return A list of WlsServerConfig objects containing configurations of WLS dynamic server that can be started under + * the current cluster size + */ + public List getServerConfigs() { + return serverConfigs; + } + + /** + * Helper method to extract the server template name from the Map obtained from parsing the "dynamicServers" element + * from the REST result. + * + * @param dynamicServerConfig Map containing the "dynamicServers" element from the REST call + * @return Name of the server template associated with this dynamic server configuration + * + */ + private static String getServerTemplateNameFromConfig(Map dynamicServerConfig) { + // dynamicServerConfig contains a "serverTemplates" entry from the REST call which is in the form: "serverTemplate": ["serverTemplates", "my-server-template-name"] + List serverTemplatesList = (List) dynamicServerConfig.get("serverTemplate"); + if (serverTemplatesList != null) { + for (Object value : serverTemplatesList) { + // the first entry that is not "serverTemplates" is assumed to be the server template name + if (!"serverTemplates".equals(value)) { + return (String) value; + } + } + } + return null; + } + + /** + * Return the server template associated with this dynamic servers configuration + * @return The server template associated with this dynamic servers configuration + */ + public WlsServerConfig getServerTemplate() { + return serverTemplate; + } + + /** + * Returns a String containing the fields that we are interested in from the dynamic servers configuration which will + * used in the payload to the REST call to WLS admin server + * + * @return a String containing the fields that we are interested in from the dynamic servers configuration which will + * used in the payload to the REST call to WLS admin server + */ + static String getSearchFields() { + return "'serverTemplate', 'dynamicClusterSize', 'maxDynamicClusterSize', 'serverNamePrefix', 'calculatedListenPorts', 'dynamicServerNames' "; + } + + @Override + public String toString() { + return "WlsDynamicServersConfig{" + + "dynamicClusterSize=" + dynamicClusterSize + + ", maxDynamicClusterSize=" + maxDynamicClusterSize + + ", serverNamePrefix='" + serverNamePrefix + '\'' + + ", calculatedListenPorts=" + calculatedListenPorts + + ", serverTemplate=" + serverTemplate + + ", serverConfigs=" + serverConfigs + + '}'; + } + +} diff --git a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsServerConfig.java b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsServerConfig.java index e7c2b3cf906..fd5c0ddb4a5 100644 --- a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsServerConfig.java +++ b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsServerConfig.java @@ -4,12 +4,11 @@ package oracle.kubernetes.operator.wlsconfig; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; /** - * Contains configuration of a WLS server + * Contains configuration of a statically configured WLS server or a WLS server template *

* Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. */ @@ -17,57 +16,214 @@ public class WlsServerConfig { final String name; final Integer listenPort; final String listenAddress; - final Map networkAccessPoints = new HashMap<>(); + final Integer sslListenPort; + final boolean sslPortEnabled; + final String machineName; + final List networkAccessPoints; + /** + * Return the name of this WLS server + * @return The name of this WLS server + */ public String getName() { return name; } + /** + * Return the configured listen port of this WLS server + * @return The configured listen port of this WLS server + */ public Integer getListenPort() { return listenPort; } + /** + * Return the configured listen address of this WLS server + * @return The configured listen address of this WLS server + */ public String getListenAddress() { return listenAddress; } + /** + * Return the configured SSL listen port of this WLS server + * @return The configured SSL listen port of this WLS server + */ + public Integer getSslListenPort() { + return sslListenPort; + } + + /** + * Return whether the SSL listen port is configured to be enabled or not + * @return True if the SSL listen port should be enabled, false otherwise + */ + public boolean isSslPortEnabled() { + return sslPortEnabled; + } + + /** + * Return the machine name configured for this WLS server + * @return The configured machine name for this WLS server + */ + public String getMachineName() { + return machineName; + } + /** * Returns an array containing all network access points configured in this WLS server * @return An array of NetworkAccessPoint containing configured network access points in this WLS server. If there * are no network access points configured in this server, an empty array is returned. */ public List getNetworkAccessPoints() { - return new ArrayList<>(networkAccessPoints.values()); + return networkAccessPoints; } - WlsServerConfig(Map serverConfigMap) { - this((String) serverConfigMap.get("name"), + /** + * Creates a WLSServerConfig object using an "servers" or "serverTemplates" item parsed from JSON result from + * WLS REST call + * + * @param serverConfigMap A Map containing the parsed "servers" or "serverTemplates" element for + * a WLS server or WLS server template. + * @return A new WlsServerConfig object using the provided configuration from the configuration map + */ + static WlsServerConfig create(Map serverConfigMap) { + // parse the configured network access points or channels + Map networkAccessPointsMap = (Map) serverConfigMap.get("networkAccessPoints"); + List networkAccessPoints = new ArrayList<>(); + if (networkAccessPointsMap != null) { + List> networkAccessPointItems = (List>) networkAccessPointsMap.get("items"); + if (networkAccessPointItems != null && networkAccessPointItems.size() > 0) { + for (Map networkAccessPointConfigMap: networkAccessPointItems) { + NetworkAccessPoint networkAccessPoint = new NetworkAccessPoint(networkAccessPointConfigMap); + networkAccessPoints.add(networkAccessPoint); + } + } + } + // parse the SSL configuration + Map sslMap = (Map) serverConfigMap.get("SSL"); + Integer sslListenPort = (sslMap == null)? null: (Integer) sslMap.get("listenPort"); + boolean sslPortEnabled = (sslMap == null)? false: (boolean) sslMap.get("enabled"); + + return new WlsServerConfig((String) serverConfigMap.get("name"), (Integer) serverConfigMap.get("listenPort"), (String) serverConfigMap.get("listenAddress"), - (Map) serverConfigMap.get("networkAccessPoints")); + sslListenPort, + sslPortEnabled, + getMachineNameFromJsonMap(serverConfigMap), + networkAccessPoints); } - public WlsServerConfig(String name, Integer listenPort, String listenAddress, Map networkAccessPointsMap) { + /** + * Construct a WlsServerConfig object using values provided + * + * @param name Name of the WLS server + * @param listenPort Configured listen port for this WLS server + * @param listenAddress Configured listen address for this WLS server + * @param sslListenPort Configured SSL listen port for this WLS server + * @param sslPortEnabled boolean indicating whether the SSL listen port should be enabled + * @param machineName Configured machine name for this WLS server + * @param networkAccessPoints List of NetworkAccessPoint containing channels configured for this WLS server + */ + public WlsServerConfig(String name, Integer listenPort, String listenAddress, + Integer sslListenPort, boolean sslPortEnabled, + String machineName, + List networkAccessPoints) { this.name = name; this.listenPort = listenPort; this.listenAddress = listenAddress; - if (networkAccessPointsMap != null) { - List> networkAccessPointItems = (List>) networkAccessPointsMap.get("items"); - if (networkAccessPointItems != null && networkAccessPointItems.size() > 0) { - for (Map networkAccessPointConfigMap: networkAccessPointItems) { - NetworkAccessPoint networkAccessPoint = new NetworkAccessPoint(networkAccessPointConfigMap); - this.networkAccessPoints.put(networkAccessPoint.getName(), networkAccessPoint); + this.networkAccessPoints = networkAccessPoints; + this.sslListenPort = sslListenPort; + this.sslPortEnabled = sslPortEnabled; + this.machineName = machineName; + } + + /** + * Helper method to parse the cluster name from an item from the Json "servers" or "serverTemplates" element + * @param serverMap Map containing parsed Json "servers" or "serverTemplates" element + * @return Cluster name contained in the Json element + */ + static String getClusterNameFromJsonMap(Map serverMap) { + // serverMap contains a "cluster" entry from the REST call which is in the form: "cluster": ["clusters", "DockerCluster"] + List clusterList = (List) serverMap.get("cluster"); + if (clusterList != null) { + for (Object value : clusterList) { + // the first entry that is not "clusters" is assumed to be the cluster name + if (!"clusters".equals(value)) { + return (String) value; + } + } + } + return null; + } + + /** + * Helper method to parse the machine name from an item from the Json "servers" or "serverTemplates" element + * @param serverMap Map containing parsed Json "servers" or "serverTemplates" element + * @return Machine name contained in the Json element + */ + static String getMachineNameFromJsonMap(Map serverMap) { + // serverMap contains a "machine" entry from the REST call which is in the form: "machine": ["machines", "domain1-machine1"] + List clusterList = (List) serverMap.get("machine"); + if (clusterList != null) { + for (Object value : clusterList) { + // the first entry that is not "machines" is assumed to be the machine name + if (!"machines".equals(value)) { + return (String) value; } } } + return null; + } + + + /** + * Whether this server is a dynamic server, ie, not statically configured + * @return True if this server is a dynamic server, false if this server is configured statically + */ + public boolean isDynamicServer() { + return false; } /** - * Return the list of configuration attributes to be retrieved from the REST search request to the WLS admin server. - * The value would be used for constructing the REST POST request. + * Return the list of configuration attributes to be retrieved from the REST search request to the + * WLS admin server. The value would be used for constructing the REST POST request. + * + * @return The list of configuration attributes to be retrieved from the REST search request + * to the WLS admin server. The value would be used for constructing the REST POST request. */ - static String getSearchFields() { - return "'name', 'cluster', 'listenPort', 'listenAddress', 'publicPort'"; + static String getSearchPayload() { + return " fields: [ " + getSearchFields() + " ], " + + " links: [], " + + " children: { " + + " SSL: { " + + " fields: [ " + getSSLSearchFields() + " ], " + + " links: [] " + + " }, " + + " networkAccessPoints: { " + + " fields: [ " + NetworkAccessPoint.getSearchFields() + " ], " + + " links: [] " + + " } " + + " } "; + } + + /** + * Return the fields from server or server template WLS configuration that should be retrieved from the WLS REST + * request + * @return A string containing server or server template fields that should be retrieved from the WLS REST + * request, in a format that can be used in the REST request payload + */ + private static String getSearchFields() { + return "'name', 'cluster', 'listenPort', 'listenAddress', 'publicPort', 'machine' "; + } + + /** + * Return the fields from SSL WLS configuration that should be retrieved from the WLS REST + * request + * @return A string containing SSL fields that should be retrieved from the WLS REST request, in a format that can be + * used in the REST request payload + */ + private static String getSSLSearchFields() { + return "'enabled', 'listenPort'"; } @Override @@ -76,7 +232,11 @@ public String toString() { "name='" + name + '\'' + ", listenPort=" + listenPort + ", listenAddress='" + listenAddress + '\'' + + ", sslListenPort=" + sslListenPort + + ", sslPortEnabled=" + sslPortEnabled + + ", machineName='" + machineName + '\'' + ", networkAccessPoints=" + networkAccessPoints + '}'; } + } diff --git a/src/main/resources/Operator.properties b/src/main/resources/Operator.properties index 126b67c408e..b5f3e515b3f 100644 --- a/src/main/resources/Operator.properties +++ b/src/main/resources/Operator.properties @@ -110,5 +110,10 @@ WLSKO-0108=Cycling of servers for Domain with UID {0} in the list {1} now WLSKO-0109=Rolling of servers for Domain with UID {0} in the list {1} now with ready servers {2} WLSKO-0110=Removing Ingress {0} in namespace {1} WLSKO-0111=List Ingress for domain with domainUID {0} in namespace {1} +WLSKO-0112=Failed to update WebLogic dynamic cluster size for cluster {0} due to exception: {1} +WLSKO-0113=Failed to update WebLogic dynamic cluster size for cluster {0} within timeout of {1} milliseconds +WLSKO-0114=Failed to update WebLogic dynamic cluster size for cluster {0}. Cluster is not a dynamic cluster. +WLSKO-0115=Updated cluster size for WebLogic dynamic cluster {0} to {1}. Time taken {2} ms. +WLSKO-0116=Cannot find WebLogic server template with name {0} which is referenced by WebLogic cluster {1}. diff --git a/src/test/java/oracle/kubernetes/operator/ExternalChannelTest.java b/src/test/java/oracle/kubernetes/operator/ExternalChannelTest.java index 434ddcd54c5..b752bdd420b 100644 --- a/src/test/java/oracle/kubernetes/operator/ExternalChannelTest.java +++ b/src/test/java/oracle/kubernetes/operator/ExternalChannelTest.java @@ -26,7 +26,7 @@ public class ExternalChannelTest { private DomainSpec spec = new DomainSpec(); private V1ObjectMeta meta = new V1ObjectMeta(); private List chlist = new ArrayList<>(); - private WlsDomainConfig wlsDomainConfig = new WlsDomainConfig(); + private WlsDomainConfig wlsDomainConfig; private Field protocol; private Field listenPort; @@ -71,7 +71,7 @@ public void setupConstants() throws NoSuchFieldException { spec.setAsName("admin-server"); spec.setExportT3Channels(chlist); - wlsDomainConfig = wlsDomainConfig.load(jsonString); + wlsDomainConfig = WlsDomainConfig.create(jsonString); WlsServerConfig adminServerConfig = wlsDomainConfig.getServerConfig(spec.getAsName()); List naps = adminServerConfig.getNetworkAccessPoints(); diff --git a/src/test/java/oracle/kubernetes/operator/helpers/IngressHelperTest.java b/src/test/java/oracle/kubernetes/operator/helpers/IngressHelperTest.java index 8e9867bc0fb..beeccaf6667 100644 --- a/src/test/java/oracle/kubernetes/operator/helpers/IngressHelperTest.java +++ b/src/test/java/oracle/kubernetes/operator/helpers/IngressHelperTest.java @@ -75,9 +75,9 @@ public void setUp() throws ApiException { info = new DomainPresenceInfo(domain); // Create scan - WlsDomainConfig scan = new WlsDomainConfig(); - WlsServerConfig server1Scan = new WlsServerConfig(server1Name, server1Port, server1Name, null); - WlsServerConfig server2Scan = new WlsServerConfig(server2Name, server2Port, server2Name, null); + WlsDomainConfig scan = new WlsDomainConfig(null); + WlsServerConfig server1Scan = new WlsServerConfig(server1Name, server1Port, server1Name, null, false, null, null); + WlsServerConfig server2Scan = new WlsServerConfig(server2Name, server2Port, server2Name, null, false, null, null); scan.getServerConfigs().put(server1Name, server1Scan); scan.getServerConfigs().put(server2Name, server2Scan); diff --git a/src/test/java/oracle/kubernetes/operator/wlsconfig/MacroSubstitutorTest.java b/src/test/java/oracle/kubernetes/operator/wlsconfig/MacroSubstitutorTest.java new file mode 100644 index 00000000000..675c457f137 --- /dev/null +++ b/src/test/java/oracle/kubernetes/operator/wlsconfig/MacroSubstitutorTest.java @@ -0,0 +1,102 @@ +// Copyright 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.wlsconfig; + +import org.junit.Test; + +import java.util.Properties; + +import static org.junit.Assert.assertEquals; + +public class MacroSubstitutorTest { + + @Test + public void testMacros(){ + final int ID = 123; + final String SERVER = "ms-1"; + final String CLUSTER = "cluster-1"; + final String DOMAIN = "base_domain"; + final String MACHINE = "slc08urp"; + + final MacroSubstitutor MACRO_SUBSTITUTOR = new MacroSubstitutor(ID, SERVER, CLUSTER, DOMAIN, MACHINE); + + assertEquals( + "empty input string should return an empty string", + "", + MACRO_SUBSTITUTOR.substituteMacro("") + ); + + assertEquals( + "null input string should return null", + null, + MACRO_SUBSTITUTOR.substituteMacro(null) + ); + + assertEquals( + "string without macro should remains unchanged", + "abcdefg 1", + MACRO_SUBSTITUTOR.substituteMacro("abcdefg 1") + ); + + assertEquals( + "string with ${id} macro", + "myserver-" + ID, + MACRO_SUBSTITUTOR.substituteMacro("myserver-${id}") + ); + + assertEquals( + "string with ${serverName} macro", + "test-" + SERVER, + MACRO_SUBSTITUTOR.substituteMacro("test-${serverName}") + ); + + assertEquals( + "string with ${clusterName} macro", + "test-" + CLUSTER, + MACRO_SUBSTITUTOR.substituteMacro("test-${clusterName}") + ); + + assertEquals( + "string with ${domainName} macro", + "test-" + DOMAIN, + MACRO_SUBSTITUTOR.substituteMacro("test-${domainName}") + ); + + assertEquals( + "string with only macro", + SERVER, + MACRO_SUBSTITUTOR.substituteMacro("${serverName}") + ); + + assertEquals( + "string with multiple macros", + SERVER + "-" + DOMAIN + "-" + CLUSTER + "-" + ID, + MACRO_SUBSTITUTOR.substituteMacro("${serverName}-${domainName}-${clusterName}-${id}") + ); + + System.setProperty("oracle.macrosubstitutortest", "myEnv Value"); + assertEquals( + "string with system property macro", + "myEnv Value", + MACRO_SUBSTITUTOR.substituteMacro("${oracle.macrosubstitutortest}") + ); + + Properties systemProperties = System.getProperties(); + systemProperties.remove("oracle.macrosubstitutortest"); + assertEquals( + "string with system property macro but system property not set", + "test--1", + MACRO_SUBSTITUTOR.substituteMacro("test-${oracle.macrosubstitutortest}-1") + ); + + assertEquals( + "string without complete macro", + "test${", + MACRO_SUBSTITUTOR.substituteMacro("test${") + ); + + + } + +} diff --git a/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfigTest.java b/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfigTest.java index bab71a3c8c0..0c2c413fa69 100644 --- a/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfigTest.java +++ b/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfigTest.java @@ -1,16 +1,17 @@ -// Copyright 2017, Oracle Corporation and/or its affiliates. All rights reserved. +// Copyright 2017, 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. package oracle.kubernetes.operator.wlsconfig; import oracle.kubernetes.operator.domain.model.oracle.kubernetes.weblogic.domain.v1.ClusterStartup; import org.junit.Test; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; /** * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. @@ -31,6 +32,70 @@ public void verifyClusterSizeIs0IfNoServers() throws Exception { assertEquals(0, wlsClusterConfig.getClusterSize()); } + @Test + public void verifyHasStaticServersIsFalseIfNoServers() throws Exception { + WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1"); + assertFalse(wlsClusterConfig.hasStaticServers()); + } + + @Test + public void verifyHasStaticServersIsTrueIfStaticServers() throws Exception { + WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1"); + wlsClusterConfig.addServerConfig(createWlsServerConfig("ms-0", null, null)); + assertTrue(wlsClusterConfig.hasStaticServers()); + } + + @Test + public void verifyHasDynamicServersIsFalsefNoDynamicServers() throws Exception { + WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1"); + assertFalse(wlsClusterConfig.hasDynamicServers()); + } + + @Test + public void verifyHasDynamicServersIsTrueForDynamicCluster() throws Exception { + WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1", + createDynamicServersConfig(2, 5, "ms-", "clsuter1")); + assertTrue(wlsClusterConfig.hasDynamicServers()); + } + + @Test + public void verifyHasStaticServersIsFalseForDynamicCluster() throws Exception { + WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1", + createDynamicServersConfig(2, 5, "ms-", "clsuter1")); + assertFalse(wlsClusterConfig.hasStaticServers()); + } + + @Test + public void verifyHasDynamicServersIsTrueForMixedCluster() throws Exception { + WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1", + createDynamicServersConfig(2, 5, "ms-", "clsuter1")); + wlsClusterConfig.addServerConfig(createWlsServerConfig("mss-0", 8011, null)); + assertTrue(wlsClusterConfig.hasDynamicServers()); + } + + @Test + public void verifyHasStaticServersIsTrueForMixedCluster() throws Exception { + WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1", + createDynamicServersConfig(2, 5, "ms-", "clsuter1")); + wlsClusterConfig.addServerConfig(createWlsServerConfig("mss-0", 8011, null)); + assertTrue(wlsClusterConfig.hasStaticServers()); + } + + @Test + public void verifyDynamicClusterSizeIsSameAsNumberOfDynamicServers() throws Exception { + WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1", + createDynamicServersConfig(2, 5, "ms-", "clsuter1")); + assertEquals(0, wlsClusterConfig.getClusterSize()); + assertEquals(2, wlsClusterConfig.getDynamicClusterSize()); + assertEquals(5, wlsClusterConfig.getMaxDynamicClusterSize()); + } + + @Test + public void verifyDynamicClusterSizeIsNeg1IfNoDynamicServers() throws Exception { + WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1"); + assertEquals(-1, wlsClusterConfig.getDynamicClusterSize()); + } + @Test public void verifyGetServerConfigsReturnListOfAllServerConfigs() throws Exception { WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1"); @@ -43,6 +108,24 @@ public void verifyGetServerConfigsReturnListOfAllServerConfigs() throws Exceptio assertTrue(containsServer(wlsClusterConfig, "ms-1")); } + @Test + public void verifyGetServerConfigsReturnListOfAllServerConfigsWithDynamicServers() throws Exception { + WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1", + createDynamicServersConfig(3, 5, "ms-", "clsuter1")); + wlsClusterConfig.addServerConfig(createWlsServerConfig("static-0", 8011, null)); + wlsClusterConfig.addServerConfig(createWlsServerConfig("static-1", 8012, null)); + + List wlsServerConfigList = wlsClusterConfig.getServerConfigs(); + assertEquals(5, wlsServerConfigList.size()); + // verify dynamic servers + assertTrue(containsServer(wlsClusterConfig, "ms-1")); + assertTrue(containsServer(wlsClusterConfig, "ms-2")); + assertTrue(containsServer(wlsClusterConfig, "ms-3")); + // verify static servers + assertTrue(containsServer(wlsClusterConfig, "static-0")); + assertTrue(containsServer(wlsClusterConfig, "static-1")); + } + private boolean containsServer(WlsClusterConfig wlsClusterConfig, String serverName) { List serverConfigs = wlsClusterConfig.getServerConfigs(); for (WlsServerConfig serverConfig : serverConfigs) { @@ -82,11 +165,71 @@ public void verifyValidateClusterStartupWarnsIfReplicasTooHigh() throws Exceptio } } + @Test + public void verifyValidateClusterStartupDoesNotWarnIfDynamicCluster() throws Exception { + WlsDynamicServersConfig wlsDynamicServersConfig = createDynamicServersConfig(1, 1, "ms-", "cluster1"); + WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1", wlsDynamicServersConfig); + ClusterStartup cs = new ClusterStartup().clusterName("cluster1").replicas(2); + TestUtil.LogHandlerImpl handler = null; + try { + handler = TestUtil.setupLogHandler(wlsClusterConfig); + wlsClusterConfig.validateClusterStartup(cs); + assertFalse("No message should be logged, but found: " + handler.getAllFormattedMessage(), handler.hasWarningMessageLogged()); + } finally { + TestUtil.removeLogHandler(wlsClusterConfig, handler); + } + } + + @Test + public void verifyGetUpdateDynamicClusterSizeUrlIncludesClusterName() { + WlsDynamicServersConfig wlsDynamicServersConfig = createDynamicServersConfig(1, 1, "ms-", "cluster1"); + WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1", wlsDynamicServersConfig); + assertEquals(wlsClusterConfig.getUpdateDynamicClusterSizeUrl(), "/management/weblogic/latest/edit/clusters/cluster1/dynamicServers"); + } + + @Test + public void verifyGetUpdateDynamicClusterSizePayload1() { + WlsDynamicServersConfig wlsDynamicServersConfig = createDynamicServersConfig(1, 5, "ms-", "cluster1"); + WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1", wlsDynamicServersConfig); + assertEquals(wlsClusterConfig.getUpdateDynamicClusterSizePayload(2), "{ dynamicClusterSize: 2, maxDynamicClusterSize: 5 }"); + } + + @Test + public void verifyGetUpdateDynamicClusterSizePayload2() { + WlsDynamicServersConfig wlsDynamicServersConfig = createDynamicServersConfig(2, 5, "ms-", "cluster1"); + WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1", wlsDynamicServersConfig); + assertEquals(wlsClusterConfig.getUpdateDynamicClusterSizePayload(1), "{ dynamicClusterSize: 1, maxDynamicClusterSize: 5 }"); + } + + @Test + public void verifyGetUpdateDynamicClusterSizePayload3() { + WlsDynamicServersConfig wlsDynamicServersConfig = createDynamicServersConfig(2, 5, "ms-", "cluster1"); + WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1", wlsDynamicServersConfig); + assertEquals(wlsClusterConfig.getUpdateDynamicClusterSizePayload(8), "{ dynamicClusterSize: 8, maxDynamicClusterSize: 8 }"); + } + private WlsServerConfig createWlsServerConfig(String serverName, Integer listenPort, String listenAddress) { Map serverConfigMap = new HashMap<>(); serverConfigMap.put("name", serverName); serverConfigMap.put("listenPort", listenPort); serverConfigMap.put("listenAddress", listenAddress); - return new WlsServerConfig(serverConfigMap); + return WlsServerConfig.create(serverConfigMap); + } + + private WlsDynamicServersConfig createDynamicServersConfig(int clusterSize, int maxClusterSize, + String serverNamePrefix, String clusterName) { + WlsServerConfig serverTemplate = new WlsServerConfig("serverTemplate1", 7001, "host1", + 7002, false, null, null); + List serverNames = new ArrayList<>(); + final int startingServerNameIndex = 1; + for (int i=0; i serverConfigs = WlsDynamicServersConfig.createServerConfigsFromTemplate( + serverNames, serverTemplate, clusterName, "base-domain", false); + + return new WlsDynamicServersConfig(clusterSize, maxClusterSize, serverNamePrefix, + false, serverTemplate, serverConfigs); + } } diff --git a/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetreiverTest.java b/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetreiverTest.java index 7336378f7b8..c85ae59490c 100644 --- a/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetreiverTest.java +++ b/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetreiverTest.java @@ -15,7 +15,7 @@ public class WlsConfigRetreiverTest { private static final LoggingFacade LOGGER = LoggingFactory.getLogger("Operator", "Operator"); - @Test + //@Test public void testWlsConfigRetriever() { final String namespace = "default"; @@ -29,7 +29,7 @@ public void testWlsConfigRetriever() { ClientHolder client = null; try { - WlsDomainConfig wlsDomainConfig = new WlsConfigRetriever(clientHelper, namespace, "wls-admin-service", SECRET_NAME).readConfig(principal); + WlsDomainConfig wlsDomainConfig = new WlsConfigRetriever(clientHelper, namespace, "wls-admin-service", SECRET_NAME).readConfig(); LOGGER.finer("Read config " + wlsDomainConfig); @@ -39,13 +39,13 @@ public void testWlsConfigRetriever() { client = clientHelper.take(); LOGGER.finer("--- trying update REST call ---"); - HttpClient httpClient = HttpClient.createAuthenticatedClientForAdminServer(client, principal, namespace, SECRET_NAME); + HttpClient httpClient = HttpClient.createAuthenticatedClientForAdminServer(client, namespace, SECRET_NAME); String url = "/management/weblogic/latest/edit/servers/ms-3"; String payload = "{listenAddress: 'ms-3.wls-subdomain.default.svc.cluster.local'}"; String result = httpClient.executePostUrlOnServiceClusterIP(url, client, "wls-admin-service", "default", payload); LOGGER.finer("REST call returns: " + result); - LOGGER.finer("Read config again: " + new WlsConfigRetriever(clientHelper, "default", "wls-admin-service", SECRET_NAME).readConfig(principal)); + LOGGER.finer("Read config again: " + new WlsConfigRetriever(clientHelper, "default", "wls-admin-service", SECRET_NAME).readConfig()); } catch (Exception e) { LOGGER.finer("namespace query failed: " + e); e.printStackTrace(); diff --git a/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfigTest.java b/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfigTest.java index 7ccc25c66be..902c3783aba 100644 --- a/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfigTest.java +++ b/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfigTest.java @@ -20,33 +20,116 @@ */ public class WlsDomainConfigTest { + @Test + public void verifyDomainNameLoadedFromJsonString() throws Exception { + WlsDomainConfig wlsDomainConfig = WlsDomainConfig.create(JSON_STRING_1_CLUSTER); + assertEquals("base_domain", wlsDomainConfig.getName()); + } + @Test public void verifyServersLoadedFromJsonString() throws Exception { - WlsDomainConfig wlsDomainConfig = WlsDomainConfig.create().load(JSON_STRING_1_CLUSTER); + WlsDomainConfig wlsDomainConfig = WlsDomainConfig.create(JSON_STRING_1_CLUSTER); Map wlsClusterConfigList = wlsDomainConfig.getClusterConfigs(); assertEquals(1, wlsClusterConfigList.size()); WlsClusterConfig wlsClusterConfig = wlsDomainConfig.getClusterConfig("DockerCluster"); assertEquals(5, wlsClusterConfig.getClusterSize()); for (WlsServerConfig wlsServerConfig : wlsClusterConfig.getServerConfigs()) { - assertEquals(wlsServerConfig.getName() + ".wls-subdomain.default.svc.cluster.local", wlsServerConfig.getListenAddress()); - assertEquals(new Integer(8011), wlsServerConfig.getListenPort()); + if (!wlsServerConfig.isDynamicServer()) { + assertEquals(wlsServerConfig.getName() + ".wls-subdomain.default.svc.cluster.local", wlsServerConfig.getListenAddress()); + assertEquals(new Integer(8011), wlsServerConfig.getListenPort()); + } } assertEquals(6, wlsDomainConfig.getServerConfigs().size()); assertEquals("AdminServer", wlsDomainConfig.getServerConfig("AdminServer").getName()); } + @Test + public void verifyDynamicServersLoadedFromJsonString() throws Exception { + WlsDomainConfig wlsDomainConfig = WlsDomainConfig.create(JSON_STRING_MIXED_CLUSTER); + WlsClusterConfig wlsClusterConfig = wlsDomainConfig.getClusterConfig("DockerCluster"); + assertEquals(2, wlsClusterConfig.getDynamicClusterSize()); + assertEquals(8, wlsClusterConfig.getMaxDynamicClusterSize()); + + List serverConfigs = wlsClusterConfig.getServerConfigs(); + assertEquals(7, serverConfigs.size()); // 5 static + 2 dynamic servers + + assertTrue(containsServer(wlsClusterConfig, "dynamic-1")); + assertTrue(containsServer(wlsClusterConfig, "dynamic-2")); + assertTrue(containsServer(wlsClusterConfig, "ms-0")); + assertTrue(containsServer(wlsClusterConfig, "ms-1")); + assertTrue(containsServer(wlsClusterConfig, "ms-2")); + assertTrue(containsServer(wlsClusterConfig, "ms-3")); + assertTrue(containsServer(wlsClusterConfig, "ms-4")); + + for (WlsServerConfig wlsServerConfig : wlsClusterConfig.getServerConfigs()) { + if (wlsServerConfig.isDynamicServer()) { + String serverName = wlsServerConfig.getName(); + assertEquals("domain1-" + serverName, wlsServerConfig.getListenAddress()); + assertTrue(wlsServerConfig.isSslPortEnabled()); + if ("dynamic-1".equals(serverName)) { + assertEquals(new Integer(8051), wlsServerConfig.getListenPort()); + assertEquals(new Integer(8151), wlsServerConfig.getSslListenPort()); + + } else { + assertEquals(new Integer(8052), wlsServerConfig.getListenPort()); + assertEquals(new Integer(8152), wlsServerConfig.getSslListenPort()); + } + } + } + + assertEquals(6, wlsDomainConfig.getServerConfigs().size()); // does not include dynamic servers + } + + @Test + public void verifyGetServerConfigsDoesNotIncludeDynamicServers() throws Exception { + WlsDomainConfig wlsDomainConfig = WlsDomainConfig.create(JSON_STRING_MIXED_CLUSTER); + WlsClusterConfig wlsClusterConfig = wlsDomainConfig.getClusterConfig("DockerCluster"); + + assertEquals(6, wlsDomainConfig.getServerConfigs().size()); + } + + @Test + public void verifyNetworkAccessPointsInDynamicServersLoadedFromJsonString() throws Exception { + WlsDomainConfig wlsDomainConfig = WlsDomainConfig.create(JSON_STRING_MIXED_CLUSTER); + WlsClusterConfig wlsClusterConfig = wlsDomainConfig.getClusterConfig("DockerCluster"); + assertEquals(2, wlsClusterConfig.getDynamicClusterSize()); + assertEquals(8, wlsClusterConfig.getMaxDynamicClusterSize()); + + for (WlsServerConfig wlsServerConfig : wlsClusterConfig.getServerConfigs()) { + if (wlsServerConfig.isDynamicServer()) { + String serverName = wlsServerConfig.getName(); + assertTrue(containsNetworkAccessPoint(wlsServerConfig, "DChannel-0")); + assertTrue(containsNetworkAccessPoint(wlsServerConfig, "DChannel-1")); + List networkAccessPoints = wlsServerConfig.getNetworkAccessPoints(); + for (NetworkAccessPoint networkAccessPoint: networkAccessPoints) { + String expectedProtocol = null; + Integer expectedListenPort = null; + if ("DChannel-0".equals(networkAccessPoint.getName())) { + expectedProtocol = "t3"; + expectedListenPort = "dynamic-1".equals(serverName)? 9011: 9012; + } else if ("DChannel-1".equals(networkAccessPoint.getName())) { + expectedProtocol = "t3s"; + expectedListenPort = "dynamic-1".equals(serverName)? 9021: 9022; + } + assertEquals("protocol for " + networkAccessPoint.getName() + " not loaded properly", expectedProtocol, networkAccessPoint.getProtocol()); + assertEquals("listen port for " + networkAccessPoint.getName() + " not loaded properly", expectedListenPort, networkAccessPoint.getListenPort()); + } + } + } + } + @Test public void verifyServerWithNoChannelLoadedFromJsonString() throws Exception { - WlsDomainConfig wlsDomainConfig = WlsDomainConfig.create().load(JSON_STRING_1_CLUSTER); + WlsDomainConfig wlsDomainConfig = WlsDomainConfig.create(JSON_STRING_1_CLUSTER); WlsServerConfig serverConfig = wlsDomainConfig.getServerConfig("ms-1"); assertEquals(0, serverConfig.getNetworkAccessPoints().size()); } @Test - public void verifyServerChannelsLoadedFromJsonString() throws Exception { - WlsDomainConfig wlsDomainConfig = WlsDomainConfig.create().load(JSON_STRING_1_CLUSTER); + public void verifyNetworkAccessPointsLoadedFromJsonString() throws Exception { + WlsDomainConfig wlsDomainConfig = WlsDomainConfig.create(JSON_STRING_1_CLUSTER); WlsServerConfig serverConfig = wlsDomainConfig.getServerConfig("ms-0"); assertEquals(3, serverConfig.getNetworkAccessPoints().size()); @@ -71,9 +154,23 @@ public void verifyServerChannelsLoadedFromJsonString() throws Exception { assertEquals("listen port for " + networkAccessPoint.getName() + " not loaded properly", expectedListenPort, networkAccessPoint.getListenPort()); } } + + @Test + public void verifySSLConfigsLoadedFromJsonString() throws Exception { + WlsDomainConfig wlsDomainConfig = WlsDomainConfig.create(JSON_STRING_1_CLUSTER); + + WlsServerConfig serverConfig = wlsDomainConfig.getServerConfig("ms-0"); + assertEquals(new Integer(8101), serverConfig.getSslListenPort()); + assertTrue(serverConfig.isSslPortEnabled()); + + serverConfig = wlsDomainConfig.getServerConfig("ms-1"); + assertNull(serverConfig.getSslListenPort()); + assertFalse(serverConfig.isSslPortEnabled()); + } + @Test public void verifyMultipleClustersLoadedFromJsonString() throws Exception { - WlsDomainConfig wlsDomainConfig = WlsDomainConfig.create().load(JSON_STRING_2_CLUSTERS); + WlsDomainConfig wlsDomainConfig = WlsDomainConfig.create(JSON_STRING_2_CLUSTERS); Map wlsClusterConfigList = wlsDomainConfig.getClusterConfigs(); assertEquals(2, wlsClusterConfigList.size()); @@ -91,7 +188,7 @@ public void verifyMultipleClustersLoadedFromJsonString() throws Exception { @Test public void verifyGetClusterConfigsDoesNotReturnNull() throws Exception { - WlsDomainConfig wlsDomainConfig = WlsDomainConfig.create(); + WlsDomainConfig wlsDomainConfig = new WlsDomainConfig(null); WlsClusterConfig wlsClusterConfig = wlsDomainConfig.getClusterConfig("DockerCluster"); assertNotNull(wlsClusterConfig); assertEquals(0, wlsClusterConfig.getClusterSize()); @@ -100,13 +197,13 @@ public void verifyGetClusterConfigsDoesNotReturnNull() throws Exception { @Test public void verifyGetServerConfigsReturnNullIfNotFound() throws Exception { - WlsDomainConfig wlsDomainConfig = WlsDomainConfig.create(); + WlsDomainConfig wlsDomainConfig = new WlsDomainConfig(null); assertNull(wlsDomainConfig.getServerConfig("noSuchServer")); } @Test public void verifyUpdateDomainSpecWarnsIfNoServersInClusterStartupCluster() throws Exception { - WlsDomainConfig wlsDomainConfig = new WlsDomainConfig(); + WlsDomainConfig wlsDomainConfig = new WlsDomainConfig(null); DomainSpec domainSpec = new DomainSpec().addClusterStartupItem(new ClusterStartup().clusterName("noSuchCluster")); TestUtil.LogHandlerImpl handler = null; WlsClusterConfig wlsClusterConfig = wlsDomainConfig.getClusterConfig("noSuchCluster"); @@ -121,7 +218,7 @@ public void verifyUpdateDomainSpecWarnsIfNoServersInClusterStartupCluster() thro @Test public void verifyUpdateDomainSpecWarnsIfReplicasTooLarge() throws Exception { - WlsDomainConfig wlsDomainConfig = WlsDomainConfig.create().load(JSON_STRING_1_CLUSTER); + WlsDomainConfig wlsDomainConfig = WlsDomainConfig.create(JSON_STRING_1_CLUSTER); DomainSpec domainSpec = new DomainSpec().addClusterStartupItem(new ClusterStartup().clusterName("DockerCluster")).replicas(10); TestUtil.LogHandlerImpl handler = null; WlsClusterConfig wlsClusterConfig = wlsDomainConfig.getClusterConfig("DockerCluster"); @@ -136,7 +233,7 @@ public void verifyUpdateDomainSpecWarnsIfReplicasTooLarge() throws Exception { @Test public void verifyUpdateDomainSpecInfoIfReplicasAndZeroClusters() throws Exception { - WlsDomainConfig wlsDomainConfig = WlsDomainConfig.create(); + WlsDomainConfig wlsDomainConfig = new WlsDomainConfig(null); DomainSpec domainSpec = new DomainSpec().addClusterStartupItem(new ClusterStartup().clusterName("DockerCluster")).replicas(10); TestUtil.LogHandlerImpl handler = null; try { @@ -150,7 +247,7 @@ public void verifyUpdateDomainSpecInfoIfReplicasAndZeroClusters() throws Excepti @Test public void verifyUpdateDomainSpecInfoIfReplicasAndTwoClusters() throws Exception { - WlsDomainConfig wlsDomainConfig = WlsDomainConfig.create().load(JSON_STRING_2_CLUSTERS); + WlsDomainConfig wlsDomainConfig = WlsDomainConfig.create(JSON_STRING_2_CLUSTERS); DomainSpec domainSpec = new DomainSpec().addClusterStartupItem(new ClusterStartup().clusterName("DockerCluster")).replicas(10); TestUtil.LogHandlerImpl handler = null; try { @@ -164,7 +261,7 @@ public void verifyUpdateDomainSpecInfoIfReplicasAndTwoClusters() throws Exceptio @Test public void verifyUpdateDomainSpecReplicasNotValidatedWithMoreThan1Clusters() throws Exception { - WlsDomainConfig wlsDomainConfig = WlsDomainConfig.create().load(JSON_STRING_2_CLUSTERS); + WlsDomainConfig wlsDomainConfig = WlsDomainConfig.create(JSON_STRING_2_CLUSTERS); DomainSpec domainSpec = new DomainSpec().addClusterStartupItem(new ClusterStartup().clusterName("DockerCluster")).replicas(10); TestUtil.LogHandlerImpl handler = null; WlsClusterConfig wlsClusterConfig = wlsDomainConfig.getClusterConfig("DockerCluster"); @@ -179,7 +276,7 @@ public void verifyUpdateDomainSpecReplicasNotValidatedWithMoreThan1Clusters() th @Test public void verifyUpdateDomainSpecNoWarningIfReplicasOK() throws Exception { - WlsDomainConfig wlsDomainConfig = WlsDomainConfig.create().load(JSON_STRING_1_CLUSTER); + WlsDomainConfig wlsDomainConfig = WlsDomainConfig.create(JSON_STRING_1_CLUSTER); DomainSpec domainSpec = new DomainSpec().addClusterStartupItem(new ClusterStartup().clusterName("DockerCluster")).replicas(5); TestUtil.LogHandlerImpl handler = null; WlsClusterConfig wlsClusterConfig = wlsDomainConfig.getClusterConfig("DockerCluster"); @@ -194,7 +291,7 @@ public void verifyUpdateDomainSpecNoWarningIfReplicasOK() throws Exception { @Test public void verifyUpdateDomainSpecWarnsIfClusterStatupReplicasTooLarge() throws Exception { - WlsDomainConfig wlsDomainConfig = WlsDomainConfig.create().load(JSON_STRING_2_CLUSTERS); + WlsDomainConfig wlsDomainConfig = WlsDomainConfig.create(JSON_STRING_2_CLUSTERS); DomainSpec domainSpec = new DomainSpec().addClusterStartupItem(new ClusterStartup().clusterName("DockerCluster2").replicas(3)).replicas(5); TestUtil.LogHandlerImpl handler = null; WlsClusterConfig wlsClusterConfig = wlsDomainConfig.getClusterConfig("DockerCluster2"); @@ -209,7 +306,7 @@ public void verifyUpdateDomainSpecWarnsIfClusterStatupReplicasTooLarge() throws @Test public void verifyUpdateDomainSpecWarnsIfClusterStatupReplicasTooLarge_2clusters() throws Exception { - WlsDomainConfig wlsDomainConfig = WlsDomainConfig.create().load(JSON_STRING_2_CLUSTERS); + WlsDomainConfig wlsDomainConfig = WlsDomainConfig.create(JSON_STRING_2_CLUSTERS); ClusterStartup dockerCluster = new ClusterStartup().clusterName("DockerCluster").replicas(10); ClusterStartup dockerCluster2 = new ClusterStartup().clusterName("DockerCluster2").replicas(10); DomainSpec domainSpec = new DomainSpec().addClusterStartupItem(dockerCluster).addClusterStartupItem(dockerCluster2); @@ -227,7 +324,7 @@ public void verifyUpdateDomainSpecWarnsIfClusterStatupReplicasTooLarge_2clusters @Test public void verifyUpdateDomainSpecNoWarningIfClusterStatupReplicasOK() throws Exception { - WlsDomainConfig wlsDomainConfig = WlsDomainConfig.create().load(JSON_STRING_2_CLUSTERS); + WlsDomainConfig wlsDomainConfig = WlsDomainConfig.create(JSON_STRING_2_CLUSTERS); DomainSpec domainSpec = new DomainSpec().addClusterStartupItem(new ClusterStartup().clusterName("DockerCluster2").replicas(2)).replicas(5); TestUtil.LogHandlerImpl handler = null; WlsClusterConfig wlsClusterConfig = wlsDomainConfig.getClusterConfig("DockerCluster2"); @@ -240,6 +337,21 @@ public void verifyUpdateDomainSpecNoWarningIfClusterStatupReplicasOK() throws Ex } } + @Test + public void verifyUpdateDomainSpecNoWarningIfClusterStatupOnDynamicCluster() throws Exception { + WlsDomainConfig wlsDomainConfig = WlsDomainConfig.create(JSON_STRING_MIXED_CLUSTER); + DomainSpec domainSpec = new DomainSpec().addClusterStartupItem(new ClusterStartup().clusterName("DockerCluster").replicas(10)).replicas(10); + TestUtil.LogHandlerImpl handler = null; + WlsClusterConfig wlsClusterConfig = wlsDomainConfig.getClusterConfig("DockerCluster"); + try { + handler = TestUtil.setupLogHandler(wlsClusterConfig); + wlsDomainConfig.updateDomainSpecAsNeeded(domainSpec); + assertFalse(handler.hasWarningMessageLogged()); + } finally { + TestUtil.removeLogHandler(wlsClusterConfig, handler); + } + } + private boolean containsServer(WlsClusterConfig wlsClusterConfig, String serverName) { List serverConfigs = wlsClusterConfig.getServerConfigs(); for (WlsServerConfig serverConfig : serverConfigs) { @@ -260,7 +372,8 @@ private boolean containsNetworkAccessPoint(WlsServerConfig wlsServerConfig, Stri return false; } - final String JSON_STRING_1_CLUSTER = "{\"servers\": {\"items\": [\n" + + final String JSON_STRING_MIXED_CLUSTER = "{ \"name\": \"base_domain\",\n " + + "\"servers\": {\"items\": [\n" + " {\n" + " \"listenAddress\": \"\",\n" + " \"name\": \"AdminServer\",\n" + @@ -292,7 +405,140 @@ private boolean containsNetworkAccessPoint(WlsServerConfig wlsServerConfig, Stri " \"name\": \"Channel-2\",\n" + " \"listenPort\": 8014\n" + " }\n" + - " ]}\n" + + " ]},\n" + + " \"SSL\": {\n" + + " \"enabled\": true,\n" + + " \"listenPort\": 8101\n" + + " }\n" + + " },\n" + + " {\n" + + " \"listenAddress\": \"ms-1.wls-subdomain.default.svc.cluster.local\",\n" + + " \"name\": \"ms-1\",\n" + + " \"listenPort\": 8011,\n" + + " \"cluster\": [\n" + + " \"clusters\",\n" + + " \"DockerCluster\"\n" + + " ],\n" + + " \"networkAccessPoints\": {\"items\": []}\n" + + " },\n" + + " {\n" + + " \"listenAddress\": \"ms-2.wls-subdomain.default.svc.cluster.local\",\n" + + " \"name\": \"ms-2\",\n" + + " \"listenPort\": 8011,\n" + + " \"cluster\": [\n" + + " \"clusters\",\n" + + " \"DockerCluster\"\n" + + " ],\n" + + " \"networkAccessPoints\": {\"items\": []}\n" + + " },\n" + + " {\n" + + " \"listenAddress\": \"ms-3.wls-subdomain.default.svc.cluster.local\",\n" + + " \"name\": \"ms-3\",\n" + + " \"listenPort\": 8011,\n" + + " \"cluster\": [\n" + + " \"clusters\",\n" + + " \"DockerCluster\"\n" + + " ],\n" + + " \"networkAccessPoints\": {\"items\": []}\n" + + " },\n" + + " {\n" + + " \"listenAddress\": \"ms-4.wls-subdomain.default.svc.cluster.local\",\n" + + " \"name\": \"ms-4\",\n" + + " \"listenPort\": 8011,\n" + + " \"cluster\": [\n" + + " \"clusters\",\n" + + " \"DockerCluster\"\n" + + " ],\n" + + " \"networkAccessPoints\": {\"items\": []}\n" + + " }\n" + + " ]},\n" + + "\"serverTemplates\": {\"items\": [\n" + + " {\n" + + " \"listenAddress\": \"domain1-${serverName}\",\n" + + " \"name\": \"server-template-1\",\n" + + " \"listenPort\": 8050,\n" + + " \"cluster\": [\n" + + " \"clusters\",\n" + + " \"DockerCluster\"\n" + + " ],\n" + + " \"networkAccessPoints\": {\"items\": [\n" + + " {\n" + + " \"protocol\": \"t3\",\n" + + " \"name\": \"DChannel-0\",\n" + + " \"listenPort\": 9010\n" + + " },\n" + + " {\n" + + " \"protocol\": \"t3s\",\n" + + " \"name\": \"DChannel-1\",\n" + + " \"listenPort\": 9020\n" + + " }\n" + + " ]},\n" + + " \"SSL\": {\n" + + " \"enabled\": true,\n" + + " \"listenPort\": 8150\n" + + " }\n" + + " }\n" + + " ]},\n" + + " \"clusters\": {\"items\": [\n" + + " {\n" + + " \"name\": \"DockerCluster\",\n" + + " \"dynamicServers\": {\n" + + " \"dynamicClusterSize\": 2,\n" + + " \"maxDynamicClusterSize\": 8,\n" + + " \"serverNamePrefix\": \"dynamic-\",\n" + + " \"dynamicServerNames\": [\n" + + " \"dynamic-1\",\n" + + " \"dynamic-2\"\n" + + " ],\n" + + " \"calculatedListenPorts\": true,\n" + + " \"serverTemplate\": [\n" + + " \"serverTemplates\",\n" + + " \"server-template-1\"\n" + + " ]\n" + + " }\n" + + " }" + + " ]}\n" + + "}"; + + + final String JSON_STRING_1_CLUSTER = "{ \"name\": \"base_domain\",\n " + + "\"servers\": {\"items\": [\n" + + " {\n" + + " \"listenAddress\": \"\",\n" + + " \"name\": \"AdminServer\",\n" + + " \"listenPort\": 8001,\n" + + " \"cluster\": null,\n" + + " \"networkAccessPoints\": {\"items\": []}\n" + + " },\n" + + " {\n" + + " \"listenAddress\": \"ms-0.wls-subdomain.default.svc.cluster.local\",\n" + + " \"name\": \"ms-0\",\n" + + " \"listenPort\": 8011,\n" + + " \"cluster\": [\n" + + " \"clusters\",\n" + + " \"DockerCluster\"\n" + + " ],\n" + + " \"networkAccessPoints\": {\"items\": [\n" + + " {\n" + + " \"protocol\": \"t3\",\n" + + " \"name\": \"Channel-0\",\n" + + " \"listenPort\": 8012\n" + + " },\n" + + " {\n" + + " \"protocol\": \"t3\",\n" + + " \"name\": \"Channel-1\",\n" + + " \"listenPort\": 8013\n" + + " },\n" + + " {\n" + + " \"protocol\": \"t3s\",\n" + + " \"name\": \"Channel-2\",\n" + + " \"listenPort\": 8014\n" + + " }\n" + + " ]},\n" + + " \"SSL\": {\n" + + " \"enabled\": true,\n" + + " \"listenPort\": 8101\n" + + " }\n" + " },\n" + " {\n" + " \"listenAddress\": \"ms-1.wls-subdomain.default.svc.cluster.local\",\n" + diff --git a/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsDynamicServerConfigTest.java b/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsDynamicServerConfigTest.java new file mode 100644 index 00000000000..c11916af683 --- /dev/null +++ b/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsDynamicServerConfigTest.java @@ -0,0 +1,95 @@ +// Copyright 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.wlsconfig; + +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +public class WlsDynamicServerConfigTest { + + @Test + public void testCreateWithFixedPorts() { + NetworkAccessPoint networkAccessPoint = new NetworkAccessPoint("Channel1", "t3", + 10000, 30001); + List networkAccessPointList = new ArrayList<>(); + networkAccessPointList.add(networkAccessPoint); + WlsServerConfig template = new WlsServerConfig("template1", 1000, + null, 2000, + true, null, networkAccessPointList); + + WlsServerConfig wlsServerConfig = WlsDynamicServerConfig.create("server1", 2, + "cluster1", "domain1", false, template); + + assertEquals(new Integer(1000), wlsServerConfig.getListenPort()); + assertEquals(new Integer(2000), wlsServerConfig.getSslListenPort()); + NetworkAccessPoint networkAccessPoint1 = wlsServerConfig.getNetworkAccessPoints().get(0); + assertEquals(new Integer(10000), networkAccessPoint1.getListenPort()); + assertEquals(new Integer(30001), networkAccessPoint1.getPublicPort()); + } + + @Test + public void testCreateWithNullPorts() { + NetworkAccessPoint networkAccessPoint = new NetworkAccessPoint("Channel1", "t3", + null, null); + List networkAccessPointList = new ArrayList<>(); + networkAccessPointList.add(networkAccessPoint); + WlsServerConfig template = new WlsServerConfig("template1", null, + null, null, + true, null, networkAccessPointList); + + WlsServerConfig wlsServerConfig = WlsDynamicServerConfig.create("server1", 2, + "cluster1", "domain1", false, template); + + assertNull(wlsServerConfig.getListenPort()); + assertNull(wlsServerConfig.getSslListenPort()); + NetworkAccessPoint networkAccessPoint1 = wlsServerConfig.getNetworkAccessPoints().get(0); + assertNull(networkAccessPoint1.getListenPort()); + assertNull(networkAccessPoint1.getPublicPort()); + } + + @Test + public void testCreateWithCalculatedPorts() { + NetworkAccessPoint networkAccessPoint = new NetworkAccessPoint("Channel1", "t3", + 10000, 30001); + List networkAccessPointList = new ArrayList<>(); + networkAccessPointList.add(networkAccessPoint); + WlsServerConfig template = new WlsServerConfig("template1", 1000, + null, 2000, + true, null, networkAccessPointList); + + WlsServerConfig wlsServerConfig = WlsDynamicServerConfig.create("server1", 2, + "cluster1", "domain1", true, template); + + assertEquals(new Integer(1002), wlsServerConfig.getListenPort()); + assertEquals(new Integer(2002), wlsServerConfig.getSslListenPort()); + NetworkAccessPoint networkAccessPoint1 = wlsServerConfig.getNetworkAccessPoints().get(0); + assertEquals(new Integer(10002), networkAccessPoint1.getListenPort()); + assertEquals(new Integer(30001), networkAccessPoint1.getPublicPort()); + } + + @Test + public void testCreateWithCalculatedDefaultPorts() { + NetworkAccessPoint networkAccessPoint = new NetworkAccessPoint("Channel1", "t3", + null, null); + List networkAccessPointList = new ArrayList<>(); + networkAccessPointList.add(networkAccessPoint); + WlsServerConfig template = new WlsServerConfig("template1", null, + null, null, + true, null, networkAccessPointList); + + WlsServerConfig wlsServerConfig = WlsDynamicServerConfig.create("server1", 2, + "cluster1", "domain1", true, template); + + assertEquals(new Integer(7102), wlsServerConfig.getListenPort()); + assertEquals(new Integer(8102), wlsServerConfig.getSslListenPort()); + NetworkAccessPoint networkAccessPoint1 = wlsServerConfig.getNetworkAccessPoints().get(0); + assertEquals(new Integer(9102), networkAccessPoint1.getListenPort()); + assertNull(networkAccessPoint1.getPublicPort()); + } +} From 4960470da5ebce09876a4feea3624ec1147aff98 Mon Sep 17 00:00:00 2001 From: Anthony Lai Date: Thu, 15 Feb 2018 09:46:01 -0800 Subject: [PATCH 002/186] adjust dynamic cluster size up if needed by ClusterStartup replicas value --- .../operator/wlsconfig/ConfigUpdate.java | 26 +++++ .../operator/wlsconfig/WlsClusterConfig.java | 81 ++++++++++++- .../wlsconfig/WlsConfigRetriever.java | 108 +++++++++++++---- .../operator/wlsconfig/WlsDomainConfig.java | 24 ++-- src/main/resources/Operator.properties | 2 +- .../wlsconfig/WlsClusterConfigTest.java | 109 +++++++++++++++++- .../wlsconfig/WlsDomainConfigTest.java | 20 ++-- 7 files changed, 319 insertions(+), 51 deletions(-) create mode 100644 src/main/java/oracle/kubernetes/operator/wlsconfig/ConfigUpdate.java diff --git a/src/main/java/oracle/kubernetes/operator/wlsconfig/ConfigUpdate.java b/src/main/java/oracle/kubernetes/operator/wlsconfig/ConfigUpdate.java new file mode 100644 index 00000000000..d2a4e4ce762 --- /dev/null +++ b/src/main/java/oracle/kubernetes/operator/wlsconfig/ConfigUpdate.java @@ -0,0 +1,26 @@ +// Copyright 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. +package oracle.kubernetes.operator.wlsconfig; + +import oracle.kubernetes.operator.work.Step; + +/** + * Each ConfigUpdate contains a suggested WebLogic configuration update that is necessary + * to make the WebLogic configuration to be compatible with the DomainSpec configuration. + */ +public interface ConfigUpdate { + + /** + * Perform the suggested WebLogic configuration update + * @param wlsConfigRetriever The WlsConfigRetriever object to be used for performing the update + * @return true if the update was successful, false otherwise + */ + boolean doUpdate(WlsConfigRetriever wlsConfigRetriever); + + /** + * Create a Step to perform the suggested WebLogic configuration update + * @param next Next Step to be performed after the WebLogic configuration update + * @return Step to perform the suggested WebLogic configuration update + */ + Step createStep(Step next); +} diff --git a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfig.java b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfig.java index 47fc1006571..d1393ffa163 100644 --- a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfig.java +++ b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfig.java @@ -7,6 +7,7 @@ import oracle.kubernetes.operator.logging.LoggingFacade; import oracle.kubernetes.operator.logging.LoggingFactory; import oracle.kubernetes.operator.logging.MessageKeys; +import oracle.kubernetes.operator.work.Step; import java.util.ArrayList; import java.util.List; @@ -154,9 +155,12 @@ public int getMaxDynamicClusterSize() { * It is the responsibility of the caller to persist the changes to ClusterStartup to kubernetes. * * @param clusterStartup The ClusterStartup to be validated against the WLS configuration + * @param suggestedConfigUpdates A List containing suggested WebLogic configuration update to be filled in by this + * method. Optional. * @return true if the DomainSpec has been updated, false otherwise */ - public boolean validateClusterStartup(ClusterStartup clusterStartup) { + public boolean validateClusterStartup(ClusterStartup clusterStartup, + List suggestedConfigUpdates) { LOGGER.entering(); boolean modified = false; @@ -167,7 +171,7 @@ public boolean validateClusterStartup(ClusterStartup clusterStartup) { } // Warns if replicas is larger than the number of servers configured in the cluster - validateReplicas(clusterStartup.getReplicas(), "clusterStartup"); + validateReplicas(clusterStartup.getReplicas(), "clusterStartup", suggestedConfigUpdates); LOGGER.exiting(modified); @@ -176,18 +180,32 @@ public boolean validateClusterStartup(ClusterStartup clusterStartup) { /** - * Validate the configured replicas value in the kubernetes weblogic domain spec against the + * Validate the configured replicas value in the kubernetes WebLogic domain spec against the * configured size of this cluster. Log warning if any inconsistencies are found. * * @param replicas The configured replicas value for this cluster in the kubernetes weblogic domain spec * for this cluster * @param source The name of the section in the domain spec where the replicas is specified, * for logging purposes + * @param suggestedConfigUpdates A List containing suggested WebLogic configuration update to be filled in by this + * method. Optional. */ - public void validateReplicas(Integer replicas, String source) { - if (replicas != null && replicas > getClusterSize() && !hasDynamicServers()) { + public void validateReplicas(Integer replicas, String source, + List suggestedConfigUpdates) { + if (replicas == null) { + return; + } + // log warning if replicas is too large and cluster only contains statically configured servers + if (!hasDynamicServers() && replicas > getClusterSize()) { LOGGER.warning(MessageKeys.REPLICA_MORE_THAN_WLS_SERVERS, source, clusterName, replicas, getClusterSize()); } + // recommend updating WLS dynamic cluster size if replicas value is larger than current + // dynamic cluster size + if (hasDynamicServers() && replicas > getDynamicClusterSize()) { + if (suggestedConfigUpdates != null) { + suggestedConfigUpdates.add(new DynamicClusterSizeConfigUpdate(this, replicas)); + } + } } /** @@ -252,4 +270,57 @@ public String toString() { '}'; } + /** + * Checks the JSON result from the dynamic cluster size update REST request + * + * @param jsonResult The JSON String result from the dynamic server cluster + * size update REST request + * @return true if the result means the update was successful, false otherwise + */ + static boolean checkUpdateDynamicClusterSizeJsonResult(String jsonResult) { + final String EXPECTED_RESULT = "{}"; + + boolean result = false; + if (EXPECTED_RESULT.equals(jsonResult)) { + result = true; + } + return result; + } + + + /** + * ConfigUpdate implementation for updating a dynamic cluster size + */ + static class DynamicClusterSizeConfigUpdate implements ConfigUpdate { + final int desiredClusterSize; + final WlsClusterConfig wlsClusterConfig; + + public DynamicClusterSizeConfigUpdate(WlsClusterConfig wlsClusterConfig, + int desiredClusterSize) { + this.desiredClusterSize = desiredClusterSize; + this.wlsClusterConfig = wlsClusterConfig; + } + + /** + * Update the cluster size of a WebLogic dynamic cluster + * + * @param wlsConfigRetriever The WlsConfigRetriever object to be used for performing the update + * @return true if the update was successful, false otherwise + */ + @Override + public boolean doUpdate(WlsConfigRetriever wlsConfigRetriever) { + return wlsConfigRetriever.updateDynamicClusterSize(wlsClusterConfig, desiredClusterSize); + } + + /** + * Create a Step to update the cluster size of a WebLogic dynamic cluster + * @param next Next Step to be performed after the WebLogic configuration update + * @return Step to update the cluster size of a WebLogic dynamic cluster + */ + + @Override + public Step createStep(Step next) { + return new WlsConfigRetriever.UpdateDynamicClusterStep(wlsClusterConfig, desiredClusterSize, next); + } + } } diff --git a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetriever.java b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetriever.java index 2c0606f486c..2a3eea1031f 100644 --- a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetriever.java +++ b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetriever.java @@ -5,7 +5,6 @@ import oracle.kubernetes.operator.domain.model.oracle.kubernetes.weblogic.domain.v1.Domain; import oracle.kubernetes.operator.domain.model.oracle.kubernetes.weblogic.domain.v1.DomainSpec; -import oracle.kubernetes.operator.helpers.CallBuilder; import oracle.kubernetes.operator.helpers.ClientHelper; import oracle.kubernetes.operator.helpers.ClientHolder; import oracle.kubernetes.operator.helpers.DomainPresenceInfo; @@ -17,6 +16,8 @@ import oracle.kubernetes.operator.work.Packet; import oracle.kubernetes.operator.work.Step; +import java.util.ArrayList; +import java.util.List; import java.util.Random; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -31,6 +32,7 @@ /** * A helper class to retrieve configuration information from WebLogic. + * It also contains method to perform configuration updates to a WebLogic domain. */ public class WlsConfigRetriever { public static final String KEY = "wlsDomainConfig"; @@ -113,15 +115,10 @@ public NextAction apply(Packet packet) { DomainSpec spec = dom.getSpec(); String namespace = meta.getNamespace(); - String domainUID = spec.getDomainUID(); - - String name = CallBuilder.toDNS1123LegalName(domainUID + "-" + spec.getAsName()); - String adminSecretName = spec.getAdminSecret() == null ? null : spec.getAdminSecret().getName(); - String adminServerServiceName = name; Step getClient = HttpClient.createAuthenticatedClientForAdminServer( - namespace, adminSecretName, new WithHttpClientStep(namespace, adminServerServiceName, next)); + namespace, adminSecretName, new WithHttpClientStep(next)); packet.remove(RETRY_COUNT); return doNext(getClient, packet); } catch (Throwable t) { @@ -142,16 +139,76 @@ public NextAction apply(Packet packet) { } } + /** + * Step for updating the cluster size of a WebLogic dynamic cluster + */ + static final class UpdateDynamicClusterStep extends Step { + + final WlsClusterConfig wlsClusterConfig; + final int desiredClusterSize; + + /** + * Constructor + * + * @param wlsClusterConfig The WlsClusterConfig object for the WebLogic dynamic cluster to be updated + * @param desiredClusterSize The desired dynamic cluster size + * @param next The next Step to be performed + */ + public UpdateDynamicClusterStep(WlsClusterConfig wlsClusterConfig, int desiredClusterSize, Step next) { + super(next); + this.wlsClusterConfig = wlsClusterConfig; + this.desiredClusterSize = desiredClusterSize; + } + + /** + * {@inheritDoc} + */ + @Override + public NextAction apply(Packet packet) { + + String clusterName = wlsClusterConfig == null? "null": wlsClusterConfig.getClusterName(); + + if (wlsClusterConfig == null || !wlsClusterConfig.hasDynamicServers()) { + LOGGER.warning(MessageKeys.WLS_UPDATE_CLUSTER_SIZE_INVALID_CLUSTER, clusterName); + } else { + try { + HttpClient httpClient = (HttpClient) packet.get(HttpClient.KEY); + DomainPresenceInfo info = packet.getSPI(DomainPresenceInfo.class); + + long startTime = System.currentTimeMillis(); + + String serviceURL = HttpClient.getServiceURL(info.getAdmin().getService()); + + String jsonResult = httpClient.executePostUrlOnServiceClusterIP( + wlsClusterConfig.getUpdateDynamicClusterSizeUrl(), + serviceURL, wlsClusterConfig.getUpdateDynamicClusterSizePayload(desiredClusterSize)); + + if (wlsClusterConfig.checkUpdateDynamicClusterSizeJsonResult(jsonResult)) { + LOGGER.info(MessageKeys.WLS_CLUSTER_SIZE_UPDATED, clusterName, desiredClusterSize, (System.currentTimeMillis() - startTime)); + } else { + LOGGER.warning(MessageKeys.WLS_UPDATE_CLUSTER_SIZE_FAILED, clusterName, null); + } + } catch (Throwable t) { + LOGGER.warning(MessageKeys.WLS_UPDATE_CLUSTER_SIZE_FAILED, clusterName, t); + } + } + return doNext(packet); + } + } + private static final class WithHttpClientStep extends Step { - private final String namespace; - private final String adminServerServiceName; - public WithHttpClientStep(String namespace, String adminServerServiceName, Step next) { + /** + * Constructor + * @param next The next Step + */ + public WithHttpClientStep(Step next) { super(next); - this.namespace = namespace; - this.adminServerServiceName = adminServerServiceName; } + /** + * {@inheritDoc} + */ @Override public NextAction apply(Packet packet) { try { @@ -167,14 +224,28 @@ public NextAction apply(Packet packet) { wlsDomainConfig = WlsDomainConfig.create(jsonResult); } - // validate domain spec against WLS configuration. Currently this only logs warning messages. - wlsDomainConfig.updateDomainSpecAsNeeded(dom.getSpec()); + // validate domain spec against WLS configuration. + // This logs warning messages as well as returning a list of suggested + // WebLogic configuration updates, but it does not update the DomainSpec. + List suggestedConfigUpdates = new ArrayList<>(); + wlsDomainConfig.validate(dom.getSpec(), suggestedConfigUpdates); info.setScan(wlsDomainConfig); info.setLastScanTime(new DateTime()); LOGGER.info(MessageKeys.WLS_CONFIGURATION_READ, (System.currentTimeMillis() - ((Long) packet.get(START_TIME))), wlsDomainConfig); + // If there are suggested WebLogic configuration update, perform them as the + // next Step, then read the updated WebLogic configuration again after the + // update(s) are performed. + if (!suggestedConfigUpdates.isEmpty()) { + Step nextStep = new WithHttpClientStep(next); // read WebLogic config again after config updates + for(ConfigUpdate suggestedConfigUpdate: suggestedConfigUpdates) { + nextStep = suggestedConfigUpdate.createStep(nextStep); + } + return doNext(nextStep, packet); + } + return doNext(packet); } catch (Throwable t) { LOGGER.warning(MessageKeys.WLS_CONFIGURATION_READ_FAILED, t); @@ -382,17 +453,12 @@ private boolean doUpdateDynamicClusterSize(final WlsClusterConfig wlsClusterConf final int clusterSize) throws Exception { LOGGER.entering(); - final String EXPECTED_RESULT = "{}"; String jsonResult = executePostUrl( wlsClusterConfig.getUpdateDynamicClusterSizeUrl(), wlsClusterConfig.getUpdateDynamicClusterSizePayload(clusterSize)); - boolean result = false; - if (EXPECTED_RESULT.equals(jsonResult)) { - result = true; - } else { - LOGGER.warning(MessageKeys.WLS_UPDATE_CLUSTER_SIZE_FAILED, wlsClusterConfig.getClusterName(), jsonResult); - } + + boolean result = wlsClusterConfig.checkUpdateDynamicClusterSizeJsonResult(jsonResult); LOGGER.exiting(result); return result; } diff --git a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfig.java b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfig.java index 33b1e08a1e5..ea1d39b45b9 100644 --- a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfig.java +++ b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfig.java @@ -15,9 +15,6 @@ import java.util.List; import java.util.Map; -/** - * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. - */ public class WlsDomainConfig { private static final LoggingFacade LOGGER = LoggingFactory.getLogger("Operator", "Operator"); @@ -232,16 +229,20 @@ private static ParsedJson parseJson(String jsonString) { return null; } + public boolean validate(DomainSpec domainSpec) { + return validate(domainSpec, null); + } + /** - * Update the provided k8s domain spec to be consistent with the configuration of the WLS domain. - * The method also logs warning if inconsistent WLS configurations are found that cannot be fixed by updating - * the provided Domain spec. - * It is the responsibility of the caller to persist the changes to DomainSpec to kubernetes. + * Checks the provided k8s domain spec to see if it is consistent with the configuration of the WLS domain. + * The method also logs warning if inconsistent WLS configurations are found. * * @param domainSpec The DomainSpec to be validated against the WLS configuration + * @param suggestedConfigUpdates a List of ConfigUpdate objects containing suggested WebLogic config updates that + * are necessary to make the WebLogic domain consistent with the DomainSpec. Optional. * @return true if the DomainSpec has been updated, false otherwise */ - public boolean updateDomainSpecAsNeeded(DomainSpec domainSpec) { + public boolean validate(DomainSpec domainSpec, List suggestedConfigUpdates) { LOGGER.entering(); @@ -254,18 +255,19 @@ public boolean updateDomainSpecAsNeeded(DomainSpec domainSpec) { String clusterName = clusterStartup.getClusterName(); if (clusterName != null) { WlsClusterConfig wlsClusterConfig = getClusterConfig(clusterName); - updated |= wlsClusterConfig.validateClusterStartup(clusterStartup); + updated |= wlsClusterConfig.validateClusterStartup(clusterStartup, suggestedConfigUpdates); } } } - // validate replicas in DomainSpec if spcified + // validate replicas in DomainSpec if specified if (domainSpec.getReplicas() != null) { Collection clusterConfigs = getClusterConfigs().values(); // WLS domain contains only one cluster if (clusterConfigs != null && clusterConfigs.size() == 1) { for (WlsClusterConfig wlsClusterConfig : clusterConfigs) { - wlsClusterConfig.validateReplicas(domainSpec.getReplicas(), "domainSpec"); + wlsClusterConfig.validateReplicas(domainSpec.getReplicas(), "domainSpec", + suggestedConfigUpdates); } } else { // log info message if replicas is specified but number of WLS clusters in domain is not 1 diff --git a/src/main/resources/Operator.properties b/src/main/resources/Operator.properties index b5f3e515b3f..b7b0674c138 100644 --- a/src/main/resources/Operator.properties +++ b/src/main/resources/Operator.properties @@ -64,7 +64,7 @@ WLSKO-0062=ApiException from TokenReview: {0} WLSKO-0063=ApiException from SubjectAccessReview: {0} WLSKO-0064=SubjectAccessReview: {0} WLSKO-0065=replicas in {0} for cluster {1} is specified with a value of {2} which is larger than the number of configured WLS servers in the cluster: {3} -WLSKO-0066=replicas specified in Domain spec is ignored because there number of configured WLS cluster is not 1. +WLSKO-0066=replicas specified in Domain spec is ignored because the number of configured WLS cluster is not 1. WLSKO-0067=Kubernetes minimum version check failed. Required minimum version is {0}, but found version {1} WLSKO-0068=Verifying Kubernetes minimum version WLSKO-0069=WebLogic domain image check failed. Required image is {0}, but found image {1} diff --git a/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfigTest.java b/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfigTest.java index 0c2c413fa69..e5f566f1cd2 100644 --- a/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfigTest.java +++ b/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfigTest.java @@ -4,8 +4,13 @@ package oracle.kubernetes.operator.wlsconfig; import oracle.kubernetes.operator.domain.model.oracle.kubernetes.weblogic.domain.v1.ClusterStartup; +import oracle.kubernetes.operator.helpers.ClientHelper; +import oracle.kubernetes.operator.work.NextAction; +import oracle.kubernetes.operator.work.Packet; +import oracle.kubernetes.operator.work.Step; import org.junit.Test; +import java.lang.reflect.Field; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -143,7 +148,7 @@ public void verifyValidateClusterStartupWarnsIfNoServersInCluster() throws Excep TestUtil.LogHandlerImpl handler = null; try { handler = TestUtil.setupLogHandler(wlsClusterConfig); - wlsClusterConfig.validateClusterStartup(cs); + wlsClusterConfig.validateClusterStartup(cs, null); assertTrue("Message logged: " + handler.getAllFormattedMessage(), handler.hasWarningMessageWithSubString("No servers configured in weblogic cluster with name cluster1")); } finally { TestUtil.removeLogHandler(wlsClusterConfig, handler); @@ -158,13 +163,30 @@ public void verifyValidateClusterStartupWarnsIfReplicasTooHigh() throws Exceptio TestUtil.LogHandlerImpl handler = null; try { handler = TestUtil.setupLogHandler(wlsClusterConfig); - wlsClusterConfig.validateClusterStartup(cs); + wlsClusterConfig.validateClusterStartup(cs, null); assertTrue("Message logged: " + handler.getAllFormattedMessage(), handler.hasWarningMessageWithSubString("replicas in clusterStartup for cluster cluster1 is specified with a value of 2 which is larger than the number of configured WLS servers in the cluster: 1")); } finally { TestUtil.removeLogHandler(wlsClusterConfig, handler); } } + @Test + public void verifyValidateClusterStartupSuggestsUpdateIfReplicasTooHigh() throws Exception { + WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1"); + wlsClusterConfig.addServerConfig(createWlsServerConfig("ms-0", 8011, null)); + ClusterStartup cs = new ClusterStartup().clusterName("cluster1").replicas(2); + TestUtil.LogHandlerImpl handler = null; + try { + handler = TestUtil.setupLogHandler(wlsClusterConfig); + ArrayList suggestedConfigUpdates = new ArrayList<>(); + wlsClusterConfig.validateClusterStartup(cs, suggestedConfigUpdates); + assertEquals(1, suggestedConfigUpdates.size()); + assertTrue(suggestedConfigUpdates.get(0) instanceof WlsClusterConfig.DynamicClusterSizeConfigUpdate); + } finally { + TestUtil.removeLogHandler(wlsClusterConfig, handler); + } + } + @Test public void verifyValidateClusterStartupDoesNotWarnIfDynamicCluster() throws Exception { WlsDynamicServersConfig wlsDynamicServersConfig = createDynamicServersConfig(1, 1, "ms-", "cluster1"); @@ -173,7 +195,7 @@ public void verifyValidateClusterStartupDoesNotWarnIfDynamicCluster() throws Exc TestUtil.LogHandlerImpl handler = null; try { handler = TestUtil.setupLogHandler(wlsClusterConfig); - wlsClusterConfig.validateClusterStartup(cs); + wlsClusterConfig.validateClusterStartup(cs, null); assertFalse("No message should be logged, but found: " + handler.getAllFormattedMessage(), handler.hasWarningMessageLogged()); } finally { TestUtil.removeLogHandler(wlsClusterConfig, handler); @@ -208,6 +230,51 @@ public void verifyGetUpdateDynamicClusterSizePayload3() { assertEquals(wlsClusterConfig.getUpdateDynamicClusterSizePayload(8), "{ dynamicClusterSize: 8, maxDynamicClusterSize: 8 }"); } + @Test + public void verifyDynamicClusterSizeConfigUpdateCallsUpdateDynamicClusterSize() { + final WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1"); + final int clusterSize = 8; + WlsClusterConfig.DynamicClusterSizeConfigUpdate dynamicClusterSizeConfigUpdate = + new WlsClusterConfig.DynamicClusterSizeConfigUpdate(wlsClusterConfig, clusterSize); + MockWlsConfigRetriever mockWlsConfigRetriever = new MockWlsConfigRetriever(null, "namespace", + "asServiceName", "adminSecretName"); + assertTrue(dynamicClusterSizeConfigUpdate.doUpdate(mockWlsConfigRetriever)); + assertSame(wlsClusterConfig, mockWlsConfigRetriever.wlsClusterConfigParamValue); + assertEquals(clusterSize, mockWlsConfigRetriever.clusterSizeParamvalue); + } + + @Test + public void verifyStepCreatedFromDynamicClusterSizeConfigUpdate() throws NoSuchFieldException, IllegalAccessException { + final WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1"); + final int clusterSize = 8; + final Step nextStep = new MockStep(null); + WlsClusterConfig.DynamicClusterSizeConfigUpdate dynamicClusterSizeConfigUpdate = + new WlsClusterConfig.DynamicClusterSizeConfigUpdate(wlsClusterConfig, clusterSize); + + WlsConfigRetriever.UpdateDynamicClusterStep updateStep = + (WlsConfigRetriever.UpdateDynamicClusterStep) dynamicClusterSizeConfigUpdate.createStep(nextStep); + assertSame(wlsClusterConfig, updateStep.wlsClusterConfig); + assertEquals(clusterSize, updateStep.desiredClusterSize); + assertSame(nextStep, getNext(updateStep)); + } + + @Test + public void checkDynamicClusterSizeJsonResultReturnsFalseOnNull() { + WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("someCluster"); + assertFalse(wlsClusterConfig.checkUpdateDynamicClusterSizeJsonResult(null)); + } + + @Test + public void checkDynamicClusterSizeJsonResultReturnsFalseOnUnexpectedString() { + WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("someCluster"); + assertFalse(wlsClusterConfig.checkUpdateDynamicClusterSizeJsonResult("{ xyz }")); + } + + @Test + public void checkDynamicClusterSizeJsonResultReturnsTrueWithExpectedString() { + WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("someCluster"); + assertTrue(wlsClusterConfig.checkUpdateDynamicClusterSizeJsonResult("{}")); + } private WlsServerConfig createWlsServerConfig(String serverName, Integer listenPort, String listenAddress) { Map serverConfigMap = new HashMap<>(); serverConfigMap.put("name", serverName); @@ -232,4 +299,40 @@ private WlsDynamicServersConfig createDynamicServersConfig(int clusterSize, int false, serverTemplate, serverConfigs); } + + static class MockWlsConfigRetriever extends WlsConfigRetriever { + + WlsClusterConfig wlsClusterConfigParamValue; + int clusterSizeParamvalue; + + public MockWlsConfigRetriever(ClientHelper clientHelper, String namespace, String asServiceName, String adminSecretName) { + super(clientHelper, namespace, asServiceName, adminSecretName); + } + + @Override + public boolean updateDynamicClusterSize(WlsClusterConfig wlsClusterConfig, int clusterSize) { + wlsClusterConfigParamValue = wlsClusterConfig; + clusterSizeParamvalue = clusterSize; + return true; + } + } + + static class MockStep extends Step { + public MockStep(Step next) { + super(next); + } + + @Override + public NextAction apply(Packet packet) { + return null; + } + } + + Field nextField; + Step getNext(Step step) throws IllegalAccessException, NoSuchFieldException { + if (nextField == null) { + nextField = Step.class.getDeclaredField("next"); + } + return (Step) nextField.get(step); + } } diff --git a/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfigTest.java b/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfigTest.java index 902c3783aba..965c93ae003 100644 --- a/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfigTest.java +++ b/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfigTest.java @@ -209,7 +209,7 @@ public void verifyUpdateDomainSpecWarnsIfNoServersInClusterStartupCluster() thro WlsClusterConfig wlsClusterConfig = wlsDomainConfig.getClusterConfig("noSuchCluster"); try { handler = TestUtil.setupLogHandler(wlsClusterConfig); - wlsDomainConfig.updateDomainSpecAsNeeded(domainSpec); + wlsDomainConfig.validate(domainSpec); assertTrue("Message logged: " + handler.getAllFormattedMessage(), handler.hasWarningMessageWithSubString("No servers configured in weblogic cluster with name noSuchCluster")); } finally { TestUtil.removeLogHandler(wlsClusterConfig, handler); @@ -224,7 +224,7 @@ public void verifyUpdateDomainSpecWarnsIfReplicasTooLarge() throws Exception { WlsClusterConfig wlsClusterConfig = wlsDomainConfig.getClusterConfig("DockerCluster"); try { handler = TestUtil.setupLogHandler(wlsClusterConfig); - wlsDomainConfig.updateDomainSpecAsNeeded(domainSpec); + wlsDomainConfig.validate(domainSpec); assertTrue("Message logged: " + handler.getAllFormattedMessage(), handler.hasWarningMessageWithSubString("replicas in domainSpec for cluster DockerCluster is specified with a value of 10 which is larger than the number of configured WLS servers in the cluster: 5")); } finally { TestUtil.removeLogHandler(wlsClusterConfig, handler); @@ -238,7 +238,7 @@ public void verifyUpdateDomainSpecInfoIfReplicasAndZeroClusters() throws Excepti TestUtil.LogHandlerImpl handler = null; try { handler = TestUtil.setupLogHandler(wlsDomainConfig); - wlsDomainConfig.updateDomainSpecAsNeeded(domainSpec); + wlsDomainConfig.validate(domainSpec); assertTrue("Message logged: " + handler.getAllFormattedMessage(), handler.hasInfoMessageWithSubString("replicas specified in Domain spec is ignored because there number of configured WLS cluster is not 1.")); } finally { TestUtil.removeLogHandler(wlsDomainConfig, handler); @@ -252,7 +252,7 @@ public void verifyUpdateDomainSpecInfoIfReplicasAndTwoClusters() throws Exceptio TestUtil.LogHandlerImpl handler = null; try { handler = TestUtil.setupLogHandler(wlsDomainConfig); - wlsDomainConfig.updateDomainSpecAsNeeded(domainSpec); + wlsDomainConfig.validate(domainSpec); assertTrue("Message logged: " + handler.getAllFormattedMessage(), handler.hasInfoMessageWithSubString("replicas specified in Domain spec is ignored because there number of configured WLS cluster is not 1.")); } finally { TestUtil.removeLogHandler(wlsDomainConfig, handler); @@ -267,7 +267,7 @@ public void verifyUpdateDomainSpecReplicasNotValidatedWithMoreThan1Clusters() th WlsClusterConfig wlsClusterConfig = wlsDomainConfig.getClusterConfig("DockerCluster"); try { handler = TestUtil.setupLogHandler(wlsClusterConfig); - wlsDomainConfig.updateDomainSpecAsNeeded(domainSpec); + wlsDomainConfig.validate(domainSpec); assertFalse(handler.hasWarningMessageLogged()); } finally { TestUtil.removeLogHandler(wlsClusterConfig, handler); @@ -282,7 +282,7 @@ public void verifyUpdateDomainSpecNoWarningIfReplicasOK() throws Exception { WlsClusterConfig wlsClusterConfig = wlsDomainConfig.getClusterConfig("DockerCluster"); try { handler = TestUtil.setupLogHandler(wlsClusterConfig); - wlsDomainConfig.updateDomainSpecAsNeeded(domainSpec); + wlsDomainConfig.validate(domainSpec); assertFalse(handler.hasWarningMessageLogged()); } finally { TestUtil.removeLogHandler(wlsClusterConfig, handler); @@ -297,7 +297,7 @@ public void verifyUpdateDomainSpecWarnsIfClusterStatupReplicasTooLarge() throws WlsClusterConfig wlsClusterConfig = wlsDomainConfig.getClusterConfig("DockerCluster2"); try { handler = TestUtil.setupLogHandler(wlsClusterConfig); - wlsDomainConfig.updateDomainSpecAsNeeded(domainSpec); + wlsDomainConfig.validate(domainSpec); assertTrue("Message logged: " + handler.getAllFormattedMessage(), handler.hasWarningMessageWithSubString("replicas in clusterStartup for cluster DockerCluster2 is specified with a value of 3 which is larger than the number of configured WLS servers in the cluster: 2")); } finally { TestUtil.removeLogHandler(wlsClusterConfig, handler); @@ -314,7 +314,7 @@ public void verifyUpdateDomainSpecWarnsIfClusterStatupReplicasTooLarge_2clusters WlsClusterConfig wlsClusterConfig = wlsDomainConfig.getClusterConfig("DockerCluster2"); try { handler = TestUtil.setupLogHandler(wlsClusterConfig); - wlsDomainConfig.updateDomainSpecAsNeeded(domainSpec); + wlsDomainConfig.validate(domainSpec); assertTrue("Message logged: " + handler.getAllFormattedMessage(), handler.hasWarningMessageWithSubString("replicas in clusterStartup for cluster DockerCluster is specified with a value of 10 which is larger than the number of configured WLS servers in the cluster: 3")); assertTrue("Message logged: " + handler.getAllFormattedMessage(), handler.hasWarningMessageWithSubString("replicas in clusterStartup for cluster DockerCluster2 is specified with a value of 10 which is larger than the number of configured WLS servers in the cluster: 2")); } finally { @@ -330,7 +330,7 @@ public void verifyUpdateDomainSpecNoWarningIfClusterStatupReplicasOK() throws Ex WlsClusterConfig wlsClusterConfig = wlsDomainConfig.getClusterConfig("DockerCluster2"); try { handler = TestUtil.setupLogHandler(wlsClusterConfig); - wlsDomainConfig.updateDomainSpecAsNeeded(domainSpec); + wlsDomainConfig.validate(domainSpec); assertFalse(handler.hasWarningMessageLogged()); } finally { TestUtil.removeLogHandler(wlsClusterConfig, handler); @@ -345,7 +345,7 @@ public void verifyUpdateDomainSpecNoWarningIfClusterStatupOnDynamicCluster() thr WlsClusterConfig wlsClusterConfig = wlsDomainConfig.getClusterConfig("DockerCluster"); try { handler = TestUtil.setupLogHandler(wlsClusterConfig); - wlsDomainConfig.updateDomainSpecAsNeeded(domainSpec); + wlsDomainConfig.validate(domainSpec); assertFalse(handler.hasWarningMessageLogged()); } finally { TestUtil.removeLogHandler(wlsClusterConfig, handler); From 05e9be59f3321b8ec6805dfa81b55e19c3465ef7 Mon Sep 17 00:00:00 2001 From: Anthony Lai Date: Tue, 20 Feb 2018 12:31:50 -0800 Subject: [PATCH 003/186] add dynamic cluster support in create-domain-job.sh --- kubernetes/create-domain-job-inputs.yaml | 8 +- kubernetes/create-domain-job.sh | 2 + kubernetes/internal/domain-job-template.yaml | 309 ++++++++++++------ .../operator/helpers/PodHelper.java | 4 +- 4 files changed, 221 insertions(+), 102 deletions(-) diff --git a/kubernetes/create-domain-job-inputs.yaml b/kubernetes/create-domain-job-inputs.yaml index 4fa4735aa4c..4a36f61d1dc 100644 --- a/kubernetes/create-domain-job-inputs.yaml +++ b/kubernetes/create-domain-job-inputs.yaml @@ -17,6 +17,9 @@ domainName: base_domain # This id must be lowercase and unique across all domains in a Kubernetes cluster domainUid: domain1 +# Type of WebLogic Cluster (static or dynamic) +clusterType: dynamic + # Cluster name clusterName: cluster-1 @@ -24,7 +27,10 @@ clusterName: cluster-1 managedServerCount: 2 # Number of managed servers to initially start for the domain -managedServerStartCount: 2 +managedServerStartCount: 1 + +# Maximum number of managed server instances allowed to be created +maxManagedServerCount: 8 # Base string used to generate managed server names managedServerNameBase: managed-server diff --git a/kubernetes/create-domain-job.sh b/kubernetes/create-domain-job.sh index 0eccb634458..fbabaa95f6e 100755 --- a/kubernetes/create-domain-job.sh +++ b/kubernetes/create-domain-job.sh @@ -293,6 +293,8 @@ function createYamlFiles { sed -i -e "s:%T3_CHANNEL_PORT%:${t3ChannelPort}:g" ${jobOutput} sed -i -e "s:%T3_PUBLIC_ADDRESS%:${t3PublicAddress}:g" ${jobOutput} sed -i -e "s:%CLUSTER_NAME%:${clusterName}:g" ${jobOutput} + sed -i -e "s:%CLUSTER_TYPE%:${clusterType}:g" ${jobOutput} + sed -i -e "s:%MAX_MS_COUNT%:${maxManagedServerCount}:g" ${jobOutput} # Generate the yaml to create the domain custom resource echo Generating ${dcrOutput} diff --git a/kubernetes/internal/domain-job-template.yaml b/kubernetes/internal/domain-job-template.yaml index 8ecd8a33c40..096594e8a86 100644 --- a/kubernetes/internal/domain-job-template.yaml +++ b/kubernetes/internal/domain-job-template.yaml @@ -101,60 +101,21 @@ data: export DOMAIN_HOME=${SHARED_PATH}/domain/%DOMAIN_NAME% - # Function to create node manager home for a server - # $1 - Domain UID - # $2 - Server Name - # $3 - Admin Server Hostname (only passed for managed servers) - function createNodeMgrHome() { - - # Create startup.properties file - datadir=${DOMAIN_HOME}/servers/$2/data/nodemanager - startProp=${datadir}/startup.properties - createFolder ${datadir} - echo "# Server startup properties" > ${startProp} - echo "AutoRestart=true" >> ${startProp} - if [ -n "$3" ]; then - echo "AdminURL=http\://$3\:%ADMIN_PORT%" >> ${startProp} - fi - echo "RestartMax=2" >> ${startProp} - echo "RotateLogOnStartup=false" >> ${startProp} - echo "RotationType=bySize" >> ${startProp} - echo "RotationTimeStart=00\:00" >> ${startProp} - echo "RotatedFileCount=100" >> ${startProp} - echo "RestartDelaySeconds=0" >> ${startProp} - echo "FileSizeKB=5000" >> ${startProp} - echo "FileTimeSpanFactor=3600000" >> ${startProp} - echo "RestartInterval=3600" >> ${startProp} - echo "NumberOfFilesLimited=true" >> ${startProp} - echo "FileTimeSpan=24" >> ${startProp} - echo "NMHostName=$1-$2" >> ${startProp} + function createCommonNodeMgrHome() { # Create nodemanager home for the server - nmdir=${DOMAIN_HOME}/servers/$2/nodemgr_home + nmdir=${DOMAIN_HOME}/nodemgr_home createFolder ${nmdir} prop=${nmdir}/nodemanager.properties cp ${DOMAIN_HOME}/nodemanager/nodemanager.properties ${nmdir} - cp ${DOMAIN_HOME}/nodemanager/nodemanager.domains ${nmdir} - cp ${DOMAIN_HOME}/bin/startNodeManager.sh ${nmdir} - - # Edit the start nodemanager script to use the home for the server - sed -i -e "s:/nodemanager:/servers/$2/nodemgr_home:g" ${nmdir}/startNodeManager.sh - - # Edit the nodemanager properties file to use the home for the server - sed -i -e "s:DomainsFile=.*:DomainsFile=${nmdir}/nodemanager.domains:g" ${prop} - sed -i -e "s:NodeManagerHome=.*:NodeManagerHome=${nmdir}:g" ${prop} - sed -i -e "s:ListenAddress=.*:ListenAddress=$1-$2:g" ${prop} - sed -i -e "s:LogFile=.*:LogFile=/shared/logs/nodemanager-$2.log:g" ${prop} - } # Function to create script for starting a server - # $1 - Domain UID - # $2 - Server Name - function createStartScript() { + # \$1 - Domain UID + # \$2 - Server Name + function createCommonStartScripts() { - nmdir=${DOMAIN_HOME}/servers/$2/nodemgr_home - stateFile=${DOMAIN_HOME}/servers/$2/data/nodemanager/$2.state + nmdir=${DOMAIN_HOME}/nodemgr_home scriptFile=${nmdir}/startServer.sh pyFile=${nmdir}/start-server.py @@ -164,24 +125,118 @@ data: cat << EOF > ${scriptFile} #!/bin/bash + domain_uid=\$1 + server_name=\$2 + as_name=\$3 + as_port=\$4 + as_hostname="\$1-\$3" + + echo "debug arguments are \$1 \$2 \$3 \$4" + + nmProp="/u01/nodemanager/nodemanager.properties" + + # TODO: parameterize shared home and domain name + export DOMAIN_HOME=/shared/domain/%DOMAIN_NAME% + + # + # Create a folder + # \$1 - path of folder to create + function createFolder { + mkdir -m 777 -p \$1 + if [ ! -d \$1 ]; then + fail "Unable to create folder \$1" + fi + } + + # Function to create server specific scripts and properties (e.g startup.properties, etc) + # \$1 - Domain UID + # \$2 - Server Name + # \$3 - Admin Server Hostname (only passed for managed servers) + # \$4 - Admin Server port (only passed for managed servers) + function createServerScriptsProperties() { + + # Create startup.properties file + datadir=\${DOMAIN_HOME}/servers/\$2/data/nodemanager + nmdir=\${DOMAIN_HOME}/nodemgr_home + stateFile=\${datadir}/\$2.state + startProp=\${datadir}/startup.properties + if [ -f "\$startProp" ]; then + echo "startup.properties already exists" + return 0 + fi + + createFolder \${datadir} + echo "# Server startup properties" > \${startProp} + echo "AutoRestart=true" >> \${startProp} + if [ -n "\$3" ]; then + echo "AdminURL=http\://\$3\:\$4" >> \${startProp} + fi + echo "RestartMax=2" >> \${startProp} + echo "RotateLogOnStartup=false" >> \${startProp} + echo "RotationType=bySize" >> \${startProp} + echo "RotationTimeStart=00\:00" >> \${startProp} + echo "RotatedFileCount=100" >> \${startProp} + echo "RestartDelaySeconds=0" >> \${startProp} + echo "FileSizeKB=5000" >> \${startProp} + echo "FileTimeSpanFactor=3600000" >> \${startProp} + echo "RestartInterval=3600" >> \${startProp} + echo "NumberOfFilesLimited=true" >> \${startProp} + echo "FileTimeSpan=24" >> \${startProp} + echo "NMHostName=\$1-\$2" >> \${startProp} + + # Create nodemanager home for the server + srvr_nmdir=${DOMAIN_HOME}/servers/\$2/nodemgr_home + createFolder \${srvr_nmdir} + cp ${DOMAIN_HOME}/nodemanager/nodemanager.domains \${srvr_nmdir} + cp ${DOMAIN_HOME}/bin/startNodeManager.sh \${srvr_nmdir} + + # Edit the start nodemanager script to use the home for the server + sed -i -e "s:/nodemanager:/servers/\$2/nodemgr_home:g" \${srvr_nmdir}/startNodeManager.sh + } + # Check for stale state file and remove if found" - if [ -f ${stateFile} ]; then - echo "Removing stale file ${stateFile}" - rm ${stateFile} + if [ -f \${stateFile} ]; then + echo "Removing stale file \${stateFile}" + rm \${stateFile} + fi + + # alai- create nodemanager home directory that is local to the k8s node + mkdir -p /u01/nodemanager + cp \${DOMAIN_HOME}/nodemanager/* /u01/nodemanager/ + cp \${DOMAIN_HOME}/nodemgr_home/nodemanager.properties /u01/nodemanager + + # Edit the nodemanager properties file to use the home for the server + sed -i -e "s:DomainsFile=.*:DomainsFile=/u01/nodemanager/nodemanager.domains:g" /u01/nodemanager/nodemanager.properties + sed -i -e "s:NodeManagerHome=.*:NodeManagerHome=/u01/nodemanager:g" /u01/nodemanager/nodemanager.properties + sed -i -e "s:ListenAddress=.*:ListenAddress=\$1-\$2:g" /u01/nodemanager/nodemanager.properties + sed -i -e "s:LogFile=.*:LogFile=/shared/logs/nodemanager-\$2.log:g" /u01/nodemanager/nodemanager.properties + + export JAVA_PROPERTIES="-DLogFile=/shared/logs/nodemanager-\$server_name.log -DNodeManagerHome=/u01/nodemanager" + export NODEMGR_HOME="/u01/nodemanager" + + + # alai- create startup.properties + echo "Create startup.properties" + if [ -n "\$3" ]; then + echo "this is managed server" + createServerScriptsProperties \$domain_uid \$server_name \$as_hostname \$as_port + else + echo "this is admin server" + createServerScriptsProperties \$domain_uid \$server_name fi echo "Start the nodemanager" - . ${nmdir}/startNodeManager.sh & + . \${DOMAIN_HOME}/servers/\$2/nodemgr_home/startNodeManager.sh & echo "Allow the nodemanager some time to start before attempting to connect" sleep 15 echo "Finished waiting for the nodemanager to start" echo "Update JVM arguments" - echo "Arguments=\${USER_MEM_ARGS} -XX\:+UnlockExperimentalVMOptions -XX\:+UseCGroupMemoryLimitForHeap \${JAVA_OPTIONS}" >> ${startProp} + echo "Arguments=\${USER_MEM_ARGS} -XX\:+UnlockExperimentalVMOptions -XX\:+UseCGroupMemoryLimitForHeap \${JAVA_OPTIONS}" >> \${startProp} echo "Start the server" - wlst.sh -skipWLSModuleScanning ${pyFile} + wlst.sh -skipWLSModuleScanning ${pyFile} \$domain_uid \$server_name echo "Wait indefinitely so that the Kubernetes pod does not exit and try to restart" while true; do sleep 60; done @@ -192,12 +247,47 @@ data: # Create a python script to execute the wlst commands. # The script and 'EOF' on the following lines must not be indented! - cat /u01/weblogic/read-domain-secret.py > ${pyFile} + echo "import sys;" > ${pyFile} + cat /u01/weblogic/read-domain-secret.py >> ${pyFile} cat << EOF >> ${pyFile} + domain_uid = sys.argv[1] + server_name = sys.argv[2] + + domain_path= "${DOMAIN_HOME}" + + print 'admin username is %s' % admin_username + print 'domain path is %s' % domain_path + print 'server name is %s' % server_name + + # Encrypt the admin username and password + adminUsernameEncrypted=encrypt(admin_username, domain_path) + adminPasswordEncrypted=encrypt(admin_password, domain_path) + + print 'Create boot.properties files for this server' + + # Define the folder path + secdir='%s/servers/%s/security' % (domain_path, server_name) + + # Create the security folder (if it does not already exist) + try: + os.makedirs(secdir) + except OSError: + if not os.path.isdir(secdir): + raise + + print 'writing boot.properties to %s/servers/%s/security/boot.properties' % (domain_path, server_name) + + bpFile=open('%s/servers/%s/security/boot.properties' % (domain_path, server_name), 'w+') + bpFile.write("username=%s\n" % adminUsernameEncrypted) + bpFile.write("password=%s\n" % adminPasswordEncrypted) + bpFile.close() + + service_name = domain_uid + "-" + server_name + # Connect to nodemanager and start server - nmConnect(admin_username, admin_password, '$1-$2', '5556', '%DOMAIN_NAME%', '${DOMAIN_HOME}', 'plain') - nmStart('$2') + nmConnect(admin_username, admin_password, service_name, '5556', '%DOMAIN_NAME%', '${DOMAIN_HOME}', 'plain') + nmStart(server_name) # Exit WLST nmDisconnect() @@ -218,7 +308,7 @@ data: # the pod should be restarted. The script checks a WebLogic Server state file which # is updated by the node manager. - STATEFILE=${stateFile} + STATEFILE=/shared/domain/\$1/servers/\$2/data/nodemanager/\$2.state if [ \`jps -l | grep -c " weblogic.NodeManager"\` -eq 0 ]; then echo "Error: WebLogic NodeManager process not found." @@ -240,11 +330,11 @@ data: } # Function to create script for stopping a server - # $1 - Domain UID - # $2 - Server Name - function createStopScript() { + # \$1 - Domain UID + # \$2 - Server Name + function createCommonStopScripts() { - nmdir=${DOMAIN_HOME}/servers/$2/nodemgr_home + nmdir=${DOMAIN_HOME}/nodemgr_home scriptFile=${nmdir}/stopServer.sh pyFile=${nmdir}/stop-server.py @@ -259,7 +349,7 @@ data: # Return status of 2 means failed to stop a server through the NodeManager. # Look to see if there is a server process that can be killed. if [ \$? -eq 2 ]; then - pid=\$(jps -v | grep '[D]weblogic.Name=$2' | awk '{print \$1}') + pid=\$(jps -v | grep '[D]weblogic.Name=\$2' | awk '{print \$1}') if [ ! -z \$pid ]; then echo "Killing the server process \$pid" kill -15 \$pid @@ -278,14 +368,14 @@ data: # Connect to nodemanager and stop server try: - nmConnect(admin_username, admin_password, '$1-$2', '5556', '%DOMAIN_NAME%', '${DOMAIN_HOME}', 'plain') + nmConnect(admin_username, admin_password, '\$1-\$2', '5556', '%DOMAIN_NAME%', '${DOMAIN_HOME}', 'plain') except: print('Failed to connect to the NodeManager') exit(exitcode=2) # Kill the server try: - nmKill('$2') + nmKill('\$2') except: print('Connected to the NodeManager, but failed to stop the server') exit(exitcode=2) @@ -301,20 +391,9 @@ data: # Create the domain wlst.sh -skipWLSModuleScanning /u01/weblogic/create-domain.py - # Setup admin server - createNodeMgrHome %DOMAIN_UID% %ADMIN_SERVER_NAME% - createStartScript %DOMAIN_UID% %ADMIN_SERVER_NAME% - createStopScript %DOMAIN_UID% %ADMIN_SERVER_NAME% - - # Create the managed servers - index=0 - while [ $index -lt %NUMBER_OF_MS% ] - do - ((index++)) - createNodeMgrHome %DOMAIN_UID% %MANAGED_SERVER_NAME_BASE%${index} %DOMAIN_UID%-%ADMIN_SERVER_NAME% - createStartScript %DOMAIN_UID% %MANAGED_SERVER_NAME_BASE%${index} - createStopScript %DOMAIN_UID% %MANAGED_SERVER_NAME_BASE%${index} - done + createCommonNodeMgrHome + createCommonStartScripts + createCommonStopScripts echo "Successfully Completed" @@ -328,6 +407,8 @@ data: domain_path = os.environ.get("DOMAIN_HOME") cluster_name = "%CLUSTER_NAME%" number_of_ms = %NUMBER_OF_MS% + cluster_type = "%CLUSTER_TYPE%" + max_ms_count = %MAX_MS_COUNT% print('domain_path : [%s]' % domain_path); print('domain_name : [%DOMAIN_NAME%]'); @@ -335,6 +416,8 @@ data: print('admin_port : [%ADMIN_PORT%]'); print('cluster_name : [%s]' % cluster_name); print('server_port : [%s]' % server_port); + print('cluster_type : [%s]' % cluster_type); + print('max_ms_count : [%s]' % max_ms_count); # Open default domain template # ============================ @@ -388,7 +471,7 @@ data: set('DomainsDirRemoteSharingEnabled', 'true') # Set the Node Manager user name and password (domain name will change after writeDomain) - cd('/SecurityConfiguration/base_domain') + cd('/SecurityConfiguration/%DOMAIN_NAME%') set('NodeManagerUsername', admin_username) set('NodeManagerPasswordEncrypted', admin_password) @@ -407,29 +490,55 @@ data: # Create a cluster cd('/') - create(cluster_name, 'Cluster') - - # Create managed servers - for index in range(0, number_of_ms): - cd('/') - - msIndex = index+1 - name = '%MANAGED_SERVER_NAME_BASE%%s' % msIndex - machineName = '%DOMAIN_UID%-machine%s' % msIndex - - create(name, 'Server') - cd('/Servers/%s/' % name ) - print('managed server name is %s' % name); - set('ListenAddress', '%DOMAIN_UID%-%s' % name) - set('ListenPort', server_port) - set('NumOfRetriesBeforeMSIMode', 0) - set('RetryIntervalBeforeMSIMode', 1) - set('Cluster', cluster_name) - set('Machine', machineName) - - create(name,'Log') - cd('/Servers/%s/Log/%s' % (name, name)) - set('FileName', '/shared/logs/%s.log' % name) + cl=create(cluster_name, 'Cluster') + + if cluster_type == "static": + + # Create managed servers + for index in range(0, number_of_ms): + cd('/') + + msIndex = index+1 + name = '%MANAGED_SERVER_NAME_BASE%%s' % msIndex + machineName = '%DOMAIN_UID%-machine%s' % msIndex + + create(name, 'Server') + cd('/Servers/%s/' % name ) + print('managed server name is %s' % name); + set('ListenAddress', '%DOMAIN_UID%-%s' % name) + set('ListenPort', server_port) + set('NumOfRetriesBeforeMSIMode', 0) + set('RetryIntervalBeforeMSIMode', 1) + set('Cluster', cluster_name) + set('Machine', machineName) + + create(name,'Log') + cd('/Servers/%s/Log/%s' % (name, name)) + set('FileName', '/shared/logs/%s.log' % name) + else: + print('Configuring Dynamic Cluster %s' % cluster_name) + + print('Creating Server Template: serverTemplate-1'); + st1=create('serverTemplate-1', 'ServerTemplate') + print('Done creating Server Template: serverTemplate-1'); + cd('/ServerTemplates/serverTemplate-1') + cmo.setListenPort(server_port) + cmo.setListenAddress('%DOMAIN_UID%-%MANAGED_SERVER_NAME_BASE%${id}') + cmo.setCluster(cl) + print('Done setting attributes for Server Template: serverTemplate-1'); + + cd('/Clusters/%s' % cluster_name) + create(cluster_name, 'DynamicServers') + cd('DynamicServers/%s' % cluster_name) + set('ServerTemplate', st1) + set('ServerNamePrefix', "%MANAGED_SERVER_NAME_BASE%") + set('DynamicClusterSize', number_of_ms) + set('MaxDynamicClusterSize', max_ms_count) + set('CalculatedMachineNames', true) + set('CalculatedListenPorts', false) + set('Id', 1) + + print('Done setting attributes for Dynamic Cluster: %s' % cluster_name); # Write Domain # ============ diff --git a/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java b/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java index 962776d2297..51a3923c96e 100644 --- a/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java +++ b/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java @@ -134,7 +134,7 @@ public NextAction apply(Packet packet) { V1Lifecycle lifecycle = new V1Lifecycle(); V1Handler preStopHandler = new V1Handler(); V1ExecAction lifecycleExecAction = new V1ExecAction(); - lifecycleExecAction.addCommandItem("/shared/domain/" + weblogicDomainName + "/servers/" + spec.getAsName() + "/nodemgr_home/stopServer.sh"); + lifecycleExecAction.addCommandItem("/shared/domain/" + weblogicDomainName + "/nodemgr_home/stopServer.sh"); preStopHandler.setExec(lifecycleExecAction); lifecycle.setPreStop(preStopHandler); container.setLifecycle(lifecycle); @@ -166,6 +166,7 @@ public NextAction apply(Packet packet) { V1Probe livenessProbe = new V1Probe(); V1ExecAction livenessExecAction = new V1ExecAction(); livenessExecAction.addCommandItem("/shared/domain/" + weblogicDomainName + "/nodemgr_home/livenessProbe.sh"); + livenessExecAction.addCommandItem(weblogicDomainName); livenessExecAction.addCommandItem(spec.getAsName()); livenessProbe.exec(livenessExecAction); livenessProbe.setInitialDelaySeconds(10); @@ -504,6 +505,7 @@ public NextAction apply(Packet packet) { V1Probe livenessProbe = new V1Probe(); V1ExecAction execAction = new V1ExecAction(); execAction.addCommandItem("/shared/domain/" + weblogicDomainName + "/nodemgr_home/livenessProbe.sh"); + execAction.addCommandItem(weblogicDomainName); execAction.addCommandItem(weblogicServerName); livenessProbe.exec(execAction); livenessProbe.setInitialDelaySeconds(10); From 81fa34dabecc475249f42c9a639ee78dfdc1ca19 Mon Sep 17 00:00:00 2001 From: anthony_lai Date: Thu, 22 Feb 2018 16:18:31 -0800 Subject: [PATCH 004/186] updated max dynamic server count processing, removed boot.properties for admin server, removed per server nodemgr_home, add dynamic cluster size verification in RestBackendImpl --- kubernetes/create-domain-job-inputs.yaml | 3 - kubernetes/create-domain-job.sh | 1 - kubernetes/internal/domain-job-template.yaml | 72 ++++++------------- .../operator/logging/MessageKeys.java | 11 +-- .../operator/rest/RestBackendImpl.java | 18 +++-- .../wlsconfig/WlsConfigRetriever.java | 1 + src/main/resources/Operator.properties | 11 +-- 7 files changed, 47 insertions(+), 70 deletions(-) diff --git a/kubernetes/create-domain-job-inputs.yaml b/kubernetes/create-domain-job-inputs.yaml index 4a36f61d1dc..953539e31d7 100644 --- a/kubernetes/create-domain-job-inputs.yaml +++ b/kubernetes/create-domain-job-inputs.yaml @@ -29,9 +29,6 @@ managedServerCount: 2 # Number of managed servers to initially start for the domain managedServerStartCount: 1 -# Maximum number of managed server instances allowed to be created -maxManagedServerCount: 8 - # Base string used to generate managed server names managedServerNameBase: managed-server diff --git a/kubernetes/create-domain-job.sh b/kubernetes/create-domain-job.sh index fbabaa95f6e..4c760f10875 100755 --- a/kubernetes/create-domain-job.sh +++ b/kubernetes/create-domain-job.sh @@ -294,7 +294,6 @@ function createYamlFiles { sed -i -e "s:%T3_PUBLIC_ADDRESS%:${t3PublicAddress}:g" ${jobOutput} sed -i -e "s:%CLUSTER_NAME%:${clusterName}:g" ${jobOutput} sed -i -e "s:%CLUSTER_TYPE%:${clusterType}:g" ${jobOutput} - sed -i -e "s:%MAX_MS_COUNT%:${maxManagedServerCount}:g" ${jobOutput} # Generate the yaml to create the domain custom resource echo Generating ${dcrOutput} diff --git a/kubernetes/internal/domain-job-template.yaml b/kubernetes/internal/domain-job-template.yaml index 096594e8a86..29f1aba1850 100644 --- a/kubernetes/internal/domain-job-template.yaml +++ b/kubernetes/internal/domain-job-template.yaml @@ -155,6 +155,15 @@ data: # \$4 - Admin Server port (only passed for managed servers) function createServerScriptsProperties() { + # Create nodemanager home for the server + srvr_nmdir=/u01/nodemanager + createFolder \${srvr_nmdir} + cp ${DOMAIN_HOME}/nodemanager/nodemanager.domains \${srvr_nmdir} + cp ${DOMAIN_HOME}/bin/startNodeManager.sh \${srvr_nmdir} + + # Edit the start nodemanager script to use the home for the server + sed -i -e "s:/shared/domain/base_domain/nodemanager:/u01/nodemanager:g" ${srvr_nmdir}/startNodeManager.sh + # Create startup.properties file datadir=\${DOMAIN_HOME}/servers/\$2/data/nodemanager nmdir=\${DOMAIN_HOME}/nodemgr_home @@ -183,15 +192,6 @@ data: echo "NumberOfFilesLimited=true" >> \${startProp} echo "FileTimeSpan=24" >> \${startProp} echo "NMHostName=\$1-\$2" >> \${startProp} - - # Create nodemanager home for the server - srvr_nmdir=${DOMAIN_HOME}/servers/\$2/nodemgr_home - createFolder \${srvr_nmdir} - cp ${DOMAIN_HOME}/nodemanager/nodemanager.domains \${srvr_nmdir} - cp ${DOMAIN_HOME}/bin/startNodeManager.sh \${srvr_nmdir} - - # Edit the start nodemanager script to use the home for the server - sed -i -e "s:/nodemanager:/servers/\$2/nodemgr_home:g" \${srvr_nmdir}/startNodeManager.sh } # Check for stale state file and remove if found" @@ -200,7 +200,7 @@ data: rm \${stateFile} fi - # alai- create nodemanager home directory that is local to the k8s node + # Create nodemanager home directory that is local to the k8s node mkdir -p /u01/nodemanager cp \${DOMAIN_HOME}/nodemanager/* /u01/nodemanager/ cp \${DOMAIN_HOME}/nodemgr_home/nodemanager.properties /u01/nodemanager @@ -215,7 +215,7 @@ data: export NODEMGR_HOME="/u01/nodemanager" - # alai- create startup.properties + # Create startup.properties echo "Create startup.properties" if [ -n "\$3" ]; then echo "this is managed server" @@ -226,7 +226,7 @@ data: fi echo "Start the nodemanager" - . \${DOMAIN_HOME}/servers/\$2/nodemgr_home/startNodeManager.sh & + . \${NODEMGR_HOME}/startNodeManager.sh & echo "Allow the nodemanager some time to start before attempting to connect" sleep 15 @@ -408,7 +408,6 @@ data: cluster_name = "%CLUSTER_NAME%" number_of_ms = %NUMBER_OF_MS% cluster_type = "%CLUSTER_TYPE%" - max_ms_count = %MAX_MS_COUNT% print('domain_path : [%s]' % domain_path); print('domain_name : [%DOMAIN_NAME%]'); @@ -417,7 +416,6 @@ data: print('cluster_name : [%s]' % cluster_name); print('server_port : [%s]' % server_port); print('cluster_type : [%s]' % cluster_type); - print('max_ms_count : [%s]' % max_ms_count); # Open default domain template # ============================ @@ -518,14 +516,16 @@ data: else: print('Configuring Dynamic Cluster %s' % cluster_name) - print('Creating Server Template: serverTemplate-1'); - st1=create('serverTemplate-1', 'ServerTemplate') - print('Done creating Server Template: serverTemplate-1'); - cd('/ServerTemplates/serverTemplate-1') + templateName = cluster_name + "-template" + print('Creating Server Template: %s' % templateName) + st1=create(templateName, 'ServerTemplate') + print('Done creating Server Template: %s' % templateName) + cd('/ServerTemplates/%s' % templateName) cmo.setListenPort(server_port) cmo.setListenAddress('%DOMAIN_UID%-%MANAGED_SERVER_NAME_BASE%${id}') cmo.setCluster(cl) - print('Done setting attributes for Server Template: serverTemplate-1'); + print('Done setting attributes for Server Template: %s' % templateName); + cd('/Clusters/%s' % cluster_name) create(cluster_name, 'DynamicServers') @@ -533,7 +533,7 @@ data: set('ServerTemplate', st1) set('ServerNamePrefix', "%MANAGED_SERVER_NAME_BASE%") set('DynamicClusterSize', number_of_ms) - set('MaxDynamicClusterSize', max_ms_count) + set('MaxDynamicClusterSize', 800) set('CalculatedMachineNames', true) set('CalculatedListenPorts', false) set('Id', 1) @@ -553,38 +553,6 @@ data: updateDomain() closeDomain() print 'Domain Updated' - - # Encrypt the admin username and password - adminUsernameEncrypted=encrypt(admin_username, domain_path) - adminPasswordEncrypted=encrypt(admin_password, domain_path) - - print 'Create boot.properties files for admin and managed servers' - - asbpFile=open('%s/servers/%ADMIN_SERVER_NAME%/security/boot.properties' % domain_path, 'w+') - asbpFile.write("username=%s\n" % adminUsernameEncrypted) - asbpFile.write("password=%s\n" % adminPasswordEncrypted) - asbpFile.close() - - import os - - # Create boot.properties file for each managed server - for index in range(0, number_of_ms): - - # Define the folder path - secdir='%s/servers/%MANAGED_SERVER_NAME_BASE%%s/security' % (domain_path, index+1) - - # Create the security folder (if it does not already exist) - try: - os.makedirs(secdir) - except OSError: - if not os.path.isdir(secdir): - raise - - bpFile=open('%s/boot.properties' % secdir, 'w+') - bpFile.write("username=%s\n" % adminUsernameEncrypted) - bpFile.write("password=%s\n" % adminPasswordEncrypted) - bpFile.close() - print 'Done' # Exit WLST diff --git a/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java b/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java index 7812ef7ff0d..53a45f35620 100644 --- a/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java +++ b/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java @@ -129,9 +129,10 @@ private MessageKeys() { public static final String POD_DELETED = "WLSKO-0113"; public static final String SERVICE_DELETED = "WLSKO-0114"; public static final String INGRESS_DELETED = "WLSKO-0115"; - public static final String WLS_UPDATE_CLUSTER_SIZE_FAILED = "WLSKO-0116"; - public static final String WLS_UPDATE_CLUSTER_SIZE_TIMED_OUT = "WLSKO-0117"; - public static final String WLS_UPDATE_CLUSTER_SIZE_INVALID_CLUSTER = "WLSKO-0118"; - public static final String WLS_CLUSTER_SIZE_UPDATED = "WLSKO-0119"; - public static final String WLS_SERVER_TEMPLATE_NOT_FOUND = "WLSKO-0120"; + public static final String WLS_UPDATE_CLUSTER_SIZE_STARTING = "WLSKO-0116"; + public static final String WLS_UPDATE_CLUSTER_SIZE_FAILED = "WLSKO-0117"; + public static final String WLS_UPDATE_CLUSTER_SIZE_TIMED_OUT = "WLSKO-0118"; + public static final String WLS_UPDATE_CLUSTER_SIZE_INVALID_CLUSTER = "WLSKO-0119"; + public static final String WLS_CLUSTER_SIZE_UPDATED = "WLSKO-0120"; + public static final String WLS_SERVER_TEMPLATE_NOT_FOUND = "WLSKO-0121"; } diff --git a/src/main/java/oracle/kubernetes/operator/rest/RestBackendImpl.java b/src/main/java/oracle/kubernetes/operator/rest/RestBackendImpl.java index 4a1e4d6d39c..3ea6d6c3561 100644 --- a/src/main/java/oracle/kubernetes/operator/rest/RestBackendImpl.java +++ b/src/main/java/oracle/kubernetes/operator/rest/RestBackendImpl.java @@ -377,7 +377,13 @@ private void verifyWLSConfiguredClusterCapacity(ClientHolder client, String name // and verify we have enough configured managed servers to auto-scale String adminServerServiceName = getAdminServerServiceName(domain); String adminSecretName = getAdminServiceSecretName(domain); - int clusterSize = getWLSConfiguredClusterSize(client, adminServerServiceName, cluster, namespace, adminSecretName); + WlsClusterConfig wlsClusterConfig = getWlsClusterConfig(client, namespace, cluster, adminServerServiceName, adminSecretName); + + // Verify the current configured cluster size + int clusterSize = wlsClusterConfig.getClusterSize(); + if (wlsClusterConfig.hasDynamicServers()) { + clusterSize += wlsClusterConfig.getDynamicClusterSize(); + } if (managedServerCount > clusterSize) { throw createWebApplicationException(Status.BAD_REQUEST, MessageKeys.SCALE_COUNT_GREATER_THAN_CONFIGURED, managedServerCount, clusterSize, cluster, cluster); } @@ -399,11 +405,15 @@ private ClusterStartup getClusterStartup(Domain domain, String cluster) { return null; } - private int getWLSConfiguredClusterSize(ClientHolder client, String adminServerServiceName, String cluster, String namespace, String adminSecretName) { + private int getWLSConfiguredClusterSize(ClientHolder client, String namespace, String cluster, String adminServerServiceName, String adminSecretName) { + WlsClusterConfig wlsClusterConfig = getWlsClusterConfig(client, namespace, cluster, adminServerServiceName, adminSecretName); + return wlsClusterConfig.getClusterSize(); + } + + private WlsClusterConfig getWlsClusterConfig(ClientHolder client, String namespace, String cluster, String adminServerServiceName, String adminSecretName) { WlsConfigRetriever wlsConfigRetriever = WlsConfigRetriever.create(client.getHelper(), namespace, adminServerServiceName, adminSecretName); WlsDomainConfig wlsDomainConfig = wlsConfigRetriever.readConfig(); - WlsClusterConfig wlsClusterConfig = wlsDomainConfig.getClusterConfig(cluster); - return wlsClusterConfig.getClusterSize(); + return wlsDomainConfig.getClusterConfig(cluster); } private Map getWLSConfiguredClusters(ClientHolder client, String namespace, String adminServerServiceName, String adminSecretName) { diff --git a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetriever.java b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetriever.java index 181f5fea3c9..7912cdcdb2f 100644 --- a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetriever.java +++ b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetriever.java @@ -174,6 +174,7 @@ public NextAction apply(Packet packet) { LOGGER.warning(MessageKeys.WLS_UPDATE_CLUSTER_SIZE_INVALID_CLUSTER, clusterName); } else { try { + LOGGER.info(MessageKeys.WLS_UPDATE_CLUSTER_SIZE_STARTING, clusterName, desiredClusterSize); HttpClient httpClient = (HttpClient) packet.get(HttpClient.KEY); DomainPresenceInfo info = packet.getSPI(DomainPresenceInfo.class); diff --git a/src/main/resources/Operator.properties b/src/main/resources/Operator.properties index fd8319eaba5..f18829c85e7 100644 --- a/src/main/resources/Operator.properties +++ b/src/main/resources/Operator.properties @@ -114,8 +114,9 @@ WLSKO-0112=List Ingress for domain with domainUID {0} in namespace {1} WLSKO-0113=Pod for domain with domainUID {0} in namespace {1} and with server name {2} deleted; validating domain WLSKO-0114=Service for domain with domainUID {0} in namespace {1} and with server name {2} deleted; validating domain WLSKO-0115=Ingress for domain with domainUID {0} in namespace {1} and with cluster name {2} deleted; validating domain -WLSKO-0116=Failed to update WebLogic dynamic cluster size for cluster {0} due to exception: {1} -WLSKO-0117=Failed to update WebLogic dynamic cluster size for cluster {0} within timeout of {1} milliseconds -WLSKO-0118=Failed to update WebLogic dynamic cluster size for cluster {0}. Cluster is not a dynamic cluster. -WLSKO-0119=Updated cluster size for WebLogic dynamic cluster {0} to {1}. Time taken {2} ms. -WLSKO-0120=Cannot find WebLogic server template with name {0} which is referenced by WebLogic cluster {1}. +WLSKO-0116=Updating cluster size for WebLogic dynamic cluster {0} to {1}. +WLSKO-0117=Failed to update WebLogic dynamic cluster size for cluster {0} due to exception: {1} +WLSKO-0118=Failed to update WebLogic dynamic cluster size for cluster {0} within timeout of {1} milliseconds +WLSKO-0119=Failed to update WebLogic dynamic cluster size for cluster {0}. Cluster is not a dynamic cluster. +WLSKO-0120=Updated cluster size for WebLogic dynamic cluster {0} to {1}. Time taken {2} ms. +WLSKO-0121=Cannot find WebLogic server template with name {0} which is referenced by WebLogic cluster {1}. From f7f697fd3597c7db7f3f281416c474910c551072 Mon Sep 17 00:00:00 2001 From: anthony_lai Date: Fri, 23 Feb 2018 16:01:12 -0800 Subject: [PATCH 005/186] cluster_type should be configured or dynamic --- kubernetes/create-domain-job-inputs.yaml | 2 +- kubernetes/internal/domain-job-template.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/create-domain-job-inputs.yaml b/kubernetes/create-domain-job-inputs.yaml index 953539e31d7..44722183d1d 100644 --- a/kubernetes/create-domain-job-inputs.yaml +++ b/kubernetes/create-domain-job-inputs.yaml @@ -17,7 +17,7 @@ domainName: base_domain # This id must be lowercase and unique across all domains in a Kubernetes cluster domainUid: domain1 -# Type of WebLogic Cluster (static or dynamic) +# Type of WebLogic Cluster (configured or dynamic) clusterType: dynamic # Cluster name diff --git a/kubernetes/internal/domain-job-template.yaml b/kubernetes/internal/domain-job-template.yaml index 29f1aba1850..e1e89cc96be 100644 --- a/kubernetes/internal/domain-job-template.yaml +++ b/kubernetes/internal/domain-job-template.yaml @@ -490,7 +490,7 @@ data: cd('/') cl=create(cluster_name, 'Cluster') - if cluster_type == "static": + if cluster_type == "configured": # Create managed servers for index in range(0, number_of_ms): From c16018fc46afe88e4daf5be48499989415434872 Mon Sep 17 00:00:00 2001 From: Anthony Lai Date: Mon, 26 Feb 2018 10:03:50 -0800 Subject: [PATCH 006/186] Fixes unit tests in wlsconfig --- .../wlsconfig/WlsClusterConfigTest.java | 27 +++++++++++++++---- .../wlsconfig/WlsDomainConfigTest.java | 14 +++++----- 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfigTest.java b/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfigTest.java index e5f566f1cd2..d85517df4dc 100644 --- a/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfigTest.java +++ b/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfigTest.java @@ -149,7 +149,7 @@ public void verifyValidateClusterStartupWarnsIfNoServersInCluster() throws Excep try { handler = TestUtil.setupLogHandler(wlsClusterConfig); wlsClusterConfig.validateClusterStartup(cs, null); - assertTrue("Message logged: " + handler.getAllFormattedMessage(), handler.hasWarningMessageWithSubString("No servers configured in weblogic cluster with name cluster1")); + assertTrue("Message logged: " + handler.getAllFormattedMessage(), handler.hasWarningMessageWithSubString("No servers configured in WebLogic cluster with name cluster1")); } finally { TestUtil.removeLogHandler(wlsClusterConfig, handler); } @@ -164,14 +164,14 @@ public void verifyValidateClusterStartupWarnsIfReplicasTooHigh() throws Exceptio try { handler = TestUtil.setupLogHandler(wlsClusterConfig); wlsClusterConfig.validateClusterStartup(cs, null); - assertTrue("Message logged: " + handler.getAllFormattedMessage(), handler.hasWarningMessageWithSubString("replicas in clusterStartup for cluster cluster1 is specified with a value of 2 which is larger than the number of configured WLS servers in the cluster: 1")); + assertTrue("Message logged: " + handler.getAllFormattedMessage(), handler.hasWarningMessageWithSubString("Replicas in clusterStartup for cluster cluster1 is specified with a value of 2 which is larger than the number of configured WLS servers in the cluster: 1")); } finally { TestUtil.removeLogHandler(wlsClusterConfig, handler); } } @Test - public void verifyValidateClusterStartupSuggestsUpdateIfReplicasTooHigh() throws Exception { + public void verifyValidateClusterStartupDoNotSuggestsUpdateToConfiguredClusterIfReplicasTooHigh() throws Exception { WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1"); wlsClusterConfig.addServerConfig(createWlsServerConfig("ms-0", 8011, null)); ClusterStartup cs = new ClusterStartup().clusterName("cluster1").replicas(2); @@ -180,8 +180,7 @@ public void verifyValidateClusterStartupSuggestsUpdateIfReplicasTooHigh() throws handler = TestUtil.setupLogHandler(wlsClusterConfig); ArrayList suggestedConfigUpdates = new ArrayList<>(); wlsClusterConfig.validateClusterStartup(cs, suggestedConfigUpdates); - assertEquals(1, suggestedConfigUpdates.size()); - assertTrue(suggestedConfigUpdates.get(0) instanceof WlsClusterConfig.DynamicClusterSizeConfigUpdate); + assertEquals(0, suggestedConfigUpdates.size()); } finally { TestUtil.removeLogHandler(wlsClusterConfig, handler); } @@ -202,6 +201,23 @@ public void verifyValidateClusterStartupDoesNotWarnIfDynamicCluster() throws Exc } } + @Test + public void verifyValidateClusterStartupSuggestsUpdateToDynamicClusterIfReplicasTooHigh() throws Exception { + WlsDynamicServersConfig wlsDynamicServersConfig = createDynamicServersConfig(1, 1, "ms-", "cluster1"); + WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1", wlsDynamicServersConfig); + ClusterStartup cs = new ClusterStartup().clusterName("cluster1").replicas(2); + TestUtil.LogHandlerImpl handler = null; + try { + handler = TestUtil.setupLogHandler(wlsClusterConfig); + ArrayList suggestedConfigUpdates = new ArrayList<>(); + wlsClusterConfig.validateClusterStartup(cs, suggestedConfigUpdates); + assertEquals(1, suggestedConfigUpdates.size()); + assertTrue(suggestedConfigUpdates.get(0) instanceof WlsClusterConfig.DynamicClusterSizeConfigUpdate); + } finally { + TestUtil.removeLogHandler(wlsClusterConfig, handler); + } + } + @Test public void verifyGetUpdateDynamicClusterSizeUrlIncludesClusterName() { WlsDynamicServersConfig wlsDynamicServersConfig = createDynamicServersConfig(1, 1, "ms-", "cluster1"); @@ -332,6 +348,7 @@ public NextAction apply(Packet packet) { Step getNext(Step step) throws IllegalAccessException, NoSuchFieldException { if (nextField == null) { nextField = Step.class.getDeclaredField("next"); + nextField.setAccessible(true); } return (Step) nextField.get(step); } diff --git a/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfigTest.java b/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfigTest.java index 965c93ae003..2626f6a93d5 100644 --- a/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfigTest.java +++ b/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfigTest.java @@ -210,7 +210,7 @@ public void verifyUpdateDomainSpecWarnsIfNoServersInClusterStartupCluster() thro try { handler = TestUtil.setupLogHandler(wlsClusterConfig); wlsDomainConfig.validate(domainSpec); - assertTrue("Message logged: " + handler.getAllFormattedMessage(), handler.hasWarningMessageWithSubString("No servers configured in weblogic cluster with name noSuchCluster")); + assertTrue("Message logged: " + handler.getAllFormattedMessage(), handler.hasWarningMessageWithSubString("No servers configured in WebLogic cluster with name noSuchCluster")); } finally { TestUtil.removeLogHandler(wlsClusterConfig, handler); } @@ -225,7 +225,7 @@ public void verifyUpdateDomainSpecWarnsIfReplicasTooLarge() throws Exception { try { handler = TestUtil.setupLogHandler(wlsClusterConfig); wlsDomainConfig.validate(domainSpec); - assertTrue("Message logged: " + handler.getAllFormattedMessage(), handler.hasWarningMessageWithSubString("replicas in domainSpec for cluster DockerCluster is specified with a value of 10 which is larger than the number of configured WLS servers in the cluster: 5")); + assertTrue("Message logged: " + handler.getAllFormattedMessage(), handler.hasWarningMessageWithSubString("Replicas in domainSpec for cluster DockerCluster is specified with a value of 10 which is larger than the number of configured WLS servers in the cluster: 5")); } finally { TestUtil.removeLogHandler(wlsClusterConfig, handler); } @@ -239,7 +239,7 @@ public void verifyUpdateDomainSpecInfoIfReplicasAndZeroClusters() throws Excepti try { handler = TestUtil.setupLogHandler(wlsDomainConfig); wlsDomainConfig.validate(domainSpec); - assertTrue("Message logged: " + handler.getAllFormattedMessage(), handler.hasInfoMessageWithSubString("replicas specified in Domain spec is ignored because there number of configured WLS cluster is not 1.")); + assertTrue("Message logged: " + handler.getAllFormattedMessage(), handler.hasInfoMessageWithSubString("Replicas specified in Domain spec is ignored because there number of configured WLS cluster is not 1.")); } finally { TestUtil.removeLogHandler(wlsDomainConfig, handler); } @@ -253,7 +253,7 @@ public void verifyUpdateDomainSpecInfoIfReplicasAndTwoClusters() throws Exceptio try { handler = TestUtil.setupLogHandler(wlsDomainConfig); wlsDomainConfig.validate(domainSpec); - assertTrue("Message logged: " + handler.getAllFormattedMessage(), handler.hasInfoMessageWithSubString("replicas specified in Domain spec is ignored because there number of configured WLS cluster is not 1.")); + assertTrue("Message logged: " + handler.getAllFormattedMessage(), handler.hasInfoMessageWithSubString("Replicas specified in Domain spec is ignored because there number of configured WLS cluster is not 1.")); } finally { TestUtil.removeLogHandler(wlsDomainConfig, handler); } @@ -298,7 +298,7 @@ public void verifyUpdateDomainSpecWarnsIfClusterStatupReplicasTooLarge() throws try { handler = TestUtil.setupLogHandler(wlsClusterConfig); wlsDomainConfig.validate(domainSpec); - assertTrue("Message logged: " + handler.getAllFormattedMessage(), handler.hasWarningMessageWithSubString("replicas in clusterStartup for cluster DockerCluster2 is specified with a value of 3 which is larger than the number of configured WLS servers in the cluster: 2")); + assertTrue("Message logged: " + handler.getAllFormattedMessage(), handler.hasWarningMessageWithSubString("Replicas in clusterStartup for cluster DockerCluster2 is specified with a value of 3 which is larger than the number of configured WLS servers in the cluster: 2")); } finally { TestUtil.removeLogHandler(wlsClusterConfig, handler); } @@ -315,8 +315,8 @@ public void verifyUpdateDomainSpecWarnsIfClusterStatupReplicasTooLarge_2clusters try { handler = TestUtil.setupLogHandler(wlsClusterConfig); wlsDomainConfig.validate(domainSpec); - assertTrue("Message logged: " + handler.getAllFormattedMessage(), handler.hasWarningMessageWithSubString("replicas in clusterStartup for cluster DockerCluster is specified with a value of 10 which is larger than the number of configured WLS servers in the cluster: 3")); - assertTrue("Message logged: " + handler.getAllFormattedMessage(), handler.hasWarningMessageWithSubString("replicas in clusterStartup for cluster DockerCluster2 is specified with a value of 10 which is larger than the number of configured WLS servers in the cluster: 2")); + assertTrue("Message logged: " + handler.getAllFormattedMessage(), handler.hasWarningMessageWithSubString("Replicas in clusterStartup for cluster DockerCluster is specified with a value of 10 which is larger than the number of configured WLS servers in the cluster: 3")); + assertTrue("Message logged: " + handler.getAllFormattedMessage(), handler.hasWarningMessageWithSubString("Replicas in clusterStartup for cluster DockerCluster2 is specified with a value of 10 which is larger than the number of configured WLS servers in the cluster: 2")); } finally { TestUtil.removeLogHandler(wlsClusterConfig, handler); } From c25879a39e541752ae2e74952f1c78f772e279a6 Mon Sep 17 00:00:00 2001 From: Anthony Lai Date: Wed, 28 Feb 2018 15:17:08 -0800 Subject: [PATCH 007/186] Read WebLogic machines configuration --- kubernetes/create-domain-job-inputs.yaml | 2 +- .../operator/wlsconfig/WlsDomainConfig.java | 52 ++++++- .../operator/wlsconfig/WlsMachineConfig.java | 129 ++++++++++++++++++ .../wlsconfig/WlsDomainConfigTest.java | 50 ++++++- 4 files changed, 227 insertions(+), 6 deletions(-) create mode 100644 src/main/java/oracle/kubernetes/operator/wlsconfig/WlsMachineConfig.java diff --git a/kubernetes/create-domain-job-inputs.yaml b/kubernetes/create-domain-job-inputs.yaml index 44722183d1d..0e12d0bf13d 100644 --- a/kubernetes/create-domain-job-inputs.yaml +++ b/kubernetes/create-domain-job-inputs.yaml @@ -27,7 +27,7 @@ clusterName: cluster-1 managedServerCount: 2 # Number of managed servers to initially start for the domain -managedServerStartCount: 1 +managedServerStartCount: 2 # Base string used to generate managed server names managedServerNameBase: managed-server diff --git a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfig.java b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfig.java index ea1d39b45b9..006042935b1 100644 --- a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfig.java +++ b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfig.java @@ -24,6 +24,8 @@ public class WlsDomainConfig { private Map wlsServerConfigs = new HashMap<>(); // Contains all configured server templates in the WLS domain private Map wlsServerTemplates = new HashMap<>(); + // Contains all configured machines in the WLS domain + private Map wlsMachineConfigs = new HashMap<>(); // Name of this WLS domain (This is NOT the domain UID in the weblogic domain kubernetes CRD) private final String name; @@ -56,13 +58,16 @@ public WlsDomainConfig(String name) { * @param wlsClusterConfigs A Map containing clusters configured in this WLS domain * @param wlsServerConfigs A Map containing servers configured in the WLS domain * @param wlsServerTemplates A Map containing server templates configued in this WLS domain + * @param wlsMachineConfigs A Map containing machines configured in the WLS domain */ WlsDomainConfig(String name, Map wlsClusterConfigs, Map wlsServerConfigs, - Map wlsServerTemplates) { + Map wlsServerTemplates, + Map wlsMachineConfigs) { this.wlsClusterConfigs = wlsClusterConfigs; this.wlsServerConfigs = wlsServerConfigs; this.wlsServerTemplates = wlsServerTemplates; + this.wlsMachineConfigs = wlsMachineConfigs; this.name = name; } @@ -95,6 +100,15 @@ public synchronized Map getServerConfigs() { return wlsServerConfigs; } + /** + * Returns configuration of machines found in the WLS domain. + * + * @return A Map of WlsMachineConfig, keyed by name, for each machine configured the WLS domain + */ + public synchronized Map getMachineConfigs() { + return wlsMachineConfigs; + } + /** * Returns the configuration for the WLS cluster with the given name * @@ -130,6 +144,21 @@ public synchronized WlsServerConfig getServerConfig(String serverName) { return result; } + /** + * Returns the configuration for the WLS machine with the given name. + * + * @param machineName name of the WLS machine + * @return The WlsMachineConfig object containing configuration of the WLS machine with the given name. This methods + * return null if no WLS machine is configured with the given name. + */ + public synchronized WlsMachineConfig getMachineConfig(String machineName) { + WlsMachineConfig result = null; + if (machineName != null) { + result = wlsMachineConfigs.get(machineName); + } + return result; + } + /** * Create a new WlsDomainConfig object based on the parsed JSON result from WLS admin server * @@ -146,6 +175,7 @@ private static WlsDomainConfig create(ParsedJson parsedResult) { Map wlsClusterConfigs = new HashMap<>(); Map wlsServerConfigs = new HashMap<>(); Map wlsServerTemplates = new HashMap<>(); + Map wlsMachineConfigs = new HashMap<>(); // process list of server templates if (parsedResult.serverTemplates != null) { @@ -154,7 +184,7 @@ private static WlsDomainConfig create(ParsedJson parsedResult) { wlsServerTemplates.put(wlsServerTemplate.getName(), wlsServerTemplate); } } - // process list of clusters + // process list of clusters (Note: must process server templates before processing clusters) if (parsedResult.clusters != null) { for (Map clusterConfig : parsedResult.clusters) { WlsClusterConfig wlsClusterConfig = WlsClusterConfig.create(clusterConfig, wlsServerTemplates, name); @@ -173,7 +203,14 @@ private static WlsDomainConfig create(ParsedJson parsedResult) { } } } - return new WlsDomainConfig(name, wlsClusterConfigs, wlsServerConfigs, wlsServerTemplates); + // process list of machines + if (parsedResult.machines != null) { + for (Map machineConfig : parsedResult.machines) { + WlsMachineConfig wlsMachineConfig = WlsMachineConfig.create(machineConfig); + wlsMachineConfigs.put(wlsMachineConfig.getName(), wlsMachineConfig); + } + } + return new WlsDomainConfig(name, wlsClusterConfigs, wlsServerConfigs, wlsServerTemplates, wlsMachineConfigs); } public static String getRetrieveServersSearchUrl() { @@ -186,7 +223,8 @@ public static String getRetrieveServersSearchPayload() { " children: { " + " servers: { " + WlsServerConfig.getSearchPayload() + " }, " + " serverTemplates: { " + WlsServerConfig.getSearchPayload() + " }, " + - " clusters: { " + WlsClusterConfig.getSearchPayload() + " } " + + " clusters: { " + WlsClusterConfig.getSearchPayload() + " }, " + + " machines: { " + WlsMachineConfig.getSearchPayload() + " } " + " } " + "}"; } @@ -222,6 +260,10 @@ private static ParsedJson parseJson(String jsonString) { if (clusters != null) { parsedJson.clusters = (List>) clusters.get("items"); } + Map machines = (Map) result.get("machines"); + if (machines != null) { + parsedJson.machines = (List>) machines.get("items"); + } return parsedJson; } catch (Exception e) { LOGGER.warning(MessageKeys.JSON_PARSING_FAILED, jsonString, e.getMessage()); @@ -287,6 +329,7 @@ static class ParsedJson { List> servers; List> serverTemplates; List> clusters; + List> machines; } @Override @@ -295,6 +338,7 @@ public String toString() { "wlsClusterConfigs=" + wlsClusterConfigs + ", wlsServerConfigs=" + wlsServerConfigs + ", wlsServerTemplates=" + wlsServerTemplates + + ", wlsMachineConfigs=" + wlsMachineConfigs + ", name='" + name + '\'' + '}'; } diff --git a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsMachineConfig.java b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsMachineConfig.java new file mode 100644 index 00000000000..5bd0b710666 --- /dev/null +++ b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsMachineConfig.java @@ -0,0 +1,129 @@ +// Copyright 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.wlsconfig; + + +import java.util.Map; + +/** + * Contains values from a WLS machine configuration + *

+ */ +public class WlsMachineConfig { + + final String name; + final Integer nodeManagerListenPort; + final String nodeManagerListenAddress; + final String nodeManagerType; + + public WlsMachineConfig(String name, Integer nodeManagerListenPort, String nodeManagerListenAddress, String nodeManagerType) { + this.name = name; + this.nodeManagerListenPort = nodeManagerListenPort; + this.nodeManagerListenAddress = nodeManagerListenAddress; + this.nodeManagerType = nodeManagerType; + } + + /** + * Creates a WlsMachineConfig object using an "machines" item parsed from JSON result from WLS REST call + * + * @param machineConfigMap Map containing "machine" item parsed from JSON result from WLS REST call + * + * @return A new WlsMachineConfig object created based on the JSON result + */ + static WlsMachineConfig create(Map machineConfigMap) { + String machineName = (String) machineConfigMap.get("name"); + Map nodeManager = (Map) machineConfigMap.get("nodeManager"); + Integer nodeManagerListenPort = null; + String nodeManagerListenAddress = null; + String nodeManagerType = null; + if (nodeManager != null) { + nodeManagerListenAddress = (String) nodeManager.get("listenAddress"); + nodeManagerListenPort = (Integer) nodeManager.get("listenPort"); + nodeManagerType = (String) nodeManager.get("NMType"); + } + return new WlsMachineConfig(machineName, nodeManagerListenPort, nodeManagerListenAddress, nodeManagerType); + } + + /** + * + * @return Name of the machine that this WlsMachineConfig is created for + */ + public String getName() { + return name; + } + + /** + * + * @return Listen port of the node manager for the machine that this WlsMachineConfig is created for + */ + public Integer getNodeManagerListenPort() { + return nodeManagerListenPort; + } + + /** + * + * @return Listen address of the node manager for the machine that this WlsMachineConfig is created for + */ + public String getNodeManagerListenAddress() { + return nodeManagerListenAddress; + } + + /** + * + * @return Type of node manager (Plain, SSL, etc) for the machine that this WlsMachineConfig is created for + */ + public String getNodeManagerType() { + return nodeManagerType; + } + + /** + * Return the list of configuration attributes to be retrieved from the REST search request to the + * WLS admin server. The value would be used for constructing the REST POST request. + * + * @return The list of configuration attributes to be retrieved from the REST search request + * to the WLS admin server. The value would be used for constructing the REST POST request. + */ + static String getSearchPayload() { + return " fields: [ " + getSearchFields() + " ], " + + " links: [], " + + " children: { " + + " nodeManager: { " + + " fields: [ " + getNodeManagerSearchFields() + " ], " + + " links: [] " + + " }" + + " } "; + } + + /** + * Return the fields from machine WLS configuration that should be retrieved from the WLS REST + * request. + * + * @return A string containing machine configuration fields that should be retrieved from the WLS REST + * request, in a format that can be used in the REST request payload + */ + private static String getSearchFields() { + return "'name' "; + } + + /** + * Return the fields from node manager WLS configuration that should be retrieved from the WLS REST + * request. + * + * @return A string containing node manager configuration fields that should be retrieved from the WLS REST + * request, in a format that can be used in the REST request payload + */ + private static String getNodeManagerSearchFields() { + return "'listenAddress', 'listenPort', 'NMType' "; + } + + @Override + public String toString() { + return "WlsMachineConfig{" + + "name='" + name + '\'' + + ", nodeManagerListenPort=" + nodeManagerListenPort + + ", nodeManagerListenAddress='" + nodeManagerListenAddress + '\'' + + ", nodeManagerType='" + nodeManagerType + '\'' + + '}'; + } +} diff --git a/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfigTest.java b/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfigTest.java index 9b7586439dc..75636922ef1 100644 --- a/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfigTest.java +++ b/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfigTest.java @@ -100,6 +100,27 @@ public void verifyDynamicServersLoadedFromJsonString() throws Exception { assertEquals(6, wlsDomainConfig.getServerConfigs().size()); // does not include dynamic servers } + @Test + public void verifyMachinesLoadedFromJsonString() throws Exception { + WlsDomainConfig wlsDomainConfig = WlsDomainConfig.create(JSON_STRING_1_CLUSTER); + Map wlsMachineConfigList = wlsDomainConfig.getMachineConfigs(); + assertEquals(2, wlsMachineConfigList.size()); + + WlsMachineConfig domain1_machine1 = wlsDomainConfig.getMachineConfig("domain1-machine1"); + assertEquals("domain1-machine1", domain1_machine1.getName()); + assertEquals(new Integer(5556), domain1_machine1.getNodeManagerListenPort()); + assertEquals("domain1-managed-server1", domain1_machine1.getNodeManagerListenAddress()); + assertEquals("Plain", domain1_machine1.getNodeManagerType()); + + WlsMachineConfig domain1_machine2 = wlsDomainConfig.getMachineConfig("domain1-machine2"); + assertEquals("domain1-machine2", domain1_machine2.getName()); + assertEquals(new Integer(5556), domain1_machine2.getNodeManagerListenPort()); + assertEquals("domain1-managed-server2", domain1_machine2.getNodeManagerListenAddress()); + assertEquals("SSL", domain1_machine2.getNodeManagerType()); + + } + + @Test public void verifyGetServerConfigsDoesNotIncludeDynamicServers() throws Exception { WlsDomainConfig wlsDomainConfig = WlsDomainConfig.create(JSON_STRING_MIXED_CLUSTER); @@ -220,6 +241,12 @@ public void verifyGetServerConfigsReturnNullIfNotFound() throws Exception { assertNull(wlsDomainConfig.getServerConfig("noSuchServer")); } + @Test + public void verifyGetMachineConfigsReturnNullIfNotFound() throws Exception { + WlsDomainConfig wlsDomainConfig = new WlsDomainConfig(null); + assertNull(wlsDomainConfig.getMachineConfig("noSuchMachine")); + } + @Test public void verifyUpdateDomainSpecWarnsIfNoServersInClusterStartupCluster() throws Exception { WlsDomainConfig wlsDomainConfig = new WlsDomainConfig(null); @@ -599,7 +626,28 @@ private boolean containsNetworkAccessPoint(WlsServerConfig wlsServerConfig, Stri " ],\n" + " \"networkAccessPoints\": {\"items\": []}\n" + " }\n" + - "]}}"; + " ]}, " + + " \"machines\": {\"items\": [\n" + + " {\n" + + " \"name\": \"domain1-machine1\",\n" + + " \"nodeManager\": {\n" + + " \"NMType\": \"Plain\",\n" + + " \"listenAddress\": \"domain1-managed-server1\",\n" + + " \"name\": \"domain1-machine1\",\n" + + " \"listenPort\": 5556\n" + + " }\n" + + " },\n" + + " {\n" + + " \"name\": \"domain1-machine2\",\n" + + " \"nodeManager\": {\n" + + " \"NMType\": \"SSL\",\n" + + " \"listenAddress\": \"domain1-managed-server2\",\n" + + " \"name\": \"domain1-machine2\",\n" + + " \"listenPort\": 5556\n" + + " }\n" + + " }\n" + + " ]}\n" + + "}"; final String JSON_STRING_2_CLUSTERS = "{\"servers\": {\"items\": [\n" + " {\n" + From 4dff0e9783a48461a6c53a14e185cf9384348da3 Mon Sep 17 00:00:00 2001 From: Anthony Lai Date: Fri, 2 Mar 2018 15:29:36 -0800 Subject: [PATCH 008/186] create WLS machine if needed during updating dynamic cluster size --- .../operator/http/HTTPException.java | 16 ++ .../kubernetes/operator/http/HttpClient.java | 65 +++++-- .../kubernetes/operator/http/Result.java | 53 ++++++ .../operator/logging/MessageKeys.java | 1 + .../operator/wlsconfig/ConfigUpdate.java | 7 - .../operator/wlsconfig/WlsClusterConfig.java | 69 ++++++-- .../wlsconfig/WlsConfigRetriever.java | 166 ++++++++++++------ .../operator/wlsconfig/WlsDomainConfig.java | 13 +- .../wlsconfig/WlsDynamicServersConfig.java | 22 ++- .../operator/wlsconfig/WlsMachineConfig.java | 8 + src/main/resources/Operator.properties | 9 +- .../wlsconfig/WlsClusterConfigTest.java | 148 ++++++++++++---- .../wlsconfig/WlsConfigRetreiverTest.java | 60 ------- .../wlsconfig/WlsDomainConfigTest.java | 1 + 14 files changed, 441 insertions(+), 197 deletions(-) create mode 100644 src/main/java/oracle/kubernetes/operator/http/HTTPException.java create mode 100644 src/main/java/oracle/kubernetes/operator/http/Result.java delete mode 100644 src/test/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetreiverTest.java diff --git a/src/main/java/oracle/kubernetes/operator/http/HTTPException.java b/src/main/java/oracle/kubernetes/operator/http/HTTPException.java new file mode 100644 index 00000000000..559d45e5416 --- /dev/null +++ b/src/main/java/oracle/kubernetes/operator/http/HTTPException.java @@ -0,0 +1,16 @@ +// Copyright 2017, 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.http; + +/** + * Exception when a HTTP status code is received that indicates the request was not successful + */ +public class HTTPException extends Exception { + + final int statusCode; + + public HTTPException(int statusCode) { + this.statusCode = statusCode; + } +} diff --git a/src/main/java/oracle/kubernetes/operator/http/HttpClient.java b/src/main/java/oracle/kubernetes/operator/http/HttpClient.java index cb48c7382d1..83f3cfd326d 100644 --- a/src/main/java/oracle/kubernetes/operator/http/HttpClient.java +++ b/src/main/java/oracle/kubernetes/operator/http/HttpClient.java @@ -46,28 +46,60 @@ private HttpClient(Client httpClient, String encodedCredentials) { this.encodedCredentials = encodedCredentials; } - public String executeGetOnServiceClusterIP(String requestUrl, ClientHolder client, String serviceName, String namespace) { - String serviceURL = SERVICE_URL == null ? getServiceURL(client, serviceName, namespace) : SERVICE_URL; + /** + * Constructs a URL using the provided service URL and request URL, and use the resulting URL to issue a HTTP GET request + * + * @param requestUrl The request URL containing the request of the REST call + * @param serviceURL The service URL containing the host and port of the server where the HTTP + * request is to be sent to + * + * @return A Result object containing the respond from the REST call + */ + public Result executeGetOnServiceClusterIP(String requestUrl, String serviceURL) { String url = serviceURL + requestUrl; WebTarget target = httpClient.target(url); Invocation.Builder invocationBuilder = target.request().accept("application/json").header("Authorization", "Basic " + encodedCredentials); Response response = invocationBuilder.get(); + String responseString = null; + int status = response.getStatus(); + boolean successful = false; if (response.getStatusInfo().getFamily() == Response.Status.Family.SUCCESSFUL) { + successful = true; if (response.hasEntity()) { - return String.valueOf(response.readEntity(String.class)); + responseString = String.valueOf(response.readEntity(String.class)); } } else { LOGGER.warning(MessageKeys.HTTP_METHOD_FAILED, "GET", url, response.getStatus()); } - return null; + return new Result(responseString, status, successful); } - public String executePostUrlOnServiceClusterIP(String requestUrl, ClientHolder client, String serviceName, String namespace, String payload) { - String serviceURL = SERVICE_URL == null ? getServiceURL(client, serviceName, namespace) : SERVICE_URL; - return executePostUrlOnServiceClusterIP(requestUrl, serviceURL, payload); + public Result executePostUrlOnServiceClusterIP(String requestUrl, String serviceURL, String payload) { + Result result = null; + try { + result = executePostUrlOnServiceClusterIP(requestUrl, serviceURL, payload, false); + } catch (HTTPException httpException) { + // ignore as executePostUrlOnServiceClusterIP only throw HTTPException if throwOnFailure is true + } + return result; } - - public String executePostUrlOnServiceClusterIP(String requestUrl, String serviceURL, String payload) { + + /** + * Constructs a URL using the provided service URL and request URL, and use the resulting URL and the + * payload provided to issue a HTTP POST request + * + * @param requestUrl The request URL containing the request of the REST call + * @param serviceURL The service URL containing the host and port of the server where the HTTP + * request is to be sent to + * @param payload The payload to be used in the HTTP POST request + * @param throwOnFailure Throws HTTPException if the status code in the HTTP response indicates any error + * + * @return A Result object containing the respond from the REST call + * @throws HTTPException if throwOnFailure is true and the status of the HTTP response indicates the request was not + * successful + */ + public Result executePostUrlOnServiceClusterIP(String requestUrl, String serviceURL, String payload, + boolean throwOnFailure) throws HTTPException { String url = serviceURL + requestUrl; WebTarget target = httpClient.target(url); Invocation.Builder invocationBuilder = target.request().accept("application/json") @@ -75,19 +107,25 @@ public String executePostUrlOnServiceClusterIP(String requestUrl, String service .header("X-Requested-By", "Weblogic Operator"); Response response = invocationBuilder.post(Entity.json(payload)); LOGGER.finer("Response is " + response.getStatusInfo()); + String responseString = null; + int status = response.getStatus(); + boolean successful = false; if (response.getStatusInfo().getFamily() == Response.Status.Family.SUCCESSFUL) { + successful = true; if (response.hasEntity()) { - return String.valueOf(response.readEntity(String.class)); + responseString = String.valueOf(response.readEntity(String.class)); } } else { LOGGER.warning(MessageKeys.HTTP_METHOD_FAILED, "POST", url, response.getStatus()); + if (throwOnFailure) { + throw new HTTPException(status); + } } - return null; + return new Result(responseString, status, successful); } /** * Asynchronous {@link Step} for creating an authenticated HTTP client targeted at an admin server - * @param principal Principal * @param namespace Namespace * @param adminSecretName Admin secret name * @param next Next processing step @@ -189,6 +227,9 @@ public static HttpClient createAuthenticatedClient(final byte[] username, * @return The URL of the Service, or null if it is not found or principal does not have sufficient permissions. */ public static String getServiceURL(ClientHolder client, String name, String namespace) { + if (SERVICE_URL != null) { + return SERVICE_URL; + } try { return getServiceURL(client.callBuilder().readService(name, namespace)); } catch (ApiException e) { diff --git a/src/main/java/oracle/kubernetes/operator/http/Result.java b/src/main/java/oracle/kubernetes/operator/http/Result.java new file mode 100644 index 00000000000..46517766149 --- /dev/null +++ b/src/main/java/oracle/kubernetes/operator/http/Result.java @@ -0,0 +1,53 @@ +// Copyright 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.http; + +/** + * Holder of response received from REST requests invoked using methods in {@link HttpClient} class + */ +public class Result { + + final String response; + final int status; + final boolean successful; + + public Result(String response, int status, boolean successful) { + this.response = response; + this.status = status; + this.successful = successful; + } + + /** + * + * @return The String response received from the REST request + */ + public String getResponse() { + return response; + } + + /** + * + * @return HTTP status code from the REST request + */ + public int getStatus() { + return status; + } + + /** + * + * @return True if the REST request returns a status code that indicates successful request, false otherwise + */ + public boolean isSuccessful() { + return successful; + } + + @Override + public String toString() { + return "Result{" + + "response='" + response + '\'' + + ", status=" + status + + ", successful=" + successful + + '}'; + } +} diff --git a/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java b/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java index 978fbafcb4d..9311a98b8a3 100644 --- a/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java +++ b/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java @@ -142,4 +142,5 @@ private MessageKeys() { public static final String WLS_UPDATE_CLUSTER_SIZE_INVALID_CLUSTER = "WLSKO-0204"; public static final String WLS_CLUSTER_SIZE_UPDATED = "WLSKO-0205"; public static final String WLS_SERVER_TEMPLATE_NOT_FOUND = "WLSKO-0206"; + public static final String WLS_CREATING_MACHINE = "WLSKO-0207"; } diff --git a/src/main/java/oracle/kubernetes/operator/wlsconfig/ConfigUpdate.java b/src/main/java/oracle/kubernetes/operator/wlsconfig/ConfigUpdate.java index d2a4e4ce762..320a08b90b5 100644 --- a/src/main/java/oracle/kubernetes/operator/wlsconfig/ConfigUpdate.java +++ b/src/main/java/oracle/kubernetes/operator/wlsconfig/ConfigUpdate.java @@ -10,13 +10,6 @@ */ public interface ConfigUpdate { - /** - * Perform the suggested WebLogic configuration update - * @param wlsConfigRetriever The WlsConfigRetriever object to be used for performing the update - * @return true if the update was successful, false otherwise - */ - boolean doUpdate(WlsConfigRetriever wlsConfigRetriever); - /** * Create a Step to perform the suggested WebLogic configuration update * @param next Next Step to be performed after the WebLogic configuration update diff --git a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfig.java b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfig.java index d1393ffa163..fbe72dceed4 100644 --- a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfig.java +++ b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfig.java @@ -24,6 +24,7 @@ public class WlsClusterConfig { private final String clusterName; private List serverConfigs = new ArrayList<>(); private final WlsDynamicServersConfig dynamicServersConfig; + private WlsDomainConfig wlsDomainConfig; /** * Constructor for a static cluster when Json result is not available @@ -36,7 +37,7 @@ public WlsClusterConfig(String clusterName) { } /** - * Constructor for a dynamic cluster + * Constructor that can also be used for a dynamic cluster * * * @param clusterName Name of the WLS cluster * @param dynamicServersConfig A WlsDynamicServersConfig object containing the dynamic servers configuration for this @@ -96,6 +97,22 @@ public String getClusterName() { return clusterName; } + /** + * Associate this cluster to the WlsDomainConfig object for the WLS domain that this cluster belongs to + * @param wlsDomainConfig the WlsDomainConfig object for the WLS domain that this cluster belongs to + */ + public void setWlsDomainConfig(WlsDomainConfig wlsDomainConfig) { + this.wlsDomainConfig = wlsDomainConfig; + } + + /** + * Returns the WlsDomainConfig object for the WLS domain that this cluster belongs to + * @return the WlsDomainConfig object for the WLS domain that this cluster belongs to + */ + public WlsDomainConfig getWlsDomainConfig() { + return wlsDomainConfig; + } + /** * Returns a list of server configurations for servers that belong to this cluster, which includes * both statically configured servers and dynamic servers @@ -190,8 +207,7 @@ public boolean validateClusterStartup(ClusterStartup clusterStartup, * @param suggestedConfigUpdates A List containing suggested WebLogic configuration update to be filled in by this * method. Optional. */ - public void validateReplicas(Integer replicas, String source, - List suggestedConfigUpdates) { + public void validateReplicas(Integer replicas, String source, List suggestedConfigUpdates) { if (replicas == null) { return; } @@ -208,6 +224,34 @@ public void validateReplicas(Integer replicas, String source, } } + /** + * Finds the names of a machine to be created for newly created dynamic servers in this dynamic cluster + * + * @param machineNamePrefix Prefix for the new machine names (should match machineNameMatchExpression in dynamic servers config) + * @param targetClusterSize the target dynamic cluster size + * @return A String array containing names of new machines that are not currently in used in the WebLogic domain + */ + String[] getMachineNamesForNewDynamicServers(String machineNamePrefix, int targetClusterSize) { + if (targetClusterSize < 1 || !hasDynamicServers() || wlsDomainConfig == null) { + return new String[0]; + } + // machine names needed are [machineNamePrefix] appended by id of the new dynamic servers + // for example, if prefix is "domain1-cluster1-machine" and the current cluster size is 2, and targetClusterSize + // is 4, the ids of the new dynamic servers would be 3 and 4, + // the method should return {"domain1-cluster1-machine3", "domain1-cluster1-machine4"} + ArrayList names = new ArrayList<>(); + for (int suffix=getDynamicClusterSize() + 1; suffix <= targetClusterSize; suffix++) { + String newMachineName = machineNamePrefix == null? "" + suffix: machineNamePrefix + suffix; + if (wlsDomainConfig.getMachineConfig(newMachineName) == null) { + // only need to create machine if it does not already exist + names.add(newMachineName); + } + } + String[] machineNameArray = new String[names.size()]; + names.toArray(machineNameArray); + return machineNameArray; + } + /** * Return the list of configuration attributes to be retrieved from the REST search request to the * WLS admin server. The value would be used for constructing the REST POST request. @@ -292,26 +336,15 @@ static boolean checkUpdateDynamicClusterSizeJsonResult(String jsonResult) { * ConfigUpdate implementation for updating a dynamic cluster size */ static class DynamicClusterSizeConfigUpdate implements ConfigUpdate { - final int desiredClusterSize; + final int targetClusterSize; final WlsClusterConfig wlsClusterConfig; public DynamicClusterSizeConfigUpdate(WlsClusterConfig wlsClusterConfig, - int desiredClusterSize) { - this.desiredClusterSize = desiredClusterSize; + int targetClusterSize) { + this.targetClusterSize = targetClusterSize; this.wlsClusterConfig = wlsClusterConfig; } - /** - * Update the cluster size of a WebLogic dynamic cluster - * - * @param wlsConfigRetriever The WlsConfigRetriever object to be used for performing the update - * @return true if the update was successful, false otherwise - */ - @Override - public boolean doUpdate(WlsConfigRetriever wlsConfigRetriever) { - return wlsConfigRetriever.updateDynamicClusterSize(wlsClusterConfig, desiredClusterSize); - } - /** * Create a Step to update the cluster size of a WebLogic dynamic cluster * @param next Next Step to be performed after the WebLogic configuration update @@ -320,7 +353,7 @@ public boolean doUpdate(WlsConfigRetriever wlsConfigRetriever) { @Override public Step createStep(Step next) { - return new WlsConfigRetriever.UpdateDynamicClusterStep(wlsClusterConfig, desiredClusterSize, next); + return new WlsConfigRetriever.UpdateDynamicClusterStep(wlsClusterConfig, targetClusterSize, next); } } } diff --git a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetriever.java b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetriever.java index 7912cdcdb2f..27d6c35365c 100644 --- a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetriever.java +++ b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetriever.java @@ -8,6 +8,7 @@ import oracle.kubernetes.operator.helpers.ClientHelper; import oracle.kubernetes.operator.helpers.ClientHolder; import oracle.kubernetes.operator.helpers.DomainPresenceInfo; +import oracle.kubernetes.operator.http.HTTPException; import oracle.kubernetes.operator.http.HttpClient; import oracle.kubernetes.operator.logging.LoggingFacade; import oracle.kubernetes.operator.logging.LoggingFactory; @@ -54,6 +55,15 @@ public class WlsConfigRetriever { // wait time before retrying to read server configured for the cluster from admin server - default is 1s private static final int READ_CONFIG_RETRY_MILLIS = Integer.getInteger("read.config.retry.ms", 1000); + // REST URL for starting a WebLogic edit session + private static final String START_EDIT_SESSION_URL = "/management/weblogic/latest/edit/changeManager/startEdit"; + + // REST URL for activating a WebLogic edit session + private static final String ACTIVATE_EDIT_SESSION_URL = "/management/weblogic/latest/edit/changeManager/activate"; + + // REST URL for canceling a WebLogic edit session + private static final String CANCEL_EDIT_SESSION_URL = "/management/weblogic/latest/edit/changeManager/cancelEdit"; + /** * Constructor. * @@ -147,19 +157,20 @@ public NextAction apply(Packet packet) { static final class UpdateDynamicClusterStep extends Step { final WlsClusterConfig wlsClusterConfig; - final int desiredClusterSize; + final int targetClusterSize; /** * Constructor * * @param wlsClusterConfig The WlsClusterConfig object for the WebLogic dynamic cluster to be updated - * @param desiredClusterSize The desired dynamic cluster size + * @param targetClusterSize The target dynamic cluster size * @param next The next Step to be performed */ - public UpdateDynamicClusterStep(WlsClusterConfig wlsClusterConfig, int desiredClusterSize, Step next) { + public UpdateDynamicClusterStep(WlsClusterConfig wlsClusterConfig, + int targetClusterSize, Step next) { super(next); this.wlsClusterConfig = wlsClusterConfig; - this.desiredClusterSize = desiredClusterSize; + this.targetClusterSize = targetClusterSize; } /** @@ -174,7 +185,7 @@ public NextAction apply(Packet packet) { LOGGER.warning(MessageKeys.WLS_UPDATE_CLUSTER_SIZE_INVALID_CLUSTER, clusterName); } else { try { - LOGGER.info(MessageKeys.WLS_UPDATE_CLUSTER_SIZE_STARTING, clusterName, desiredClusterSize); + LOGGER.info(MessageKeys.WLS_UPDATE_CLUSTER_SIZE_STARTING, clusterName, targetClusterSize); HttpClient httpClient = (HttpClient) packet.get(HttpClient.KEY); DomainPresenceInfo info = packet.getSPI(DomainPresenceInfo.class); @@ -182,12 +193,15 @@ public NextAction apply(Packet packet) { String serviceURL = HttpClient.getServiceURL(info.getAdmin().getService().get()); - String jsonResult = httpClient.executePostUrlOnServiceClusterIP( - wlsClusterConfig.getUpdateDynamicClusterSizeUrl(), - serviceURL, wlsClusterConfig.getUpdateDynamicClusterSizePayload(desiredClusterSize)); + Domain dom = info.getDomain(); + DomainSpec domainSpec = dom.getSpec(); + String machineNamePrefix = domainSpec.getDomainUID() + "-" + wlsClusterConfig.getClusterName() + "-machine"; + + boolean successful = updateDynamicClusterSizeWithServiceURL(wlsClusterConfig, + machineNamePrefix, targetClusterSize, httpClient, serviceURL); - if (wlsClusterConfig.checkUpdateDynamicClusterSizeJsonResult(jsonResult)) { - LOGGER.info(MessageKeys.WLS_CLUSTER_SIZE_UPDATED, clusterName, desiredClusterSize, (System.currentTimeMillis() - startTime)); + if (successful) { + LOGGER.info(MessageKeys.WLS_CLUSTER_SIZE_UPDATED, clusterName, targetClusterSize, (System.currentTimeMillis() - startTime)); } else { LOGGER.warning(MessageKeys.WLS_UPDATE_CLUSTER_SIZE_FAILED, clusterName, null); } @@ -222,7 +236,9 @@ public NextAction apply(Packet packet) { String serviceURL = HttpClient.getServiceURL(info.getAdmin().getService().get()); WlsDomainConfig wlsDomainConfig = null; - String jsonResult = httpClient.executePostUrlOnServiceClusterIP(WlsDomainConfig.getRetrieveServersSearchUrl(), serviceURL, WlsDomainConfig.getRetrieveServersSearchPayload()); + String jsonResult = + httpClient.executePostUrlOnServiceClusterIP(WlsDomainConfig.getRetrieveServersSearchUrl(), + serviceURL, WlsDomainConfig.getRetrieveServersSearchPayload()).getResponse(); if (jsonResult != null) { wlsDomainConfig = WlsDomainConfig.create(jsonResult); } @@ -350,7 +366,8 @@ private String executePostUrlWithRetry(final String url, final String payload, f LOGGER.info(MessageKeys.WLS_CONFIGURATION_READ_TRYING, timeRemaining); exception = null; try { - jsonResult = executePostUrl(url, payload); + String serviceURL = connectAndGetServiceURL(); + jsonResult = httpClient.executePostUrlOnServiceClusterIP(url, serviceURL, payload).getResponse(); } catch (Exception e) { exception = e; LOGGER.info(MessageKeys.WLS_CONFIGURATION_READ_RETRY, e, READ_CONFIG_RETRY_MILLIS); @@ -373,42 +390,18 @@ private String executePostUrlWithRetry(final String url, final String payload, f return jsonResult; } - /** - * Invokes a HTTP POST request using the provided URL and payload - * - * @param url The URL of the HTTP post request to be invoked - * @param payload The payload of the HTTP Post request to be invoked - * - * @return The Json string returned from the HTTP POST request - * @throws Exception Any exception thrown while invoking the HTTP POST request - */ - private String executePostUrl(final String url, final String payload) - throws Exception { - LOGGER.entering(); - - String jsonResult = null; - ClientHolder client = null; - try { - client = clientHelper.take(); - - connectAdminServer(client); - jsonResult = httpClient.executePostUrlOnServiceClusterIP(url, client, asServiceName, namespace, payload); - } finally { - if (client != null) - clientHelper.recycle(client); - } - LOGGER.exiting(jsonResult); - return jsonResult; - } /** * Update the dynamic cluster size of the WLS cluster configuration. * * @param wlsClusterConfig The WlsClusterConfig object of the WLS cluster whose cluster size needs to be updated - * @param clusterSize The desire dynamic cluster size + * @param machineNamePrefix Prefix of names of new machines to be created + * @param targetClusterSize The target dynamic cluster size * @return true if the request to update the cluster size is successful, false if it was not successful within the * time period, or the cluster is not a dynamic cluster */ - public boolean updateDynamicClusterSize(final WlsClusterConfig wlsClusterConfig, int clusterSize) { + public boolean updateDynamicClusterSize(final WlsClusterConfig wlsClusterConfig, + final String machineNamePrefix, + final int targetClusterSize) { LOGGER.entering(); @@ -423,13 +416,13 @@ public boolean updateDynamicClusterSize(final WlsClusterConfig wlsClusterConfig, ExecutorService executorService = Executors.newSingleThreadExecutor(); long startTime = System.currentTimeMillis(); - Future future = executorService.submit(() -> doUpdateDynamicClusterSize(wlsClusterConfig, clusterSize)); + Future future = executorService.submit(() -> doUpdateDynamicClusterSize(wlsClusterConfig, machineNamePrefix, targetClusterSize)); executorService.shutdown(); boolean result = false; try { result = future.get(timeout, TimeUnit.MILLISECONDS); if (result) { - LOGGER.info(MessageKeys.WLS_CLUSTER_SIZE_UPDATED, clusterName, clusterSize, (System.currentTimeMillis() - startTime)); + LOGGER.info(MessageKeys.WLS_CLUSTER_SIZE_UPDATED, clusterName, targetClusterSize, (System.currentTimeMillis() - startTime)); } else { LOGGER.warning(MessageKeys.WLS_UPDATE_CLUSTER_SIZE_FAILED, clusterName, null); } @@ -448,33 +441,98 @@ public boolean updateDynamicClusterSize(final WlsClusterConfig wlsClusterConfig, * * @param wlsClusterConfig The WlsClusterConfig object of the WLS cluster whose cluster size needs to be updated. The * caller should make sure that the cluster is a dynamic cluster. - * @param clusterSize The desire dynamic cluster size + * @param wlsDomainConfig The WlsDomainConfig object for the WLS domain that contains the dynamic cluster to be updated + * @param machineNamePrefix Prefix of names of new machines to be created + * @param targetClusterSize The target dynamic cluster size * @return true if the request to update the cluster size is successful, false if it was not successful */ private boolean doUpdateDynamicClusterSize(final WlsClusterConfig wlsClusterConfig, - final int clusterSize) throws Exception { + final String machineNamePrefix, + final int targetClusterSize) throws Exception { + LOGGER.entering(); + + String serviceURL = connectAndGetServiceURL(); + + boolean result = updateDynamicClusterSizeWithServiceURL(wlsClusterConfig, machineNamePrefix, + targetClusterSize, httpClient, serviceURL); + + LOGGER.exiting(result); + return result; + } + + /** + * Static method to update the WebLogic dynamic cluster size configuration. + * + * @param wlsClusterConfig The WlsClusterConfig object of the WLS cluster whose cluster size needs to be updated. The + * caller should make sure that the cluster is a dynamic cluster. + * @param machineNamePrefix Prefix of names of new machines to be created + * @param targetClusterSize The target dynamic cluster size + * @param httpClient HttpClient object for issuing the REST request + * @param serviceURL service URL of the WebLogic admin server + * + * @return true if the request to update the cluster size is successful, false if it was not successful + */ + + private static boolean updateDynamicClusterSizeWithServiceURL(final WlsClusterConfig wlsClusterConfig, + final String machineNamePrefix, + final int targetClusterSize, + final HttpClient httpClient, + final String serviceURL) { LOGGER.entering(); + boolean result = false; + try { + // start a WebLogic edit session + httpClient.executePostUrlOnServiceClusterIP(START_EDIT_SESSION_URL, serviceURL, ""); + + // Create machine(s) + String newMachineNames[] = wlsClusterConfig.getMachineNamesForNewDynamicServers(machineNamePrefix, targetClusterSize); + for (String machineName: newMachineNames) { + LOGGER.info(MessageKeys.WLS_CREATING_MACHINE, machineName); + httpClient.executePostUrlOnServiceClusterIP(WlsMachineConfig.getCreateUrl(), + serviceURL, WlsMachineConfig.getCreatePayload(machineName), true); + } - String jsonResult = executePostUrl( - wlsClusterConfig.getUpdateDynamicClusterSizeUrl(), - wlsClusterConfig.getUpdateDynamicClusterSizePayload(clusterSize)); + // Update the dynamic cluster size of the WebLogic cluster + String jsonResult = httpClient.executePostUrlOnServiceClusterIP( + wlsClusterConfig.getUpdateDynamicClusterSizeUrl(), + serviceURL, + wlsClusterConfig.getUpdateDynamicClusterSizePayload(targetClusterSize), true).getResponse(); - boolean result = wlsClusterConfig.checkUpdateDynamicClusterSizeJsonResult(jsonResult); + // activate the WebLogic edit session + httpClient.executePostUrlOnServiceClusterIP(ACTIVATE_EDIT_SESSION_URL, serviceURL, "", true); + + result = wlsClusterConfig.checkUpdateDynamicClusterSizeJsonResult(jsonResult); + } catch (HTTPException httpException) { + // cancel the WebLogic edit session + httpClient.executePostUrlOnServiceClusterIP(CANCEL_EDIT_SESSION_URL, serviceURL, ""); + } LOGGER.exiting(result); return result; } /** - * Connect to the WebLogic Administration Server. + * Connect to the WebLogic Administration Server and returns the service URL * - * @param clientHolder The ClientHolder from which to get the Kubernetes API client. + * @return serviceURL for issuing HTTP requests to the admin server */ - public void connectAdminServer(ClientHolder clientHolder) { - if (httpClient == null) { - httpClient = HttpClient.createAuthenticatedClientForAdminServer(clientHolder, namespace, adminSecretName); + String connectAndGetServiceURL() { + ClientHolder clientHolder = null; + try { + clientHolder = clientHelper.take(); + + if (httpClient == null) { + httpClient = HttpClient.createAuthenticatedClientForAdminServer(clientHolder, namespace, adminSecretName); + } + + return HttpClient.getServiceURL(clientHolder, asServiceName, namespace); + + } finally { + if (clientHolder != null) + clientHelper.recycle(clientHolder); } } + } diff --git a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfig.java b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfig.java index 006042935b1..fcd5208e0f5 100644 --- a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfig.java +++ b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfig.java @@ -15,6 +15,9 @@ import java.util.List; import java.util.Map; +/** + * Contains a snapshot of WebLogic domain configuration + */ public class WlsDomainConfig { private static final LoggingFacade LOGGER = LoggingFactory.getLogger("Operator", "Operator"); @@ -69,6 +72,12 @@ public WlsDomainConfig(String name) { this.wlsServerTemplates = wlsServerTemplates; this.wlsMachineConfigs = wlsMachineConfigs; this.name = name; + // set domainConfig for each WlsClusterConfig + if (wlsClusterConfigs != null) { + for (WlsClusterConfig wlsClusterConfig: wlsClusterConfigs.values()) { + wlsClusterConfig.setWlsDomainConfig(this); + } + } } /** @@ -138,7 +147,7 @@ public synchronized WlsClusterConfig getClusterConfig(String clusterName) { */ public synchronized WlsServerConfig getServerConfig(String serverName) { WlsServerConfig result = null; - if (serverName != null) { + if (serverName != null && wlsServerConfigs != null) { result = wlsServerConfigs.get(serverName); } return result; @@ -153,7 +162,7 @@ public synchronized WlsServerConfig getServerConfig(String serverName) { */ public synchronized WlsMachineConfig getMachineConfig(String machineName) { WlsMachineConfig result = null; - if (machineName != null) { + if (machineName != null && wlsMachineConfigs != null) { result = wlsMachineConfigs.get(machineName); } return result; diff --git a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDynamicServersConfig.java b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDynamicServersConfig.java index ac9565a2212..8e11320b12c 100644 --- a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDynamicServersConfig.java +++ b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDynamicServersConfig.java @@ -24,6 +24,7 @@ public class WlsDynamicServersConfig { final String serverNamePrefix; final boolean calculatedListenPorts; final WlsServerConfig serverTemplate; + final String machineNameMatchExpression; List serverConfigs; /** @@ -43,6 +44,7 @@ static WlsDynamicServersConfig create(Map dynamicServerConfig, Map serverConfigs = null; if (dynamicServerConfig != null) { @@ -50,6 +52,7 @@ static WlsDynamicServersConfig create(Map dynamicServerConfig, Map serverConfigs) { + boolean calculatedListenPorts, String machineNameMatchExpression, + WlsServerConfig serverTemplate, List serverConfigs) { this.dynamicClusterSize = dynamicClusterSize; this.maxDynamicClusterSize = maxDynamicClusterSize; this.serverNamePrefix = serverNamePrefix; this.calculatedListenPorts = calculatedListenPorts; + this.machineNameMatchExpression = machineNameMatchExpression; this.serverTemplate = serverTemplate; this.serverConfigs = serverConfigs; } @@ -133,6 +138,14 @@ public Integer getMaxDynamicClusterSize() { return maxDynamicClusterSize; } + /** + * Return the expression used in matching machine names assigned to dynamic servers + * @return the expression used in matching machine names assigned to dynamic servers + */ + public String getMachineNameMatchExpression() { + return machineNameMatchExpression; + } + /** * Return list of WlsServerConfig objects containing configurations of WLS dynamic server that can be started under * the current cluster size @@ -182,7 +195,7 @@ public WlsServerConfig getServerTemplate() { * used in the payload to the REST call to WLS admin server */ static String getSearchFields() { - return "'serverTemplate', 'dynamicClusterSize', 'maxDynamicClusterSize', 'serverNamePrefix', 'calculatedListenPorts', 'dynamicServerNames' "; + return "'serverTemplate', 'dynamicClusterSize', 'maxDynamicClusterSize', 'serverNamePrefix', 'calculatedListenPorts', 'dynamicServerNames', 'machineNameMatchExpression' "; } @Override @@ -192,6 +205,7 @@ public String toString() { ", maxDynamicClusterSize=" + maxDynamicClusterSize + ", serverNamePrefix='" + serverNamePrefix + '\'' + ", calculatedListenPorts=" + calculatedListenPorts + + ", machineNameMatchExpression=" + machineNameMatchExpression + ", serverTemplate=" + serverTemplate + ", serverConfigs=" + serverConfigs + '}'; diff --git a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsMachineConfig.java b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsMachineConfig.java index 5bd0b710666..29c00d32ecb 100644 --- a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsMachineConfig.java +++ b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsMachineConfig.java @@ -106,6 +106,14 @@ private static String getSearchFields() { return "'name' "; } + static String getCreateUrl() { + return "/management/weblogic/latest/edit/machines"; + } + + static String getCreatePayload(String machineName) { + return " { name: \'" + machineName + "\' }"; + } + /** * Return the fields from node manager WLS configuration that should be retrieved from the WLS REST * request. diff --git a/src/main/resources/Operator.properties b/src/main/resources/Operator.properties index 8085970ea3e..75430ecb554 100644 --- a/src/main/resources/Operator.properties +++ b/src/main/resources/Operator.properties @@ -121,9 +121,10 @@ WLSKO-0119=Pod for domain with domainUID {0} in namespace {1} and with server na WLSKO-0120=Service for domain with domainUID {0} in namespace {1} and with server name {2} deleted; validating domain WLSKO-0121=Service for domain with domainUID {0} in namespace {1} and with cluster name {2} deleted; validating domain WLSKO-0122=Ingress for domain with domainUID {0} in namespace {1} and with cluster name {2} deleted; validating domain -WLSKO-0201=Updating cluster size for WebLogic dynamic cluster {0} to {1}. +WLSKO-0201=Updating cluster size for WebLogic dynamic cluster {0} to {1} WLSKO-0202=Failed to update WebLogic dynamic cluster size for cluster {0} due to exception: {1} WLSKO-0203=Failed to update WebLogic dynamic cluster size for cluster {0} within timeout of {1} milliseconds -WLSKO-0204=Failed to update WebLogic dynamic cluster size for cluster {0}. Cluster is not a dynamic cluster. -WLSKO-0205=Updated cluster size for WebLogic dynamic cluster {0} to {1}. Time taken {2} ms. -WLSKO-0206=Cannot find WebLogic server template with name {0} which is referenced by WebLogic cluster {1}. +WLSKO-0204=Failed to update WebLogic dynamic cluster size for cluster {0}. Cluster is not a dynamic cluster +WLSKO-0205=Updated cluster size for WebLogic dynamic cluster {0} to {1}. Time taken {2} ms +WLSKO-0206=Cannot find WebLogic server template with name {0} which is referenced by WebLogic cluster {1} +WLSKO-0207=Creating WebLogic machine named {0} diff --git a/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfigTest.java b/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfigTest.java index b9f65955b4a..4122de179ca 100644 --- a/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfigTest.java +++ b/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfigTest.java @@ -24,9 +24,6 @@ import static org.junit.Assert.*; -/** - * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. - */ public class WlsClusterConfigTest { private static final Logger UNDERLYING_LOGGER = LoggingFactory.getLogger("Operator", "Operator").getUnderlyingLogger(); @@ -265,19 +262,6 @@ public void verifyGetUpdateDynamicClusterSizePayload3() { assertEquals(wlsClusterConfig.getUpdateDynamicClusterSizePayload(8), "{ dynamicClusterSize: 8, maxDynamicClusterSize: 8 }"); } - @Test - public void verifyDynamicClusterSizeConfigUpdateCallsUpdateDynamicClusterSize() { - final WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1"); - final int clusterSize = 8; - WlsClusterConfig.DynamicClusterSizeConfigUpdate dynamicClusterSizeConfigUpdate = - new WlsClusterConfig.DynamicClusterSizeConfigUpdate(wlsClusterConfig, clusterSize); - MockWlsConfigRetriever mockWlsConfigRetriever = new MockWlsConfigRetriever(null, "namespace", - "asServiceName", "adminSecretName"); - assertTrue(dynamicClusterSizeConfigUpdate.doUpdate(mockWlsConfigRetriever)); - assertSame(wlsClusterConfig, mockWlsConfigRetriever.wlsClusterConfigParamValue); - assertEquals(clusterSize, mockWlsConfigRetriever.clusterSizeParamvalue); - } - @Test public void verifyStepCreatedFromDynamicClusterSizeConfigUpdate() throws NoSuchFieldException, IllegalAccessException { final WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1"); @@ -289,7 +273,7 @@ public void verifyStepCreatedFromDynamicClusterSizeConfigUpdate() throws NoSuchF WlsConfigRetriever.UpdateDynamicClusterStep updateStep = (WlsConfigRetriever.UpdateDynamicClusterStep) dynamicClusterSizeConfigUpdate.createStep(nextStep); assertSame(wlsClusterConfig, updateStep.wlsClusterConfig); - assertEquals(clusterSize, updateStep.desiredClusterSize); + assertEquals(clusterSize, updateStep.targetClusterSize); assertSame(nextStep, getNext(updateStep)); } @@ -318,7 +302,116 @@ private WlsServerConfig createWlsServerConfig(String serverName, Integer listenP return WlsServerConfig.create(serverConfigMap); } - private WlsDynamicServersConfig createDynamicServersConfig(int clusterSize, int maxClusterSize, + @Test + public void verifyGetMachinesNameReturnsExpectedMachineName() { + WlsMachineConfig machine1 = new WlsMachineConfig("domain1-machine1", 5556, "localhost", "SSL"); + WlsMachineConfig machine2 = new WlsMachineConfig("domain1-machine2", 5556, "localhost", "SSL"); + Map machines = new HashMap(); + machines.put(machine1.getName(), machine1); + machines.put(machine2.getName(), machine2); + + WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1", + WlsClusterConfigTest.createDynamicServersConfig(2, 5, "ms-", "cluster1")); + Map clusters = new HashMap(); + clusters.put(wlsClusterConfig.getClusterName(), wlsClusterConfig); + + WlsDomainConfig wlsDomainConfig = new WlsDomainConfig("base_domain", clusters, null, null, machines); + wlsClusterConfig.setWlsDomainConfig(wlsDomainConfig); + + String names[] = wlsClusterConfig.getMachineNamesForNewDynamicServers("domain1-machine", 4); + assertEquals(2, names.length); + assertEquals("domain1-machine3", names[0]); + assertEquals("domain1-machine4", names[1]); + } + + @Test + public void verifyGetMachinesNameDoesNotReturnExistingMachines() { + WlsMachineConfig machine1 = new WlsMachineConfig("domain1-machine1", 5556, "localhost", "SSL"); + WlsMachineConfig machine2 = new WlsMachineConfig("domain1-machine2", 5556, "localhost", "SSL"); + WlsMachineConfig machine3 = new WlsMachineConfig("domain1-machine3", 5556, "localhost", "SSL"); + Map machines = new HashMap(); + machines.put(machine1.getName(), machine1); + machines.put(machine2.getName(), machine2); + machines.put(machine3.getName(), machine3); + + WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1", + WlsClusterConfigTest.createDynamicServersConfig(1, 5, "ms-", "cluster1")); + Map clusters = new HashMap(); + clusters.put(wlsClusterConfig.getClusterName(), wlsClusterConfig); + + WlsDomainConfig wlsDomainConfig = new WlsDomainConfig("base_domain", clusters, null, null, machines); + wlsClusterConfig.setWlsDomainConfig(wlsDomainConfig); + + String names[] = wlsClusterConfig.getMachineNamesForNewDynamicServers("domain1-machine", 3); + assertEquals(0, names.length); + } + + @Test + public void verifyGetMachinesNameReturnsEmptyArrayIfNoDomainConfig() { + WlsMachineConfig machine1 = new WlsMachineConfig("domain1-machine1", 5556, "localhost", "SSL"); + WlsMachineConfig machine2 = new WlsMachineConfig("domain1-machine2", 5556, "localhost", "SSL"); + Map machines = new HashMap(); + machines.put(machine1.getName(), machine1); + machines.put(machine2.getName(), machine2); + + WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1", + WlsClusterConfigTest.createDynamicServersConfig(2, 5, "ms-", "cluster1")); + + assertNull("verify no domain config is setup", wlsClusterConfig.getWlsDomainConfig()); + + String names[] = wlsClusterConfig.getMachineNamesForNewDynamicServers("domain1-machine", 4); + assertEquals(0, names.length); + } + + + @Test + public void verifyGetMachineNamesNoExceptionWhenPrefixIsNull() { + WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1", + WlsClusterConfigTest.createDynamicServersConfig(1, 5, "ms-", "cluster1")); + Map clusters = new HashMap(); + clusters.put(wlsClusterConfig.getClusterName(), wlsClusterConfig); + + WlsDomainConfig wlsDomainConfig = new WlsDomainConfig("base_domain", clusters, null, null, null); + wlsClusterConfig.setWlsDomainConfig(wlsDomainConfig); + + String[] names = wlsClusterConfig.getMachineNamesForNewDynamicServers(null, 2); + assertEquals(1, names.length); + assertEquals("2", names[0]); + } + + @Test + public void verifyGetMachineNamesReturnsEmptyArrayWhenNumIsInvalid() { + WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1", + WlsClusterConfigTest.createDynamicServersConfig(0, 5, "ms-", "clsuter1")); + Map clusters = new HashMap(); + clusters.put(wlsClusterConfig.getClusterName(), wlsClusterConfig); + + WlsDomainConfig wlsDomainConfig = new WlsDomainConfig("base_domain", clusters, null, null, null); + wlsClusterConfig.setWlsDomainConfig(wlsDomainConfig); + String[] names = wlsClusterConfig.getMachineNamesForNewDynamicServers("domain1-machine", 0); + assertEquals(0, names.length); + + names = wlsClusterConfig.getMachineNamesForNewDynamicServers("domain1-machine", -1); + assertEquals(0, names.length); + } + + @Test + public void verifyGetMachineNamesReturnsEmptyArrayWhenTargetSizeIsNotLarger() { + WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1", + WlsClusterConfigTest.createDynamicServersConfig(2, 5, "ms-", "clsuter1")); + Map clusters = new HashMap(); + clusters.put(wlsClusterConfig.getClusterName(), wlsClusterConfig); + + WlsDomainConfig wlsDomainConfig = new WlsDomainConfig("base_domain", clusters, null, null, null); + wlsClusterConfig.setWlsDomainConfig(wlsDomainConfig); + String[] names = wlsClusterConfig.getMachineNamesForNewDynamicServers("domain1-machine", 2); + assertEquals(0, names.length); + + names = wlsClusterConfig.getMachineNamesForNewDynamicServers("domain1-machine", 1); + assertEquals(0, names.length); + } + + static WlsDynamicServersConfig createDynamicServersConfig(int clusterSize, int maxClusterSize, String serverNamePrefix, String clusterName) { WlsServerConfig serverTemplate = new WlsServerConfig("serverTemplate1", 7001, "host1", 7002, false, null, null); @@ -331,25 +424,8 @@ private WlsDynamicServersConfig createDynamicServersConfig(int clusterSize, int serverNames, serverTemplate, clusterName, "base-domain", false); return new WlsDynamicServersConfig(clusterSize, maxClusterSize, serverNamePrefix, - false, serverTemplate, serverConfigs); - - } + false, null, serverTemplate, serverConfigs); - static class MockWlsConfigRetriever extends WlsConfigRetriever { - - WlsClusterConfig wlsClusterConfigParamValue; - int clusterSizeParamvalue; - - public MockWlsConfigRetriever(ClientHelper clientHelper, String namespace, String asServiceName, String adminSecretName) { - super(clientHelper, namespace, asServiceName, adminSecretName); - } - - @Override - public boolean updateDynamicClusterSize(WlsClusterConfig wlsClusterConfig, int clusterSize) { - wlsClusterConfigParamValue = wlsClusterConfig; - clusterSizeParamvalue = clusterSize; - return true; - } } static class MockStep extends Step { diff --git a/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetreiverTest.java b/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetreiverTest.java deleted file mode 100644 index fff4edd9c70..00000000000 --- a/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetreiverTest.java +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2017, Oracle Corporation and/or its affiliates. All rights reserved. -// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. - -package oracle.kubernetes.operator.wlsconfig; - -import oracle.kubernetes.operator.helpers.ClientHelper; -import oracle.kubernetes.operator.helpers.ClientHolder; -import oracle.kubernetes.operator.http.HttpClient; -import oracle.kubernetes.operator.logging.LoggingFacade; -import oracle.kubernetes.operator.logging.LoggingFactory; -import org.junit.Ignore; -import org.junit.Test; - - -@Ignore -public class WlsConfigRetreiverTest { - - private static final LoggingFacade LOGGER = LoggingFactory.getLogger("Operator", "Operator"); - - //@Test - public void testWlsConfigRetriever() { - - final String namespace = "default"; - final String SECRET_NAME = "wls-admin-server-credentials"; // TODO: will be getting this from configuration - String principal = "system:serviceaccount:default:weblogic-operator"; - - // set the timeout to a reasonable value so that the build is not held up by this test - System.setProperty("read.config.timeout.ms", "3000"); - - ClientHelper clientHelper = ClientHelper.getInstance(); - ClientHolder client = null; - - try { - WlsDomainConfig wlsDomainConfig = new WlsConfigRetriever(clientHelper, namespace, "wls-admin-service", SECRET_NAME).readConfig(); - - LOGGER.finer("Read config " + wlsDomainConfig); - - LOGGER.finer("sleeping....."); - Thread.sleep(10000); - - client = clientHelper.take(); - - LOGGER.finer("--- trying update REST call ---"); - HttpClient httpClient = HttpClient.createAuthenticatedClientForAdminServer(client, namespace, SECRET_NAME); - String url = "/management/weblogic/latest/edit/servers/ms-3"; - String payload = "{listenAddress: 'ms-3.wls-subdomain.default.svc.cluster.local'}"; - String result = httpClient.executePostUrlOnServiceClusterIP(url, client, "wls-admin-service", "default", payload); - LOGGER.finer("REST call returns: " + result); - - LOGGER.finer("Read config again: " + new WlsConfigRetriever(clientHelper, "default", "wls-admin-service", SECRET_NAME).readConfig()); - } catch (Exception e) { - LOGGER.finer("namespace query failed: " + e); - e.printStackTrace(); - } finally { - LOGGER.finer("End of this test"); - if (client != null) - clientHelper.recycle(client); - } - } -} diff --git a/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfigTest.java b/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfigTest.java index 75636922ef1..76ec28921cf 100644 --- a/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfigTest.java +++ b/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfigTest.java @@ -10,6 +10,7 @@ import org.junit.Before; import org.junit.Test; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.logging.Handler; From adee8d5f4a4378cac8a707f1784bf4b7ec7c2dd5 Mon Sep 17 00:00:00 2001 From: Anthony Lai Date: Fri, 2 Mar 2018 15:48:04 -0800 Subject: [PATCH 009/186] logging for edit sessions --- .../java/oracle/kubernetes/operator/logging/MessageKeys.java | 3 +++ .../kubernetes/operator/wlsconfig/WlsConfigRetriever.java | 3 +++ src/main/resources/Operator.properties | 3 +++ 3 files changed, 9 insertions(+) diff --git a/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java b/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java index 9311a98b8a3..c900a059881 100644 --- a/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java +++ b/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java @@ -143,4 +143,7 @@ private MessageKeys() { public static final String WLS_CLUSTER_SIZE_UPDATED = "WLSKO-0205"; public static final String WLS_SERVER_TEMPLATE_NOT_FOUND = "WLSKO-0206"; public static final String WLS_CREATING_MACHINE = "WLSKO-0207"; + public static final String WLS_EDIT_SESSION_STARTED = "WLSKO-0208"; + public static final String WLS_EDIT_SESSION_ACTIVATED = "WLSKO-0209"; + public static final String WLS_EDIT_SESSION_CANCELLED = "WLSKO-0210"; } diff --git a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetriever.java b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetriever.java index 27d6c35365c..1f104922be9 100644 --- a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetriever.java +++ b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetriever.java @@ -485,6 +485,7 @@ private static boolean updateDynamicClusterSizeWithServiceURL(final WlsClusterCo try { // start a WebLogic edit session httpClient.executePostUrlOnServiceClusterIP(START_EDIT_SESSION_URL, serviceURL, ""); + LOGGER.info(MessageKeys.WLS_EDIT_SESSION_STARTED); // Create machine(s) String newMachineNames[] = wlsClusterConfig.getMachineNamesForNewDynamicServers(machineNamePrefix, targetClusterSize); @@ -504,9 +505,11 @@ private static boolean updateDynamicClusterSizeWithServiceURL(final WlsClusterCo httpClient.executePostUrlOnServiceClusterIP(ACTIVATE_EDIT_SESSION_URL, serviceURL, "", true); result = wlsClusterConfig.checkUpdateDynamicClusterSizeJsonResult(jsonResult); + LOGGER.info(MessageKeys.WLS_EDIT_SESSION_ACTIVATED); } catch (HTTPException httpException) { // cancel the WebLogic edit session httpClient.executePostUrlOnServiceClusterIP(CANCEL_EDIT_SESSION_URL, serviceURL, ""); + LOGGER.info(MessageKeys.WLS_EDIT_SESSION_CANCELLED); } LOGGER.exiting(result); return result; diff --git a/src/main/resources/Operator.properties b/src/main/resources/Operator.properties index 75430ecb554..8eec02be242 100644 --- a/src/main/resources/Operator.properties +++ b/src/main/resources/Operator.properties @@ -128,3 +128,6 @@ WLSKO-0204=Failed to update WebLogic dynamic cluster size for cluster {0}. Clust WLSKO-0205=Updated cluster size for WebLogic dynamic cluster {0} to {1}. Time taken {2} ms WLSKO-0206=Cannot find WebLogic server template with name {0} which is referenced by WebLogic cluster {1} WLSKO-0207=Creating WebLogic machine named {0} +WLSKO-0208=WebLogic edit session started +WLSKO-0209=WebLogic edit session activated +WLSKO-0210=WebLogic edit session cancelled From f0209a4b85d65782bd5e8f6bcc4c6e456e0186d0 Mon Sep 17 00:00:00 2001 From: anthony_lai Date: Mon, 5 Mar 2018 08:56:39 -0800 Subject: [PATCH 010/186] expose node manager port as ClusterIP --- .../kubernetes/operator/helpers/PodHelper.java | 14 ++++++++++++++ .../kubernetes/operator/helpers/ServiceHelper.java | 7 +++++++ 2 files changed, 21 insertions(+) diff --git a/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java b/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java index 0d3bebd577c..fcc512d0425 100644 --- a/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java +++ b/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java @@ -130,8 +130,15 @@ public NextAction apply(Packet packet) { V1ContainerPort containerPort = new V1ContainerPort(); containerPort.setContainerPort(spec.getAsPort()); containerPort.setProtocol("TCP"); + containerPort.setName("weblogic"); container.addPortsItem(containerPort); + V1ContainerPort nmPort = new V1ContainerPort(); + nmPort.setContainerPort(5556); + nmPort.setProtocol("TCP"); + nmPort.setName("node-manager"); + container.addPortsItem(nmPort); + V1Lifecycle lifecycle = new V1Lifecycle(); V1Handler preStopHandler = new V1Handler(); V1ExecAction lifecycleExecAction = new V1ExecAction(); @@ -474,8 +481,15 @@ public NextAction apply(Packet packet) { V1ContainerPort containerPort = new V1ContainerPort(); containerPort.setContainerPort(scan.getListenPort()); containerPort.setProtocol("TCP"); + containerPort.setName("weblogic"); container.addPortsItem(containerPort); + V1ContainerPort nmPort = new V1ContainerPort(); + nmPort.setContainerPort(5556); + nmPort.setProtocol("TCP"); + nmPort.setName("node-manager"); + container.addPortsItem(nmPort); + V1Lifecycle lifecycle = new V1Lifecycle(); V1Handler preStop = new V1Handler(); V1ExecAction exec = new V1ExecAction(); diff --git a/src/main/java/oracle/kubernetes/operator/helpers/ServiceHelper.java b/src/main/java/oracle/kubernetes/operator/helpers/ServiceHelper.java index 399323ae9ae..1d2c113ae69 100644 --- a/src/main/java/oracle/kubernetes/operator/helpers/ServiceHelper.java +++ b/src/main/java/oracle/kubernetes/operator/helpers/ServiceHelper.java @@ -90,10 +90,17 @@ public NextAction apply(Packet packet) { List ports = new ArrayList<>(); V1ServicePort servicePort = new V1ServicePort(); servicePort.setPort(port); + servicePort.setName("weblogic"); if (nodePort != null) { servicePort.setNodePort(nodePort); } ports.add(servicePort); + + V1ServicePort nmPort = new V1ServicePort(); + nmPort.setPort(5556); + nmPort.setName("node-manager"); + ports.add(nmPort); + serviceSpec.setPorts(ports); service.setSpec(serviceSpec); From 27d3f9db10b8333b9b50132fd690193526af6bc0 Mon Sep 17 00:00:00 2001 From: anthony_lai Date: Tue, 6 Mar 2018 11:20:31 -0800 Subject: [PATCH 011/186] create wls machine for dynamic servers and assign listen address on startup script, no longer create machines for dynamic servers in create domain job --- kubernetes/internal/domain-job-template.yaml | 73 +++++-- .../kubernetes/operator/wlsconfig/Util.java | 27 +++ .../operator/wlsconfig/WlsClusterConfig.java | 65 +++++-- .../wlsconfig/WlsConfigRetriever.java | 4 +- .../operator/wlsconfig/WlsDomainConfig.java | 7 +- .../operator/wlsconfig/UtilTest.java | 31 +++ .../wlsconfig/WlsClusterConfigTest.java | 182 ++++++++++++++---- 7 files changed, 320 insertions(+), 69 deletions(-) create mode 100644 src/main/java/oracle/kubernetes/operator/wlsconfig/Util.java create mode 100644 src/test/java/oracle/kubernetes/operator/wlsconfig/UtilTest.java diff --git a/kubernetes/internal/domain-job-template.yaml b/kubernetes/internal/domain-job-template.yaml index e1e89cc96be..b8e5e32a2c6 100644 --- a/kubernetes/internal/domain-job-template.yaml +++ b/kubernetes/internal/domain-job-template.yaml @@ -129,7 +129,7 @@ data: server_name=\$2 as_name=\$3 as_port=\$4 - as_hostname="\$1-\$3" + as_hostname=\$1-\$3 echo "debug arguments are \$1 \$2 \$3 \$4" @@ -235,8 +235,13 @@ data: echo "Update JVM arguments" echo "Arguments=\${USER_MEM_ARGS} -XX\:+UnlockExperimentalVMOptions -XX\:+UseCGroupMemoryLimitForHeap \${JAVA_OPTIONS}" >> \${startProp} + admin_server_t3_url= + if [ -n "\$3" ]; then + admin_server_t3_url=t3://\$domain_uid-\$as_name:\$as_port + fi + echo "Start the server" - wlst.sh -skipWLSModuleScanning ${pyFile} \$domain_uid \$server_name + wlst.sh -skipWLSModuleScanning ${pyFile} \$domain_uid \$server_name \$admin_server_t3_url echo "Wait indefinitely so that the Kubernetes pod does not exit and try to restart" while true; do sleep 60; done @@ -253,12 +258,17 @@ data: domain_uid = sys.argv[1] server_name = sys.argv[2] + if (len(sys.argv) == 4): + admin_server_url = sys.argv[3] + else: + admin_server_url = None domain_path= "${DOMAIN_HOME}" print 'admin username is %s' % admin_username print 'domain path is %s' % domain_path print 'server name is %s' % server_name + print 'admin server url is %s' % admin_server_url # Encrypt the admin username and password adminUsernameEncrypted=encrypt(admin_username, domain_path) @@ -286,11 +296,41 @@ data: service_name = domain_uid + "-" + server_name # Connect to nodemanager and start server - nmConnect(admin_username, admin_password, service_name, '5556', '%DOMAIN_NAME%', '${DOMAIN_HOME}', 'plain') - nmStart(server_name) + try: + nmConnect(admin_username, admin_password, service_name, '5556', '%DOMAIN_NAME%', '${DOMAIN_HOME}', 'plain') + nmStart(server_name) + nmDisconnect() + except WLSTException, e: + nmDisconnect() + print e + + # Update node manager listen address + if admin_server_url is not None: + connect(admin_username, admin_password, admin_server_url) + serverConfig() + server=cmo.lookupServer(server_name) + machineName=server.getMachine().getName() + + print 'Name of machine assigned to server %s is %s' % (server_name, machineName) + + if machineName is not None: + print 'Updating listen address of machine %s' % machineName + try: + edit() + startEdit() + cd('/') + machine=cmo.lookupMachine(machineName) + print 'Machine is %s' % machine + nm=machine.getNodeManager() + nm.setListenAddress(service_name) + nm.setNMType('Plain') + activate() + print 'Updated listen address of machine %s to %s' % (machineName, service_name) + except: + cancelEdit() + raise # Exit WLST - nmDisconnect() exit() EOF @@ -475,16 +515,19 @@ data: # Configure machines # ====================== - for index in range(0, number_of_ms): + if cluster_type == "configured": + + for index in range(0, number_of_ms): - msIndex = index+1 - machineName = '%DOMAIN_UID%-machine%s' % msIndex - cd('/') - create(machineName, 'Machine') - cd('Machine/%s' % machineName) - create(machineName, 'NodeManager') - cd('NodeManager/%s' % machineName) - set('ListenAddress', '%DOMAIN_UID%-%MANAGED_SERVER_NAME_BASE%%s' % msIndex) + msIndex = index+1 + machineName = '%DOMAIN_UID%-machine%s' % msIndex + cd('/') + create(machineName, 'Machine') + cd('Machine/%s' % machineName) + create(machineName, 'NodeManager') + cd('NodeManager/%s' % machineName) + set('ListenAddress', '%DOMAIN_UID%-%MANAGED_SERVER_NAME_BASE%%s' % msIndex) + set('NMType', 'Plain') # Create a cluster cd('/') @@ -536,6 +579,8 @@ data: set('MaxDynamicClusterSize', 800) set('CalculatedMachineNames', true) set('CalculatedListenPorts', false) + machineNameExpression = '%DOMAIN_UID%-%s-machine*' % cluster_name + set('MachineNameMatchExpression', machineNameExpression) set('Id', 1) print('Done setting attributes for Dynamic Cluster: %s' % cluster_name); diff --git a/src/main/java/oracle/kubernetes/operator/wlsconfig/Util.java b/src/main/java/oracle/kubernetes/operator/wlsconfig/Util.java new file mode 100644 index 00000000000..99056a3cb12 --- /dev/null +++ b/src/main/java/oracle/kubernetes/operator/wlsconfig/Util.java @@ -0,0 +1,27 @@ +// Copyright 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. +package oracle.kubernetes.operator.wlsconfig; + +import oracle.kubernetes.operator.domain.model.oracle.kubernetes.weblogic.domain.v1.DomainSpec; + +/** + * Utility class for WebLogic configuration related classes + */ +public class Util { + + /** + * Return prefix for names of machines that will be used servers in a dynamic cluster. + * The prefix would be "[domainUID]-[clusterName]-machine" + * + * @param domainSpec DomainSpec for the weblogic operator domain + * @param wlsClusterConfig WlsClusterConfig object containing the dynamic cluster + * @return Prefix for names of machines that will be used servers in a dynamic cluster + */ + static String getMachineNamePrefix(DomainSpec domainSpec, WlsClusterConfig wlsClusterConfig) { + if (domainSpec != null && domainSpec.getDomainUID() != null + && wlsClusterConfig != null && wlsClusterConfig.getClusterName() != null) { + return domainSpec.getDomainUID() + "-" + wlsClusterConfig.getClusterName() + "-machine"; + } + return null; + } +} diff --git a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfig.java b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfig.java index fbe72dceed4..c523b982138 100644 --- a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfig.java +++ b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfig.java @@ -172,11 +172,14 @@ public int getMaxDynamicClusterSize() { * It is the responsibility of the caller to persist the changes to ClusterStartup to kubernetes. * * @param clusterStartup The ClusterStartup to be validated against the WLS configuration + * @param machineNamePrefix Optional, if this is not null, also validate whether the WebLogic domain already contains + * all the machines that will be used by the dynamic cluster * @param suggestedConfigUpdates A List containing suggested WebLogic configuration update to be filled in by this * method. Optional. * @return true if the DomainSpec has been updated, false otherwise */ public boolean validateClusterStartup(ClusterStartup clusterStartup, + String machineNamePrefix, List suggestedConfigUpdates) { LOGGER.entering(); @@ -188,7 +191,7 @@ public boolean validateClusterStartup(ClusterStartup clusterStartup, } // Warns if replicas is larger than the number of servers configured in the cluster - validateReplicas(clusterStartup.getReplicas(), "clusterStartup", suggestedConfigUpdates); + validateReplicas(clusterStartup.getReplicas(), machineNamePrefix,"clusterStartup", suggestedConfigUpdates); LOGGER.exiting(modified); @@ -202,12 +205,15 @@ public boolean validateClusterStartup(ClusterStartup clusterStartup, * * @param replicas The configured replicas value for this cluster in the kubernetes weblogic domain spec * for this cluster + * @param machineNamePrefix Optional, if this is not null, also validate whether the WebLogic domain already contains + * all the machines that will be used by the dynamic cluster * @param source The name of the section in the domain spec where the replicas is specified, * for logging purposes * @param suggestedConfigUpdates A List containing suggested WebLogic configuration update to be filled in by this * method. Optional. */ - public void validateReplicas(Integer replicas, String source, List suggestedConfigUpdates) { + public void validateReplicas(Integer replicas, String machineNamePrefix, + String source, List suggestedConfigUpdates) { if (replicas == null) { return; } @@ -215,32 +221,61 @@ public void validateReplicas(Integer replicas, String source, List if (!hasDynamicServers() && replicas > getClusterSize()) { LOGGER.warning(MessageKeys.REPLICA_MORE_THAN_WLS_SERVERS, source, clusterName, replicas, getClusterSize()); } - // recommend updating WLS dynamic cluster size if replicas value is larger than current - // dynamic cluster size - if (hasDynamicServers() && replicas > getDynamicClusterSize()) { - if (suggestedConfigUpdates != null) { - suggestedConfigUpdates.add(new DynamicClusterSizeConfigUpdate(this, replicas)); + // recommend updating WLS dynamic cluster size and machines if requested to recommend + // updates, ie, suggestedConfigUpdates is not null, and if replicas value is larger than + // the current dynamic cluster size, or if some of the machines to be used for the dynamic + // servers are not yet configured. + if (suggestedConfigUpdates != null) { + if (hasDynamicServers()) { + if (replicas > getDynamicClusterSize() || !verifyMachinesConfigured(machineNamePrefix, replicas)) { + suggestedConfigUpdates.add(new DynamicClusterSizeConfigUpdate(this, replicas)); + } } } } /** - * Finds the names of a machine to be created for newly created dynamic servers in this dynamic cluster + * Verify whether the WebLogic domain already has all the machines configured for use by + * the dynamic cluster. For example, if machineNamePrefix is "domain1-cluster1-machine" + * and numMachinesNeeded is 2, this method return true if machines named + * "domain1-cluster1-machine1" and "domain1-cluster1-machine2" are configured in the + * WebLogic domain. + * + * @param machineNamePrefix Prefix of the names of the machines + * @param numMachinesNeeded Number of machines needed for this dynamic cluster + * @return True if the WebLogic domain already has all the machines configured, or if + * there is no WlsDomainConfig object associated with this cluster, in which case we + * cannot perform the verification, or if machineNamePrefix is null, false otherwise + */ + boolean verifyMachinesConfigured(String machineNamePrefix, int numMachinesNeeded) { + if (wlsDomainConfig != null && machineNamePrefix != null) { + for (int suffix = 1; suffix <= numMachinesNeeded; suffix++) { + if (wlsDomainConfig.getMachineConfig(machineNamePrefix + suffix) == null) { + return false; + } + } + } + return true; + } + + /** + * Finds the names of a machine to be created for all dynamic servers in this dynamic cluster * * @param machineNamePrefix Prefix for the new machine names (should match machineNameMatchExpression in dynamic servers config) * @param targetClusterSize the target dynamic cluster size - * @return A String array containing names of new machines that are not currently in used in the WebLogic domain + * @return A String array containing names of new machines to be created in the WebLogic domain for use by dynamic + * servers in this cluster */ - String[] getMachineNamesForNewDynamicServers(String machineNamePrefix, int targetClusterSize) { + String[] getMachineNamesForDynamicServers(String machineNamePrefix, int targetClusterSize) { if (targetClusterSize < 1 || !hasDynamicServers() || wlsDomainConfig == null) { return new String[0]; } - // machine names needed are [machineNamePrefix] appended by id of the new dynamic servers - // for example, if prefix is "domain1-cluster1-machine" and the current cluster size is 2, and targetClusterSize - // is 4, the ids of the new dynamic servers would be 3 and 4, - // the method should return {"domain1-cluster1-machine3", "domain1-cluster1-machine4"} + // machine names needed are [machineNamePrefix] appended by id of the dynamic servers + // for example, if prefix is "domain1-cluster1-machine" and targetClusterSize is 3, and machine with name + // "domain1-cluster1-machine1" already exists, the names of machines to be created should be + // {"domain1-cluster1-machine2", "domain1-cluster1-machine3"} ArrayList names = new ArrayList<>(); - for (int suffix=getDynamicClusterSize() + 1; suffix <= targetClusterSize; suffix++) { + for (int suffix=1; suffix <= targetClusterSize; suffix++) { String newMachineName = machineNamePrefix == null? "" + suffix: machineNamePrefix + suffix; if (wlsDomainConfig.getMachineConfig(newMachineName) == null) { // only need to create machine if it does not already exist diff --git a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetriever.java b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetriever.java index 1f104922be9..b8d6877dc2e 100644 --- a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetriever.java +++ b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetriever.java @@ -195,7 +195,7 @@ public NextAction apply(Packet packet) { Domain dom = info.getDomain(); DomainSpec domainSpec = dom.getSpec(); - String machineNamePrefix = domainSpec.getDomainUID() + "-" + wlsClusterConfig.getClusterName() + "-machine"; + String machineNamePrefix = Util.getMachineNamePrefix(domainSpec, wlsClusterConfig); boolean successful = updateDynamicClusterSizeWithServiceURL(wlsClusterConfig, machineNamePrefix, targetClusterSize, httpClient, serviceURL); @@ -488,7 +488,7 @@ private static boolean updateDynamicClusterSizeWithServiceURL(final WlsClusterCo LOGGER.info(MessageKeys.WLS_EDIT_SESSION_STARTED); // Create machine(s) - String newMachineNames[] = wlsClusterConfig.getMachineNamesForNewDynamicServers(machineNamePrefix, targetClusterSize); + String newMachineNames[] = wlsClusterConfig.getMachineNamesForDynamicServers(machineNamePrefix, targetClusterSize); for (String machineName: newMachineNames) { LOGGER.info(MessageKeys.WLS_CREATING_MACHINE, machineName); httpClient.executePostUrlOnServiceClusterIP(WlsMachineConfig.getCreateUrl(), diff --git a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfig.java b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfig.java index fcd5208e0f5..1e719e41381 100644 --- a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfig.java +++ b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfig.java @@ -306,7 +306,8 @@ public boolean validate(DomainSpec domainSpec, List suggestedConfi String clusterName = clusterStartup.getClusterName(); if (clusterName != null) { WlsClusterConfig wlsClusterConfig = getClusterConfig(clusterName); - updated |= wlsClusterConfig.validateClusterStartup(clusterStartup, suggestedConfigUpdates); + updated |= wlsClusterConfig.validateClusterStartup(clusterStartup, + Util.getMachineNamePrefix(domainSpec, wlsClusterConfig), suggestedConfigUpdates); } } } @@ -317,7 +318,9 @@ public boolean validate(DomainSpec domainSpec, List suggestedConfi // WLS domain contains only one cluster if (clusterConfigs != null && clusterConfigs.size() == 1) { for (WlsClusterConfig wlsClusterConfig : clusterConfigs) { - wlsClusterConfig.validateReplicas(domainSpec.getReplicas(), "domainSpec", + wlsClusterConfig.validateReplicas(domainSpec.getReplicas(), + Util.getMachineNamePrefix(domainSpec, wlsClusterConfig), + "domainSpec", suggestedConfigUpdates); } } else { diff --git a/src/test/java/oracle/kubernetes/operator/wlsconfig/UtilTest.java b/src/test/java/oracle/kubernetes/operator/wlsconfig/UtilTest.java new file mode 100644 index 00000000000..f5803b7aa76 --- /dev/null +++ b/src/test/java/oracle/kubernetes/operator/wlsconfig/UtilTest.java @@ -0,0 +1,31 @@ +// Copyright 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.wlsconfig; + +import oracle.kubernetes.operator.domain.model.oracle.kubernetes.weblogic.domain.v1.DomainSpec; +import org.junit.Test; + +import static org.junit.Assert.*; + +public class UtilTest { + + @Test + public void verifyMachineNamePrefixFromGetMachineNamePrefix() throws Exception { + DomainSpec domainSpec = new DomainSpec().domainUID("domain1"); + WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1"); + assertEquals("domain1-cluster1-machine", Util.getMachineNamePrefix(domainSpec, wlsClusterConfig)); + } + + @Test + public void verifyNoNPEwithNullArgumentsToGetMachineNamePrefix() throws Exception { + DomainSpec domainSpec = new DomainSpec().domainUID("domain1"); + WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1"); + + assertNull(Util.getMachineNamePrefix(null, wlsClusterConfig)); + assertNull(Util.getMachineNamePrefix(domainSpec, null)); + assertNull(Util.getMachineNamePrefix(null, null)); + } + + +} \ No newline at end of file diff --git a/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfigTest.java b/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfigTest.java index 4122de179ca..f977e16749e 100644 --- a/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfigTest.java +++ b/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfigTest.java @@ -5,7 +5,6 @@ import oracle.kubernetes.TestUtils; import oracle.kubernetes.operator.domain.model.oracle.kubernetes.weblogic.domain.v1.ClusterStartup; -import oracle.kubernetes.operator.helpers.ClientHelper; import oracle.kubernetes.operator.work.NextAction; import oracle.kubernetes.operator.work.Packet; import oracle.kubernetes.operator.work.Step; @@ -164,7 +163,7 @@ public void verifyValidateClusterStartupWarnsIfNoServersInCluster() throws Excep TestUtil.LogHandlerImpl handler = null; try { handler = TestUtil.setupLogHandler(wlsClusterConfig); - wlsClusterConfig.validateClusterStartup(cs, null); + wlsClusterConfig.validateClusterStartup(cs, null,null); assertTrue("Message logged: " + handler.getAllFormattedMessage(), handler.hasWarningMessageWithSubString("No servers configured in WebLogic cluster with name cluster1")); } finally { TestUtil.removeLogHandler(wlsClusterConfig, handler); @@ -179,7 +178,7 @@ public void verifyValidateClusterStartupWarnsIfReplicasTooHigh() throws Exceptio TestUtil.LogHandlerImpl handler = null; try { handler = TestUtil.setupLogHandler(wlsClusterConfig); - wlsClusterConfig.validateClusterStartup(cs, null); + wlsClusterConfig.validateClusterStartup(cs, null,null); assertTrue("Message logged: " + handler.getAllFormattedMessage(), handler.hasWarningMessageWithSubString("Replicas in clusterStartup for cluster cluster1 is specified with a value of 2 which is larger than the number of configured WLS servers in the cluster: 1")); } finally { TestUtil.removeLogHandler(wlsClusterConfig, handler); @@ -191,15 +190,9 @@ public void verifyValidateClusterStartupDoNotSuggestsUpdateToConfiguredClusterIf WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1"); wlsClusterConfig.addServerConfig(createWlsServerConfig("ms-0", 8011, null)); ClusterStartup cs = new ClusterStartup().clusterName("cluster1").replicas(2); - TestUtil.LogHandlerImpl handler = null; - try { - handler = TestUtil.setupLogHandler(wlsClusterConfig); - ArrayList suggestedConfigUpdates = new ArrayList<>(); - wlsClusterConfig.validateClusterStartup(cs, suggestedConfigUpdates); - assertEquals(0, suggestedConfigUpdates.size()); - } finally { - TestUtil.removeLogHandler(wlsClusterConfig, handler); - } + ArrayList suggestedConfigUpdates = new ArrayList<>(); + wlsClusterConfig.validateClusterStartup(cs, null, suggestedConfigUpdates); + assertEquals(0, suggestedConfigUpdates.size()); } @Test @@ -210,7 +203,7 @@ public void verifyValidateClusterStartupDoesNotWarnIfDynamicCluster() throws Exc TestUtil.LogHandlerImpl handler = null; try { handler = TestUtil.setupLogHandler(wlsClusterConfig); - wlsClusterConfig.validateClusterStartup(cs, null); + wlsClusterConfig.validateClusterStartup(cs, null, null); assertFalse("No message should be logged, but found: " + handler.getAllFormattedMessage(), handler.hasWarningMessageLogged()); } finally { TestUtil.removeLogHandler(wlsClusterConfig, handler); @@ -222,16 +215,66 @@ public void verifyValidateClusterStartupSuggestsUpdateToDynamicClusterIfReplicas WlsDynamicServersConfig wlsDynamicServersConfig = createDynamicServersConfig(1, 1, "ms-", "cluster1"); WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1", wlsDynamicServersConfig); ClusterStartup cs = new ClusterStartup().clusterName("cluster1").replicas(2); + ArrayList suggestedConfigUpdates = new ArrayList<>(); + wlsClusterConfig.validateClusterStartup(cs, null, suggestedConfigUpdates); + assertEquals(1, suggestedConfigUpdates.size()); + assertTrue(suggestedConfigUpdates.get(0) instanceof WlsClusterConfig.DynamicClusterSizeConfigUpdate); + } + + @Test + public void verifyValidateClusterStartupDoNotSuggestsUpdateToDynamicClusterIfReplicasNotHigh() throws Exception { + WlsDynamicServersConfig wlsDynamicServersConfig = createDynamicServersConfig(1, 1, "ms-", "cluster1"); + WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1", wlsDynamicServersConfig); + ClusterStartup cs = new ClusterStartup().clusterName("cluster1").replicas(1); TestUtil.LogHandlerImpl handler = null; - try { - handler = TestUtil.setupLogHandler(wlsClusterConfig); - ArrayList suggestedConfigUpdates = new ArrayList<>(); - wlsClusterConfig.validateClusterStartup(cs, suggestedConfigUpdates); - assertEquals(1, suggestedConfigUpdates.size()); - assertTrue(suggestedConfigUpdates.get(0) instanceof WlsClusterConfig.DynamicClusterSizeConfigUpdate); - } finally { - TestUtil.removeLogHandler(wlsClusterConfig, handler); - } + ArrayList suggestedConfigUpdates = new ArrayList<>(); + wlsClusterConfig.validateClusterStartup(cs, null, suggestedConfigUpdates); + assertEquals(0, suggestedConfigUpdates.size()); + } + + @Test + public void verifyValidateClusterStartupSuggestsUpdateToDynamicClusterIfNotEnoughMachines() throws Exception { + WlsDynamicServersConfig wlsDynamicServersConfig = createDynamicServersConfig(2, 1, "ms-", "cluster1"); + WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1", wlsDynamicServersConfig); + + Map clusters = new HashMap(); + clusters.put(wlsClusterConfig.getClusterName(), wlsClusterConfig); + + WlsMachineConfig machine1 = new WlsMachineConfig("domain1-cluster1-machine1", 5556, "localhost", "SSL"); + Map machines = new HashMap(); + machines.put(machine1.getName(), machine1); + + WlsDomainConfig wlsDomainConfig = new WlsDomainConfig("base_domain", clusters, null, null, machines); + wlsClusterConfig.setWlsDomainConfig(wlsDomainConfig); + + ClusterStartup cs = new ClusterStartup().clusterName("cluster1").replicas(2); + + ArrayList suggestedConfigUpdates = new ArrayList<>(); + wlsClusterConfig.validateClusterStartup(cs, "domain1-cluster1-machine", suggestedConfigUpdates); + assertEquals(1, suggestedConfigUpdates.size()); + assertTrue(suggestedConfigUpdates.get(0) instanceof WlsClusterConfig.DynamicClusterSizeConfigUpdate); + } + + @Test + public void verifyValidateClusterStartupDoNotSuggestUpdateToDynamicClusterIfEnoughMachines() throws Exception { + WlsDynamicServersConfig wlsDynamicServersConfig = createDynamicServersConfig(1, 1, "ms-", "cluster1"); + WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1", wlsDynamicServersConfig); + + Map clusters = new HashMap(); + clusters.put(wlsClusterConfig.getClusterName(), wlsClusterConfig); + + WlsMachineConfig machine1 = new WlsMachineConfig("domain1-cluster1-machine1", 5556, "localhost", "SSL"); + Map machines = new HashMap(); + machines.put(machine1.getName(), machine1); + + WlsDomainConfig wlsDomainConfig = new WlsDomainConfig("base_domain", clusters, null, null, machines); + wlsClusterConfig.setWlsDomainConfig(wlsDomainConfig); + + ClusterStartup cs = new ClusterStartup().clusterName("cluster1").replicas(1); + + ArrayList suggestedConfigUpdates = new ArrayList<>(); + wlsClusterConfig.validateClusterStartup(cs, "domain1-cluster1-machine", suggestedConfigUpdates); + assertEquals(0, suggestedConfigUpdates.size()); } @Test @@ -302,6 +345,73 @@ private WlsServerConfig createWlsServerConfig(String serverName, Integer listenP return WlsServerConfig.create(serverConfigMap); } + @Test + public void verifyMachinesConfiguredReturnTrueIfAllMachinesConfigured() { + WlsMachineConfig machine1 = new WlsMachineConfig("domain1-machine1", 5556, "localhost", "SSL"); + WlsMachineConfig machine2 = new WlsMachineConfig("domain1-machine2", 5556, "localhost", "SSL"); + Map machines = new HashMap(); + machines.put(machine1.getName(), machine1); + machines.put(machine2.getName(), machine2); + + WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1", + WlsClusterConfigTest.createDynamicServersConfig(2, 5, "ms-", "cluster1")); + Map clusters = new HashMap(); + clusters.put(wlsClusterConfig.getClusterName(), wlsClusterConfig); + + WlsDomainConfig wlsDomainConfig = new WlsDomainConfig("base_domain", clusters, null, null, machines); + wlsClusterConfig.setWlsDomainConfig(wlsDomainConfig); + + assertTrue(wlsClusterConfig.verifyMachinesConfigured("domain1-machine", 2)); + } + + @Test + public void verifyMachinesConfiguredReturnFalseIfNotAllMachinesConfigured() { + WlsMachineConfig machine1 = new WlsMachineConfig("domain1-machine1", 5556, "localhost", "SSL"); + WlsMachineConfig machine2 = new WlsMachineConfig("domain1-machine2", 5556, "localhost", "SSL"); + Map machines = new HashMap(); + machines.put(machine1.getName(), machine1); + machines.put(machine2.getName(), machine2); + + WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1", + WlsClusterConfigTest.createDynamicServersConfig(2, 5, "ms-", "cluster1")); + Map clusters = new HashMap(); + clusters.put(wlsClusterConfig.getClusterName(), wlsClusterConfig); + + WlsDomainConfig wlsDomainConfig = new WlsDomainConfig("base_domain", clusters, null, null, machines); + wlsClusterConfig.setWlsDomainConfig(wlsDomainConfig); + + assertFalse(wlsClusterConfig.verifyMachinesConfigured("domain1-machine", 3)); + } + + @Test + public void verifyMachinesConfiguredReturnTrueIfNoWlsDomainConfig() { + Map machines = new HashMap(); + + WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1", + WlsClusterConfigTest.createDynamicServersConfig(2, 5, "ms-", "cluster1")); + Map clusters = new HashMap(); + clusters.put(wlsClusterConfig.getClusterName(), wlsClusterConfig); + + assertNull(wlsClusterConfig.getWlsDomainConfig()); + assertTrue(wlsClusterConfig.verifyMachinesConfigured("domain1-machine", 2)); + } + + @Test + public void verifyMachinesConfiguredReturnTrueIfPrefixIsNull() { + Map machines = new HashMap(); + + WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1", + WlsClusterConfigTest.createDynamicServersConfig(2, 5, "ms-", "cluster1")); + Map clusters = new HashMap(); + clusters.put(wlsClusterConfig.getClusterName(), wlsClusterConfig); + + WlsDomainConfig wlsDomainConfig = new WlsDomainConfig("base_domain", clusters, null, null, machines); + wlsClusterConfig.setWlsDomainConfig(wlsDomainConfig); + + assertNotNull(wlsClusterConfig.getWlsDomainConfig()); + assertTrue(wlsClusterConfig.verifyMachinesConfigured(null, 2)); + } + @Test public void verifyGetMachinesNameReturnsExpectedMachineName() { WlsMachineConfig machine1 = new WlsMachineConfig("domain1-machine1", 5556, "localhost", "SSL"); @@ -318,7 +428,7 @@ public void verifyGetMachinesNameReturnsExpectedMachineName() { WlsDomainConfig wlsDomainConfig = new WlsDomainConfig("base_domain", clusters, null, null, machines); wlsClusterConfig.setWlsDomainConfig(wlsDomainConfig); - String names[] = wlsClusterConfig.getMachineNamesForNewDynamicServers("domain1-machine", 4); + String names[] = wlsClusterConfig.getMachineNamesForDynamicServers("domain1-machine", 4); assertEquals(2, names.length); assertEquals("domain1-machine3", names[0]); assertEquals("domain1-machine4", names[1]); @@ -342,7 +452,7 @@ public void verifyGetMachinesNameDoesNotReturnExistingMachines() { WlsDomainConfig wlsDomainConfig = new WlsDomainConfig("base_domain", clusters, null, null, machines); wlsClusterConfig.setWlsDomainConfig(wlsDomainConfig); - String names[] = wlsClusterConfig.getMachineNamesForNewDynamicServers("domain1-machine", 3); + String names[] = wlsClusterConfig.getMachineNamesForDynamicServers("domain1-machine", 3); assertEquals(0, names.length); } @@ -359,7 +469,7 @@ public void verifyGetMachinesNameReturnsEmptyArrayIfNoDomainConfig() { assertNull("verify no domain config is setup", wlsClusterConfig.getWlsDomainConfig()); - String names[] = wlsClusterConfig.getMachineNamesForNewDynamicServers("domain1-machine", 4); + String names[] = wlsClusterConfig.getMachineNamesForDynamicServers("domain1-machine", 4); assertEquals(0, names.length); } @@ -374,9 +484,10 @@ public void verifyGetMachineNamesNoExceptionWhenPrefixIsNull() { WlsDomainConfig wlsDomainConfig = new WlsDomainConfig("base_domain", clusters, null, null, null); wlsClusterConfig.setWlsDomainConfig(wlsDomainConfig); - String[] names = wlsClusterConfig.getMachineNamesForNewDynamicServers(null, 2); - assertEquals(1, names.length); - assertEquals("2", names[0]); + String[] names = wlsClusterConfig.getMachineNamesForDynamicServers(null, 2); + assertEquals(2, names.length); + assertEquals("1", names[0]); + assertEquals("2", names[1]); } @Test @@ -388,15 +499,15 @@ public void verifyGetMachineNamesReturnsEmptyArrayWhenNumIsInvalid() { WlsDomainConfig wlsDomainConfig = new WlsDomainConfig("base_domain", clusters, null, null, null); wlsClusterConfig.setWlsDomainConfig(wlsDomainConfig); - String[] names = wlsClusterConfig.getMachineNamesForNewDynamicServers("domain1-machine", 0); + String[] names = wlsClusterConfig.getMachineNamesForDynamicServers("domain1-machine", 0); assertEquals(0, names.length); - names = wlsClusterConfig.getMachineNamesForNewDynamicServers("domain1-machine", -1); + names = wlsClusterConfig.getMachineNamesForDynamicServers("domain1-machine", -1); assertEquals(0, names.length); } @Test - public void verifyGetMachineNamesReturnsEmptyArrayWhenTargetSizeIsNotLarger() { + public void verifyGetMachineNamesReturnsUndefinedMachineNamesEvenWithSameTargetSize() { WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1", WlsClusterConfigTest.createDynamicServersConfig(2, 5, "ms-", "clsuter1")); Map clusters = new HashMap(); @@ -404,11 +515,10 @@ public void verifyGetMachineNamesReturnsEmptyArrayWhenTargetSizeIsNotLarger() { WlsDomainConfig wlsDomainConfig = new WlsDomainConfig("base_domain", clusters, null, null, null); wlsClusterConfig.setWlsDomainConfig(wlsDomainConfig); - String[] names = wlsClusterConfig.getMachineNamesForNewDynamicServers("domain1-machine", 2); - assertEquals(0, names.length); - - names = wlsClusterConfig.getMachineNamesForNewDynamicServers("domain1-machine", 1); - assertEquals(0, names.length); + String[] names = wlsClusterConfig.getMachineNamesForDynamicServers("domain1-machine", 2); + assertEquals(2, names.length); + assertEquals("domain1-machine1", names[0]); + assertEquals("domain1-machine2", names[1]); } static WlsDynamicServersConfig createDynamicServersConfig(int clusterSize, int maxClusterSize, From f9f9f5afedf29b1531452b61794d837ee51bbcbb Mon Sep 17 00:00:00 2001 From: anthony_lai Date: Tue, 6 Mar 2018 15:44:00 -0800 Subject: [PATCH 012/186] fixed an issue where dynamicClusterSize was reduced if domain created with managedServerStartCount smaller than managedServerCount --- .../operator/wlsconfig/WlsClusterConfig.java | 2 +- .../wlsconfig/WlsClusterConfigTest.java | 34 +++++++++++++++++-- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfig.java b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfig.java index c523b982138..20095e1f3dc 100644 --- a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfig.java +++ b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfig.java @@ -228,7 +228,7 @@ public void validateReplicas(Integer replicas, String machineNamePrefix, if (suggestedConfigUpdates != null) { if (hasDynamicServers()) { if (replicas > getDynamicClusterSize() || !verifyMachinesConfigured(machineNamePrefix, replicas)) { - suggestedConfigUpdates.add(new DynamicClusterSizeConfigUpdate(this, replicas)); + suggestedConfigUpdates.add(new DynamicClusterSizeConfigUpdate(this, Math.max(replicas, getDynamicClusterSize()))); } } } diff --git a/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfigTest.java b/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfigTest.java index f977e16749e..ae6b0675a2f 100644 --- a/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfigTest.java +++ b/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfigTest.java @@ -218,7 +218,9 @@ public void verifyValidateClusterStartupSuggestsUpdateToDynamicClusterIfReplicas ArrayList suggestedConfigUpdates = new ArrayList<>(); wlsClusterConfig.validateClusterStartup(cs, null, suggestedConfigUpdates); assertEquals(1, suggestedConfigUpdates.size()); - assertTrue(suggestedConfigUpdates.get(0) instanceof WlsClusterConfig.DynamicClusterSizeConfigUpdate); + WlsClusterConfig.DynamicClusterSizeConfigUpdate configUpdate = + (WlsClusterConfig.DynamicClusterSizeConfigUpdate) suggestedConfigUpdates.get(0); + assertEquals(2, configUpdate.targetClusterSize); } @Test @@ -252,7 +254,35 @@ public void verifyValidateClusterStartupSuggestsUpdateToDynamicClusterIfNotEnoug ArrayList suggestedConfigUpdates = new ArrayList<>(); wlsClusterConfig.validateClusterStartup(cs, "domain1-cluster1-machine", suggestedConfigUpdates); assertEquals(1, suggestedConfigUpdates.size()); - assertTrue(suggestedConfigUpdates.get(0) instanceof WlsClusterConfig.DynamicClusterSizeConfigUpdate); + WlsClusterConfig.DynamicClusterSizeConfigUpdate configUpdate = + (WlsClusterConfig.DynamicClusterSizeConfigUpdate) suggestedConfigUpdates.get(0); + assertEquals(2, configUpdate.targetClusterSize); + } + + @Test + public void verifyValidateClusterStartupDoNotLowerClusterSizeIfNotEnoughMachines() throws Exception { + WlsDynamicServersConfig wlsDynamicServersConfig = createDynamicServersConfig(2, 1, "ms-", "cluster1"); + WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1", wlsDynamicServersConfig); + + Map clusters = new HashMap(); + clusters.put(wlsClusterConfig.getClusterName(), wlsClusterConfig); + + Map machines = new HashMap(); + + WlsDomainConfig wlsDomainConfig = new WlsDomainConfig("base_domain", clusters, null, null, machines); + wlsClusterConfig.setWlsDomainConfig(wlsDomainConfig); + + ClusterStartup cs = new ClusterStartup().clusterName("cluster1").replicas(1); + + ArrayList suggestedConfigUpdates = new ArrayList<>(); + + // replica = 1, dynamicClusterSize = 2, num of machines = 0 ==> need to create one machine + // but need to ensure that the update will not reduce dynamicClusterSize from 2 to 1 + wlsClusterConfig.validateClusterStartup(cs, "domain1-cluster1-machine", suggestedConfigUpdates); + assertEquals(1, suggestedConfigUpdates.size()); + WlsClusterConfig.DynamicClusterSizeConfigUpdate configUpdate = + (WlsClusterConfig.DynamicClusterSizeConfigUpdate) suggestedConfigUpdates.get(0); + assertEquals(2, configUpdate.targetClusterSize); } @Test From 15e0328233c906cb90f0ef00a097f3e90623ef72 Mon Sep 17 00:00:00 2001 From: anthony_lai Date: Fri, 9 Mar 2018 09:54:54 -0800 Subject: [PATCH 013/186] updated integration test to support creation of dynamic or configured clusters. use named edit session in start-server.py when updating NodeMangerMBean --- kubernetes/internal/domain-job-template.yaml | 4 +-- src/integration-tests/bash/run.sh | 35 +++++++++++--------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/kubernetes/internal/domain-job-template.yaml b/kubernetes/internal/domain-job-template.yaml index 3f734e3e6c9..fcb1afcaff6 100644 --- a/kubernetes/internal/domain-job-template.yaml +++ b/kubernetes/internal/domain-job-template.yaml @@ -312,13 +312,12 @@ data: serverConfig() server=cmo.lookupServer(server_name) machineName=server.getMachine().getName() - print 'Name of machine assigned to server %s is %s' % (server_name, machineName) if machineName is not None: print 'Updating listen address of machine %s' % machineName try: - edit() + edit("weblogic-operator-%s" % server_name) startEdit() cd('/') machine=cmo.lookupMachine(machineName) @@ -326,6 +325,7 @@ data: nm=machine.getNodeManager() nm.setListenAddress(service_name) nm.setNMType('Plain') + save() activate() print 'Updated listen address of machine %s to %s' % (machineName, service_name) except: diff --git a/src/integration-tests/bash/run.sh b/src/integration-tests/bash/run.sh index 0225707d00c..e564903681c 100755 --- a/src/integration-tests/bash/run.sh +++ b/src/integration-tests/bash/run.sh @@ -635,7 +635,7 @@ function test_second_operator { declare_test_pass } -# dom_define DOM_KEY OP_KEY NAMESPACE DOMAIN_UID STARTUP_CONTROL WL_CLUSTER_NAME MS_BASE_NAME ADMIN_PORT ADMIN_WLST_PORT ADMIN_NODE_PORT MS_PORT LOAD_BALANCER_WEB_PORT LOAD_BALANCER_ADMIN_PORT +# dom_define DOM_KEY OP_KEY NAMESPACE DOMAIN_UID STARTUP_CONTROL WL_CLUSTER_NAME WLCLUSTER_TYPE MS_BASE_NAME ADMIN_PORT ADMIN_WLST_PORT ADMIN_NODE_PORT MS_PORT LOAD_BALANCER_WEB_PORT LOAD_BALANCER_ADMIN_PORT # Sets up a table of domain values: all of the above, plus TMP_DIR which is derived. # # dom_get DOM_KEY @@ -651,8 +651,8 @@ function test_second_operator { # echo Defined operator $opkey with `dom_echo_all $DOM_KEY` # function dom_define { - if [ "$#" != 13 ] ; then - fail "requires 13 parameters: DOM_KEY OP_KEY NAMESPACE DOMAIN_UID STARTUP_CONTROL WL_CLUSTER_NAME MS_BASE_NAME ADMIN_PORT ADMIN_WLST_PORT ADMIN_NODE_PORT MS_PORT LOAD_BALANCER_WEB_PORT LOAD_BALANCER_ADMIN_PORT" + if [ "$#" != 14 ] ; then + fail "requires 14 parameters: DOM_KEY OP_KEY NAMESPACE DOMAIN_UID STARTUP_CONTROL WL_CLUSTER_NAME WL_CLUSTER_TYPE MS_BASE_NAME ADMIN_PORT ADMIN_WLST_PORT ADMIN_NODE_PORT MS_PORT LOAD_BALANCER_WEB_PORT LOAD_BALANCER_ADMIN_PORT" fi local DOM_KEY="`echo \"${1}\" | sed 's/-/_/g'`" eval export DOM_${DOM_KEY}_OP_KEY="$2" @@ -660,13 +660,14 @@ function dom_define { eval export DOM_${DOM_KEY}_DOMAIN_UID="$4" eval export DOM_${DOM_KEY}_STARTUP_CONTROL="$5" eval export DOM_${DOM_KEY}_WL_CLUSTER_NAME="$6" - eval export DOM_${DOM_KEY}_MS_BASE_NAME="$7" - eval export DOM_${DOM_KEY}_ADMIN_PORT="$8" - eval export DOM_${DOM_KEY}_ADMIN_WLST_PORT="$9" - eval export DOM_${DOM_KEY}_ADMIN_NODE_PORT="${10}" - eval export DOM_${DOM_KEY}_MS_PORT="${11}" - eval export DOM_${DOM_KEY}_LOAD_BALANCER_WEB_PORT="${12}" - eval export DOM_${DOM_KEY}_LOAD_BALANCER_ADMIN_PORT="${13}" + eval export DOM_${DOM_KEY}_WL_CLUSTER_TYPE="$7" + eval export DOM_${DOM_KEY}_MS_BASE_NAME="$8" + eval export DOM_${DOM_KEY}_ADMIN_PORT="$9" + eval export DOM_${DOM_KEY}_ADMIN_WLST_PORT="${10}" + eval export DOM_${DOM_KEY}_ADMIN_NODE_PORT="${11}" + eval export DOM_${DOM_KEY}_MS_PORT="${12}" + eval export DOM_${DOM_KEY}_LOAD_BALANCER_WEB_PORT="${13}" + eval export DOM_${DOM_KEY}_LOAD_BALANCER_ADMIN_PORT="${14}" # derive TMP_DIR $RESULT_DIR/$NAMESPACE-$DOMAIN_UID : eval export DOM_${DOM_KEY}_TMP_DIR="$RESULT_DIR/$3-$4" @@ -696,6 +697,7 @@ function run_create_domain_job { local DOMAIN_UID="`dom_get $1 DOMAIN_UID`" local STARTUP_CONTROL="`dom_get $1 STARTUP_CONTROL`" local WL_CLUSTER_NAME="`dom_get $1 WL_CLUSTER_NAME`" + local WL_CLUSTER_TYPE="`dom_get $1 WL_CLUSTER_TYPE`" local MS_BASE_NAME="`dom_get $1 MS_BASE_NAME`" local ADMIN_PORT="`dom_get $1 ADMIN_PORT`" local ADMIN_WLST_PORT="`dom_get $1 ADMIN_WLST_PORT`" @@ -755,6 +757,7 @@ function run_create_domain_job { sed -i -e "s;^persistencePath:.*;persistencePath: $PV_ROOT/acceptance_test_pv/$PV_DIR;" ${tmp_dir}/create-domain-job-inputs.yaml sed -i -e "s/^domainUid:.*/domainUid: $DOMAIN_UID/" ${tmp_dir}/create-domain-job-inputs.yaml sed -i -e "s/^clusterName:.*/clusterName: $WL_CLUSTER_NAME/" ${tmp_dir}/create-domain-job-inputs.yaml + sed -i -e "s/^clusterType:.*/clusterType: $WL_CLUSTER_TYPE/" ${tmp_dir}/create-domain-job-inputs.yaml sed -i -e "s/^namespace:.*/namespace: $NAMESPACE/" ${tmp_dir}/create-domain-job-inputs.yaml sed -i -e "s/^t3ChannelPort:.*/t3ChannelPort: $ADMIN_WLST_PORT/" ${tmp_dir}/create-domain-job-inputs.yaml sed -i -e "s/^adminNodePort:.*/adminNodePort: $ADMIN_NODE_PORT/" ${tmp_dir}/create-domain-job-inputs.yaml @@ -2488,12 +2491,12 @@ function test_suite { op_define oper1 weblogic-operator-1 "default,test1" 31001 op_define oper2 weblogic-operator-2 test2 32001 - # DOM_KEY OP_KEY NAMESPACE DOMAIN_UID STARTUP_CONTROL WL_CLUSTER_NAME MS_BASE_NAME ADMIN_PORT ADMIN_WLST_PORT ADMIN_NODE_PORT MS_PORT LOAD_BALANCER_WEB_PORT LOAD_BALANCER_ADMIN_PORT - dom_define domain1 oper1 default domain1 AUTO cluster-1 managed-server 7001 30012 30701 8001 30305 30315 - dom_define domain2 oper1 default domain2 AUTO cluster-1 managed-server 7011 30031 30702 8021 30306 30316 - dom_define domain3 oper1 test1 domain3 AUTO cluster-1 managed-server 7021 30041 30703 8031 30307 30317 - dom_define domain4 oper2 test2 domain4 AUTO cluster-1 managed-server 7041 30051 30704 8041 30308 30318 - dom_define domain5 oper1 default domain5 ADMIN cluster-1 managed-server 7051 30061 30705 8051 30309 30319 + # DOM_KEY OP_KEY NAMESPACE DOMAIN_UID STARTUP_CONTROL WL_CLUSTER_NAME WL_CLUSTER_TYPE MS_BASE_NAME ADMIN_PORT ADMIN_WLST_PORT ADMIN_NODE_PORT MS_PORT LOAD_BALANCER_WEB_PORT LOAD_BALANCER_ADMIN_PORT + dom_define domain1 oper1 default domain1 AUTO cluster-1 dynamic managed-server 7001 30012 30701 8001 30305 30315 + dom_define domain2 oper1 default domain2 AUTO cluster-1 dynamic managed-server 7011 30031 30702 8021 30306 30316 + dom_define domain3 oper1 test1 domain3 AUTO cluster-1 dynamic managed-server 7021 30041 30703 8031 30307 30317 + dom_define domain4 oper2 test2 domain4 AUTO cluster-1 configured managed-server 7041 30051 30704 8041 30308 30318 + dom_define domain5 oper1 default domain5 ADMIN cluster-1 dynamic managed-server 7051 30061 30705 8051 30309 30319 # create namespaces for domains (the operator job creates a namespace if needed) # TODO have the op_define commands themselves create target namespace if it doesn't already exist, or test if the namespace creation is needed in the first place, and if so, ask MikeG to create them as part of domain create job From a9d5f8f97d84a2c654826265cf365e6963de38a5 Mon Sep 17 00:00:00 2001 From: anthony_lai Date: Fri, 9 Mar 2018 15:33:11 -0800 Subject: [PATCH 014/186] destroy edit session after edit completed --- kubernetes/internal/domain-job-template.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kubernetes/internal/domain-job-template.yaml b/kubernetes/internal/domain-job-template.yaml index fcb1afcaff6..f5b9fb4c1ea 100644 --- a/kubernetes/internal/domain-job-template.yaml +++ b/kubernetes/internal/domain-job-template.yaml @@ -327,9 +327,11 @@ data: nm.setNMType('Plain') save() activate() + destroyEditSession("weblogic-operator-%s" % server_name) print 'Updated listen address of machine %s to %s' % (machineName, service_name) except: cancelEdit() + destroyEditSession("weblogic-operator-%s" % server_name) raise # Exit WLST From 89543b26783230cd25b8370cc2464f8f8f5915dd Mon Sep 17 00:00:00 2001 From: anthony_lai Date: Tue, 13 Mar 2018 09:25:54 -0700 Subject: [PATCH 015/186] update NodeManagerMBean listenAddress to None in stop server script. Not use edit session in WlsConfigRetriever. Not use named edit session in start-server.py --- kubernetes/internal/domain-job-template.yaml | 45 ++++++++++++++++++- .../operator/helpers/PodHelper.java | 6 ++- .../wlsconfig/WlsConfigRetriever.java | 27 ++++++++--- 3 files changed, 68 insertions(+), 10 deletions(-) diff --git a/kubernetes/internal/domain-job-template.yaml b/kubernetes/internal/domain-job-template.yaml index 57fad119fd3..5edaf98e83b 100644 --- a/kubernetes/internal/domain-job-template.yaml +++ b/kubernetes/internal/domain-job-template.yaml @@ -317,7 +317,7 @@ data: if machineName is not None: print 'Updating listen address of machine %s' % machineName try: - edit("weblogic-operator-%s" % server_name) + edit() startEdit() cd('/') machine=cmo.lookupMachine(machineName) @@ -357,7 +357,13 @@ data: #!/bin/bash echo "Stop the server" - wlst.sh -skipWLSModuleScanning ${pyFile} + + admin_server_t3_url= + if [ -n "\$3" ]; then + admin_server_t3_url=t3://\$1-\$3:\$4 + fi + + wlst.sh -skipWLSModuleScanning ${pyFile} \$1 \$2 \$admin_server_t3_url # Return status of 2 means failed to stop a server through the NodeManager. # Look to see if there is a server process that can be killed. @@ -379,6 +385,41 @@ data: cat /u01/weblogic/read-domain-secret.py > ${pyFile} cat << EOF >> ${pyFile} + domain_uid = sys.argv[1] + server_name = sys.argv[2] + if (len(sys.argv) == 4): + admin_server_url = sys.argv[3] + else: + admin_server_url = None + + print 'admin_server_url is %s' % admin_server_url + + # Update node manager listen address to None + if admin_server_url is not None: + connect(admin_username, admin_password, admin_server_url) + serverConfig() + server=cmo.lookupServer(server_name) + machineName=server.getMachine().getName() + print 'Name of machine assigned to server %s is %s' % (server_name, machineName) + + if machineName is not None: + print 'Updating listen address of machine %s' % machineName + try: + edit() + startEdit() + cd('/') + machine=cmo.lookupMachine(machineName) + print 'Machine is %s' % machine + nm=machine.getNodeManager() + nm.setListenAddress(None) + save() + activate() + destroyEditSession("weblogic-operator-%s" % server_name) + print 'Updated listen address of machine %s to None' % machineName + except: + cancelEdit() + destroyEditSession("weblogic-operator-%s" % server_name) + # Connect to nodemanager and stop server try: nmConnect(admin_username, admin_password, '\$1-\$2', '5556', '%DOMAIN_NAME%', '${DOMAIN_HOME}', 'plain') diff --git a/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java b/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java index 0b92af3c538..db735453dda 100644 --- a/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java +++ b/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java @@ -536,7 +536,11 @@ public NextAction apply(Packet packet) { V1Lifecycle lifecycle = new V1Lifecycle(); V1Handler preStop = new V1Handler(); V1ExecAction exec = new V1ExecAction(); - exec.addCommandItem("/shared/domain/" + weblogicDomainName + "/servers/" + weblogicServerName + "/nodemgr_home/stopServer.sh"); + exec.addCommandItem("/shared/domain/" + weblogicDomainName + "/nodemgr_home/stopServer.sh"); + exec.addCommandItem(weblogicDomainUID); + exec.addCommandItem(weblogicServerName); + exec.addCommandItem(spec.getAsName()); + exec.addCommandItem(String.valueOf(spec.getAsPort())); preStop.setExec(exec); lifecycle.setPreStop(preStop); container.setLifecycle(lifecycle); diff --git a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetriever.java b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetriever.java index 788557a74e2..fb54e2ca4e5 100644 --- a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetriever.java +++ b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetriever.java @@ -64,6 +64,16 @@ public class WlsConfigRetriever { // REST URL for canceling a WebLogic edit session private static final String CANCEL_EDIT_SESSION_URL = "/management/weblogic/latest/edit/changeManager/cancelEdit"; + // REST URL for creating a WebLogic named edit session + private static final String CREATE_EDIT_SESSION = "/management/weblogic/latest/domainRuntime/editSessionConfigurationManager/editSessionConfigurations"; + + // Name of WebLogic named edit session to be used by weblogic operator + private static final String EDIT_SESSION_NAME = "weblogic-operator"; + + // Payload for creating a WebLogic named edit session + private static final String CREATE_EDIT_SESSION_PAYLOAD = + " { name: '" + EDIT_SESSION_NAME + "' , description: 'edit session for weblogic operator' } "; + /** * Constructor. * @@ -483,9 +493,12 @@ private static boolean updateDynamicClusterSizeWithServiceURL(final WlsClusterCo boolean result = false; try { + // create WebLogic named edit session +// httpClient.executePostUrlOnServiceClusterIP(CREATE_EDIT_SESSION, serviceURL, CREATE_EDIT_SESSION_PAYLOAD); + // start a WebLogic edit session - httpClient.executePostUrlOnServiceClusterIP(START_EDIT_SESSION_URL, serviceURL, ""); - LOGGER.info(MessageKeys.WLS_EDIT_SESSION_STARTED); +// httpClient.executePostUrlOnServiceClusterIP(START_EDIT_SESSION_URL, serviceURL, ""); +// LOGGER.info(MessageKeys.WLS_EDIT_SESSION_STARTED); // Create machine(s) String newMachineNames[] = wlsClusterConfig.getMachineNamesForDynamicServers(machineNamePrefix, targetClusterSize); @@ -501,15 +514,15 @@ private static boolean updateDynamicClusterSizeWithServiceURL(final WlsClusterCo serviceURL, wlsClusterConfig.getUpdateDynamicClusterSizePayload(targetClusterSize), true).getResponse(); - // activate the WebLogic edit session - httpClient.executePostUrlOnServiceClusterIP(ACTIVATE_EDIT_SESSION_URL, serviceURL, "", true); +// // activate the WebLogic edit session +// httpClient.executePostUrlOnServiceClusterIP(ACTIVATE_EDIT_SESSION_URL, serviceURL, "", true); result = wlsClusterConfig.checkUpdateDynamicClusterSizeJsonResult(jsonResult); - LOGGER.info(MessageKeys.WLS_EDIT_SESSION_ACTIVATED); +// LOGGER.info(MessageKeys.WLS_EDIT_SESSION_ACTIVATED); } catch (HTTPException httpException) { // cancel the WebLogic edit session - httpClient.executePostUrlOnServiceClusterIP(CANCEL_EDIT_SESSION_URL, serviceURL, ""); - LOGGER.info(MessageKeys.WLS_EDIT_SESSION_CANCELLED); +// httpClient.executePostUrlOnServiceClusterIP(CANCEL_EDIT_SESSION_URL, serviceURL, ""); +// LOGGER.info(MessageKeys.WLS_EDIT_SESSION_CANCELLED); } LOGGER.exiting(result); return result; From 3b9296cf22ae7404b65944ea15c1740683e26ff4 Mon Sep 17 00:00:00 2001 From: anthony_lai Date: Wed, 14 Mar 2018 14:40:47 -0700 Subject: [PATCH 016/186] moved files from nodemgr_home to configMap. Update start-server.py script to update NodeManagerMBean before starting managed server --- kubernetes/internal/domain-job-template.yaml | 343 ------------------ .../operator/helpers/ConfigMapHelper.java | 319 +++++++++++++++- .../operator/helpers/PodHelper.java | 14 +- .../wlsconfig/WlsConfigRetriever.java | 13 - 4 files changed, 328 insertions(+), 361 deletions(-) diff --git a/kubernetes/internal/domain-job-template.yaml b/kubernetes/internal/domain-job-template.yaml index 5edaf98e83b..40eebb24271 100644 --- a/kubernetes/internal/domain-job-template.yaml +++ b/kubernetes/internal/domain-job-template.yaml @@ -103,352 +103,9 @@ data: export DOMAIN_HOME=${SHARED_PATH}/domain/%DOMAIN_NAME% - function createCommonNodeMgrHome() { - - # Create nodemanager home for the server - nmdir=${DOMAIN_HOME}/nodemgr_home - createFolder ${nmdir} - prop=${nmdir}/nodemanager.properties - cp ${DOMAIN_HOME}/nodemanager/nodemanager.properties ${nmdir} - } - - # Function to create script for starting a server - # \$1 - Domain UID - # \$2 - Server Name - function createCommonStartScripts() { - - nmdir=${DOMAIN_HOME}/nodemgr_home - scriptFile=${nmdir}/startServer.sh - pyFile=${nmdir}/start-server.py - - # Create a script that starts the node manager, then uses wlst to connect - # to the nodemanager and start the server. - # The script and 'EOF' on the following lines must not be indented! - cat << EOF > ${scriptFile} - #!/bin/bash - - domain_uid=\$1 - server_name=\$2 - as_name=\$3 - as_port=\$4 - as_hostname=\$1-\$3 - - echo "debug arguments are \$1 \$2 \$3 \$4" - - nmProp="/u01/nodemanager/nodemanager.properties" - - # TODO: parameterize shared home and domain name - export DOMAIN_HOME=/shared/domain/%DOMAIN_NAME% - - # - # Create a folder - # \$1 - path of folder to create - function createFolder { - mkdir -m 777 -p \$1 - if [ ! -d \$1 ]; then - fail "Unable to create folder \$1" - fi - } - - # Function to create server specific scripts and properties (e.g startup.properties, etc) - # \$1 - Domain UID - # \$2 - Server Name - # \$3 - Admin Server Hostname (only passed for managed servers) - # \$4 - Admin Server port (only passed for managed servers) - function createServerScriptsProperties() { - - # Create nodemanager home for the server - srvr_nmdir=/u01/nodemanager - createFolder \${srvr_nmdir} - cp ${DOMAIN_HOME}/nodemanager/nodemanager.domains \${srvr_nmdir} - cp ${DOMAIN_HOME}/bin/startNodeManager.sh \${srvr_nmdir} - - # Edit the start nodemanager script to use the home for the server - sed -i -e "s:/shared/domain/base_domain/nodemanager:/u01/nodemanager:g" ${srvr_nmdir}/startNodeManager.sh - - # Create startup.properties file - datadir=\${DOMAIN_HOME}/servers/\$2/data/nodemanager - nmdir=\${DOMAIN_HOME}/nodemgr_home - stateFile=\${datadir}/\$2.state - startProp=\${datadir}/startup.properties - if [ -f "\$startProp" ]; then - echo "startup.properties already exists" - return 0 - fi - - createFolder \${datadir} - echo "# Server startup properties" > \${startProp} - echo "AutoRestart=true" >> \${startProp} - if [ -n "\$3" ]; then - echo "AdminURL=http\://\$3\:\$4" >> \${startProp} - fi - echo "RestartMax=2" >> \${startProp} - echo "RotateLogOnStartup=false" >> \${startProp} - echo "RotationType=bySize" >> \${startProp} - echo "RotationTimeStart=00\:00" >> \${startProp} - echo "RotatedFileCount=100" >> \${startProp} - echo "RestartDelaySeconds=0" >> \${startProp} - echo "FileSizeKB=5000" >> \${startProp} - echo "FileTimeSpanFactor=3600000" >> \${startProp} - echo "RestartInterval=3600" >> \${startProp} - echo "NumberOfFilesLimited=true" >> \${startProp} - echo "FileTimeSpan=24" >> \${startProp} - echo "NMHostName=\$1-\$2" >> \${startProp} - } - - # Check for stale state file and remove if found" - if [ -f \${stateFile} ]; then - echo "Removing stale file \${stateFile}" - rm \${stateFile} - fi - - # Create nodemanager home directory that is local to the k8s node - mkdir -p /u01/nodemanager - cp \${DOMAIN_HOME}/nodemanager/* /u01/nodemanager/ - cp \${DOMAIN_HOME}/nodemgr_home/nodemanager.properties /u01/nodemanager - - # Edit the nodemanager properties file to use the home for the server - sed -i -e "s:DomainsFile=.*:DomainsFile=/u01/nodemanager/nodemanager.domains:g" /u01/nodemanager/nodemanager.properties - sed -i -e "s:NodeManagerHome=.*:NodeManagerHome=/u01/nodemanager:g" /u01/nodemanager/nodemanager.properties - sed -i -e "s:ListenAddress=.*:ListenAddress=\$1-\$2:g" /u01/nodemanager/nodemanager.properties - sed -i -e "s:LogFile=.*:LogFile=/shared/logs/nodemanager-\$2.log:g" /u01/nodemanager/nodemanager.properties - - export JAVA_PROPERTIES="-DLogFile=/shared/logs/nodemanager-\$server_name.log -DNodeManagerHome=/u01/nodemanager" - export NODEMGR_HOME="/u01/nodemanager" - - - # Create startup.properties - echo "Create startup.properties" - if [ -n "\$3" ]; then - echo "this is managed server" - createServerScriptsProperties \$domain_uid \$server_name \$as_hostname \$as_port - else - echo "this is admin server" - createServerScriptsProperties \$domain_uid \$server_name - fi - - echo "Start the nodemanager" - . \${NODEMGR_HOME}/startNodeManager.sh & - - echo "Allow the nodemanager some time to start before attempting to connect" - sleep 15 - echo "Finished waiting for the nodemanager to start" - - echo "Update JVM arguments" - echo "Arguments=\${USER_MEM_ARGS} -XX\:+UnlockExperimentalVMOptions -XX\:+UseCGroupMemoryLimitForHeap \${JAVA_OPTIONS}" >> \${startProp} - - admin_server_t3_url= - if [ -n "\$3" ]; then - admin_server_t3_url=t3://\$domain_uid-\$as_name:\$as_port - fi - - echo "Start the server" - wlst.sh -skipWLSModuleScanning ${pyFile} \$domain_uid \$server_name \$admin_server_t3_url - - echo "Wait indefinitely so that the Kubernetes pod does not exit and try to restart" - while true; do sleep 60; done - EOF - - checkFileExists ${scriptFile} - chmod +x ${scriptFile} - - # Create a python script to execute the wlst commands. - # The script and 'EOF' on the following lines must not be indented! - echo "import sys;" > ${pyFile} - cat /u01/weblogic/read-domain-secret.py >> ${pyFile} - cat << EOF >> ${pyFile} - - domain_uid = sys.argv[1] - server_name = sys.argv[2] - if (len(sys.argv) == 4): - admin_server_url = sys.argv[3] - else: - admin_server_url = None - - domain_path= "${DOMAIN_HOME}" - - print 'admin username is %s' % admin_username - print 'domain path is %s' % domain_path - print 'server name is %s' % server_name - print 'admin server url is %s' % admin_server_url - - # Encrypt the admin username and password - adminUsernameEncrypted=encrypt(admin_username, domain_path) - adminPasswordEncrypted=encrypt(admin_password, domain_path) - - print 'Create boot.properties files for this server' - - # Define the folder path - secdir='%s/servers/%s/security' % (domain_path, server_name) - - # Create the security folder (if it does not already exist) - try: - os.makedirs(secdir) - except OSError: - if not os.path.isdir(secdir): - raise - - print 'writing boot.properties to %s/servers/%s/security/boot.properties' % (domain_path, server_name) - - bpFile=open('%s/servers/%s/security/boot.properties' % (domain_path, server_name), 'w+') - bpFile.write("username=%s\n" % adminUsernameEncrypted) - bpFile.write("password=%s\n" % adminPasswordEncrypted) - bpFile.close() - - service_name = domain_uid + "-" + server_name - - # Connect to nodemanager and start server - try: - nmConnect(admin_username, admin_password, service_name, '5556', '%DOMAIN_NAME%', '${DOMAIN_HOME}', 'plain') - nmStart(server_name) - nmDisconnect() - except WLSTException, e: - nmDisconnect() - print e - - # Update node manager listen address - if admin_server_url is not None: - connect(admin_username, admin_password, admin_server_url) - serverConfig() - server=cmo.lookupServer(server_name) - machineName=server.getMachine().getName() - print 'Name of machine assigned to server %s is %s' % (server_name, machineName) - - if machineName is not None: - print 'Updating listen address of machine %s' % machineName - try: - edit() - startEdit() - cd('/') - machine=cmo.lookupMachine(machineName) - print 'Machine is %s' % machine - nm=machine.getNodeManager() - nm.setListenAddress(service_name) - nm.setNMType('Plain') - save() - activate() - destroyEditSession("weblogic-operator-%s" % server_name) - print 'Updated listen address of machine %s to %s' % (machineName, service_name) - except: - cancelEdit() - destroyEditSession("weblogic-operator-%s" % server_name) - raise - - # Exit WLST - exit() - EOF - - checkFileExists ${pyFile} - - } - - # Function to create script for stopping a server - # \$1 - Domain UID - # \$2 - Server Name - function createCommonStopScripts() { - - nmdir=${DOMAIN_HOME}/nodemgr_home - scriptFile=${nmdir}/stopServer.sh - pyFile=${nmdir}/stop-server.py - - # Create a script that stops the server. - # The script and 'EOF' on the following lines must not be indented! - cat << EOF > ${scriptFile} - #!/bin/bash - - echo "Stop the server" - - admin_server_t3_url= - if [ -n "\$3" ]; then - admin_server_t3_url=t3://\$1-\$3:\$4 - fi - - wlst.sh -skipWLSModuleScanning ${pyFile} \$1 \$2 \$admin_server_t3_url - - # Return status of 2 means failed to stop a server through the NodeManager. - # Look to see if there is a server process that can be killed. - if [ \$? -eq 2 ]; then - pid=\$(jps -v | grep '[D]weblogic.Name=\$2' | awk '{print \$1}') - if [ ! -z \$pid ]; then - echo "Killing the server process \$pid" - kill -15 \$pid - fi - fi - - EOF - - checkFileExists ${scriptFile} - chmod +x ${scriptFile} - - # Create a python script to execute the wlst commands. - # The script and 'EOF' on the following lines must not be indented! - cat /u01/weblogic/read-domain-secret.py > ${pyFile} - cat << EOF >> ${pyFile} - - domain_uid = sys.argv[1] - server_name = sys.argv[2] - if (len(sys.argv) == 4): - admin_server_url = sys.argv[3] - else: - admin_server_url = None - - print 'admin_server_url is %s' % admin_server_url - - # Update node manager listen address to None - if admin_server_url is not None: - connect(admin_username, admin_password, admin_server_url) - serverConfig() - server=cmo.lookupServer(server_name) - machineName=server.getMachine().getName() - print 'Name of machine assigned to server %s is %s' % (server_name, machineName) - - if machineName is not None: - print 'Updating listen address of machine %s' % machineName - try: - edit() - startEdit() - cd('/') - machine=cmo.lookupMachine(machineName) - print 'Machine is %s' % machine - nm=machine.getNodeManager() - nm.setListenAddress(None) - save() - activate() - destroyEditSession("weblogic-operator-%s" % server_name) - print 'Updated listen address of machine %s to None' % machineName - except: - cancelEdit() - destroyEditSession("weblogic-operator-%s" % server_name) - - # Connect to nodemanager and stop server - try: - nmConnect(admin_username, admin_password, '\$1-\$2', '5556', '%DOMAIN_NAME%', '${DOMAIN_HOME}', 'plain') - except: - print('Failed to connect to the NodeManager') - exit(exitcode=2) - - # Kill the server - try: - nmKill('\$2') - except: - print('Connected to the NodeManager, but failed to stop the server') - exit(exitcode=2) - - # Exit WLST - nmDisconnect() - exit() - EOF - } - - checkFileExists ${pyFile} - # Create the domain wlst.sh -skipWLSModuleScanning /u01/weblogic/create-domain.py - createCommonNodeMgrHome - createCommonStartScripts - createCommonStopScripts - echo "Successfully Completed" create-domain.py: |- diff --git a/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java b/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java index 1ef0d5f627e..f49a9a306c3 100644 --- a/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java +++ b/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java @@ -23,6 +23,315 @@ public class ConfigMapHelper { private static final LoggingFacade LOGGER = LoggingFactory.getLogger("Operator", "Operator"); + private static final String START_SERVER_SHELL_SCRIPT = "#!/bin/bash\n" + + "\n" + + "domain_uid=$1\n" + + "server_name=$2\n" + + "domain_name=$3\n" + + "as_name=$4\n" + + "as_port=$5\n" + + "as_hostname=$1-$4\n" + + "\n" + + "echo \"debug arguments are $1 $2 $3 $4 $5\"\n" + + "\n" + + "nmProp=\"/u01/nodemanager/nodemanager.properties\"\n" + + "\n" + + "# TODO: parameterize shared home and domain name\n" + + "export DOMAIN_HOME=/shared/domain/$domain_name\n" + + "\n" + + "#\n" + + "# Create a folder\n" + + "# $1 - path of folder to create\n" + + "function createFolder {\n" + + " mkdir -m 777 -p $1\n" + + " if [ ! -d $1 ]; then\n" + + " fail \"Unable to create folder $1\"\n" + + " fi\n" + + "}\n" + + "\n" + + "# Function to create server specific scripts and properties (e.g startup.properties, etc)\n" + + "# $1 - Domain UID\n" + + "# $2 - Server Name\n" + + "# $3 - Domain Name\n" + + "# $4 - Admin Server Hostname (only passed for managed servers)\n" + + "# $5 - Admin Server port (only passed for managed servers)\n" + + "function createServerScriptsProperties() {\n" + + "\n" + + " # Create nodemanager home for the server\n" + + " srvr_nmdir=/u01/nodemanager\n" + + " createFolder ${srvr_nmdir}\n" + + " cp /shared/domain/$3/nodemanager/nodemanager.domains ${srvr_nmdir}\n" + + " cp /shared/domain/$3/bin/startNodeManager.sh ${srvr_nmdir}\n" + + "\n" + + " # Edit the start nodemanager script to use the home for the server\n" + + " sed -i -e \"s:/shared/domain/$3/nodemanager:/u01/nodemanager:g\" /startNodeManager.sh\n" + + "\n" + + " # Create startup.properties file\n" + + " datadir=${DOMAIN_HOME}/servers/$2/data/nodemanager\n" + + " nmdir=${DOMAIN_HOME}/nodemgr_home\n" + + " stateFile=${datadir}/$2.state\n" + + " startProp=${datadir}/startup.properties\n" + + " if [ -f \"$startProp\" ]; then\n" + + " echo \"startup.properties already exists\"\n" + + " return 0\n" + + " fi\n" + + "\n" + + " createFolder ${datadir}\n" + + " echo \"# Server startup properties\" > ${startProp}\n" + + " echo \"AutoRestart=true\" >> ${startProp}\n" + + " if [ -n \"$4\" ]; then\n" + + " echo \"AdminURL=http\\://$4\\:$5\" >> ${startProp}\n" + + " fi\n" + + " echo \"RestartMax=2\" >> ${startProp}\n" + + " echo \"RotateLogOnStartup=false\" >> ${startProp}\n" + + " echo \"RotationType=bySize\" >> ${startProp}\n" + + " echo \"RotationTimeStart=00\\:00\" >> ${startProp}\n" + + " echo \"RotatedFileCount=100\" >> ${startProp}\n" + + " echo \"RestartDelaySeconds=0\" >> ${startProp}\n" + + " echo \"FileSizeKB=5000\" >> ${startProp}\n" + + " echo \"FileTimeSpanFactor=3600000\" >> ${startProp}\n" + + " echo \"RestartInterval=3600\" >> ${startProp}\n" + + " echo \"NumberOfFilesLimited=true\" >> ${startProp}\n" + + " echo \"FileTimeSpan=24\" >> ${startProp}\n" + + " echo \"NMHostName=$1-$2\" >> ${startProp}\n" + + "}\n" + + "\n" + + "# Check for stale state file and remove if found\"\n" + + "if [ -f ${stateFile} ]; then\n" + + " echo \"Removing stale file ${stateFile}\"\n" + + " rm ${stateFile}\n" + + "fi\n" + + "\n" + + "# Create nodemanager home directory that is local to the k8s node\n" + + "mkdir -p /u01/nodemanager\n" + + "cp ${DOMAIN_HOME}/nodemanager/* /u01/nodemanager/\n" + + "\n" + + "# Edit the nodemanager properties file to use the home for the server\n" + + "sed -i -e \"s:DomainsFile=.*:DomainsFile=/u01/nodemanager/nodemanager.domains:g\" /u01/nodemanager/nodemanager.properties\n" + + "sed -i -e \"s:NodeManagerHome=.*:NodeManagerHome=/u01/nodemanager:g\" /u01/nodemanager/nodemanager.properties\n" + + "sed -i -e \"s:ListenAddress=.*:ListenAddress=$1-$2:g\" /u01/nodemanager/nodemanager.properties\n" + + "sed -i -e \"s:LogFile=.*:LogFile=/shared/logs/nodemanager-$2.log:g\" /u01/nodemanager/nodemanager.properties\n" + + "\n" + + "export JAVA_PROPERTIES=\"-DLogFile=/shared/logs/nodemanager-$server_name.log -DNodeManagerHome=/u01/nodemanager\"\n" + + "export NODEMGR_HOME=\"/u01/nodemanager\"\n" + + "\n" + + "\n" + + "# Create startup.properties\n" + + "echo \"Create startup.properties\"\n" + + "if [ -n \"$4\" ]; then\n" + + " echo \"this is managed server\"\n" + + " createServerScriptsProperties $domain_uid $server_name $domain_name $as_hostname $as_port\n" + + "else\n" + + " echo \"this is admin server\"\n" + + " createServerScriptsProperties $domain_uid $server_name $domain_name\n" + + "fi\n" + + "\n" + + "echo \"Start the nodemanager\"\n" + + ". ${NODEMGR_HOME}/startNodeManager.sh &\n" + + "\n" + + "echo \"Allow the nodemanager some time to start before attempting to connect\"\n" + + "sleep 15\n" + + "echo \"Finished waiting for the nodemanager to start\"\n" + + "\n" + + "echo \"Update JVM arguments\"\n" + + "echo \"Arguments=${USER_MEM_ARGS} -XX\\:+UnlockExperimentalVMOptions -XX\\:+UseCGroupMemoryLimitForHeap ${JAVA_OPTIONS}\" >> ${startProp}\n" + + "\n" + + "admin_server_t3_url=\n" + + "if [ -n \"$4\" ]; then\n" + + " admin_server_t3_url=t3://$domain_uid-$as_name:$as_port\n" + + "fi\n" + + "\n" + + "echo \"Start the server\"\n" + + "wlst.sh -skipWLSModuleScanning /weblogic-operator/scripts/start-server.py $domain_uid $server_name $domain_name $admin_server_t3_url\n" + + "\n" + + "echo \"Wait indefinitely so that the Kubernetes pod does not exit and try to restart\"\n" + + "while true; do sleep 60; done\n"; + + private static final String START_SERVER_PYTHON_SCRIPT = "import sys;\n" + + "#\n" + + "# +++ Start of common code for reading domain secrets\n" + + "\n" + + "# Read username secret\n" + + "file = open('/weblogic-operator/secrets/username', 'r')\n" + + "admin_username = file.read()\n" + + "file.close()\n" + + "\n" + + "# Read password secret\n" + + "file = open('/weblogic-operator/secrets/password', 'r')\n" + + "admin_password = file.read()\n" + + "file.close()\n" + + "\n" + + "# +++ End of common code for reading domain secrets\n" + + "#\n" + + "domain_uid = sys.argv[1]\n" + + "server_name = sys.argv[2]\n" + + "domain_name = sys.argv[3]\n" + + "if (len(sys.argv) == 5):\n" + + " admin_server_url = sys.argv[4]\n" + + "else:\n" + + " admin_server_url = None\n" + + "\n" + + "domain_path='/shared/domain/%s' % domain_name\n" + + "\n" + + "print 'admin username is %s' % admin_username\n" + + "print 'domain path is %s' % domain_path\n" + + "print 'server name is %s' % server_name\n" + + "print 'admin server url is %s' % admin_server_url\n" + + "\n" + + "# Encrypt the admin username and password\n" + + "adminUsernameEncrypted=encrypt(admin_username, domain_path)\n" + + "adminPasswordEncrypted=encrypt(admin_password, domain_path)\n" + + "\n" + + "print 'Create boot.properties files for this server'\n" + + "\n" + + "# Define the folder path\n" + + "secdir='%s/servers/%s/security' % (domain_path, server_name)\n" + + "\n" + + "# Create the security folder (if it does not already exist)\n" + + "try:\n" + + " os.makedirs(secdir)\n" + + "except OSError:\n" + + " if not os.path.isdir(secdir):\n" + + " raise\n" + + "\n" + + "print 'writing boot.properties to %s/servers/%s/security/boot.properties' % (domain_path, server_name)\n" + + "\n" + + "bpFile=open('%s/servers/%s/security/boot.properties' % (domain_path, server_name), 'w+')\n" + + "bpFile.write(\"username=%s\\n\" % adminUsernameEncrypted)\n" + + "bpFile.write(\"password=%s\\n\" % adminPasswordEncrypted)\n" + + "bpFile.close()\n" + + "\n" + + "service_name = domain_uid + \"-\" + server_name\n" + + "\n" + + "# Update node manager listen address\n" + + "if admin_server_url is not None:\n" + + " connect(admin_username, admin_password, admin_server_url)\n" + + " serverConfig()\n" + + " server=cmo.lookupServer(server_name)\n" + + " machineName=server.getMachine().getName()\n" + + " print 'Name of machine assigned to server %s is %s' % (server_name, machineName)\n" + + "\n" + + " if machineName is not None:\n" + + " print 'Updating listen address of machine %s' % machineName\n" + + " try:\n" + + " edit()\n" + + " startEdit(60000, 120000, 'true')\n" + + " cd('/')\n" + + " machine=cmo.lookupMachine(machineName)\n" + + " print 'Machine is %s' % machine\n" + + " nm=machine.getNodeManager()\n" + + " nm.setListenAddress(service_name)\n" + + " nm.setNMType('Plain')\n" + + " save()\n" + + " activate()\n" + + " print 'Updated listen address of machine %s to %s' % (machineName, service_name)\n" + + " except:\n" + + " cancelEdit('y')\n" + + "\n" + + "# Connect to nodemanager and start server\n" + + "try:\n" + + " nmConnect(admin_username, admin_password, service_name, '5556', domain_name, domain_path, 'plain')\n" + + " nmStart(server_name)\n" + + " nmDisconnect()\n" + + "except WLSTException, e:\n" + + " nmDisconnect()\n" + + " print e\n" + + "\n" + + "# Exit WLST\n" + + "exit()\n"; + + private static final String STOP_SERVER_SHELL_SCRIPT = "#!/bin/bash\n" + + "\n" + + "echo \"Stop the server\"\n" + + "\n" + + "admin_server_t3_url=\n" + + "if [ -n \"$4\" ]; then\n" + + " admin_server_t3_url=t3://$1-$4:$5\n" + + "fi\n" + + "\n" + + "wlst.sh -skipWLSModuleScanning /weblogic-operator/scripts/stop-server.py $1 $2 $3 $admin_server_t3_url\n" + + "\n" + + "# Return status of 2 means failed to stop a server through the NodeManager.\n" + + "# Look to see if there is a server process that can be killed.\n" + + "if [ $? -eq 2 ]; then\n" + + " pid=$(jps -v | grep '[D]weblogic.Name=$2' | awk '{print $1}')\n" + + " if [ ! -z $pid ]; then\n" + + " echo \"Killing the server process $pid\"\n" + + " kill -15 $pid\n" + + " fi\n" + + "fi\n" + + "\n"; + + private static final String STOP_SERVER_PYTHON_SCRIPT = "#\n" + + "# +++ Start of common code for reading domain secrets\n" + + "\n" + + "# Read username secret\n" + + "file = open('/weblogic-operator/secrets/username', 'r')\n" + + "admin_username = file.read()\n" + + "file.close()\n" + + "\n" + + "# Read password secret\n" + + "file = open('/weblogic-operator/secrets/password', 'r')\n" + + "admin_password = file.read()\n" + + "file.close()\n" + + "\n" + + "# +++ End of common code for reading domain secrets\n" + + "#\n" + + "domain_uid = sys.argv[1]\n" + + "server_name = sys.argv[2]\n" + + "domain_name = sys.argv[3]\n" + + "if (len(sys.argv) == 5):\n" + + " admin_server_url = sys.argv[4]\n" + + "else:\n" + + " admin_server_url = None\n" + + "\n" + + "print 'admin_server_url is %s' % admin_server_url\n" + + "\n" + + "# Update node manager listen address to None\n" + + "if admin_server_url is not None:\n" + + " connect(admin_username, admin_password, admin_server_url)\n" + + " serverConfig()\n" + + " server=cmo.lookupServer(server_name)\n" + + " machineName=server.getMachine().getName()\n" + + " print 'Name of machine assigned to server %s is %s' % (server_name, machineName)\n" + + "\n" + + " if machineName is not None:\n" + + " print 'Updating listen address of machine %s' % machineName\n" + + " try:\n" + + " edit()\n" + + " startEdit(60000, 120000, 'true')\n" + + " cd('/')\n" + + " machine=cmo.lookupMachine(machineName)\n" + + " print 'Machine is %s' % machine\n" + + " nm=machine.getNodeManager()\n" + + " nm.setListenAddress(None)\n" + + " save()\n" + + " activate()\n" + + " print 'Updated listen address of machine %s to None' % machineName\n" + + " except:\n" + + " cancelEdit('y')\n" + + "\n" + + "service_name = domain_uid + \"-\" + server_name\n" + + "domain_path='/shared/domain/%s' % domain_name\n" + + "\n" + + "# Connect to nodemanager and stop server\n" + + "try:\n" + + " nmConnect(admin_username, admin_password, service_name, '5556', domain_name, domain_path, 'plain')\n" + + "except:\n" + + " print('Failed to connect to the NodeManager')\n" + + " exit(exitcode=2)\n" + + "\n" + + "# Kill the server\n" + + "try:\n" + + " nmKill('$2')\n" + + "except:\n" + + " print('Connected to the NodeManager, but failed to stop the server')\n" + + " exit(exitcode=2)\n" + + "\n" + + "# Exit WLST\n" + + "nmDisconnect()\n" + + "exit()\n"; + private ConfigMapHelper() {} /** @@ -101,7 +410,15 @@ public NextAction apply(Packet packet) { "fi\n" + "\n" + "exit 0"); - + + data.put("startServer.sh", START_SERVER_SHELL_SCRIPT); + + data.put("start-server.py", START_SERVER_PYTHON_SCRIPT); + + data.put("stopServer.sh", STOP_SERVER_SHELL_SCRIPT); + + data.put("stop-server.py", STOP_SERVER_PYTHON_SCRIPT); + cm.setData(data); Step read = CallBuilder.create().readConfigMapAsync(name, namespace, new ResponseStep(next) { diff --git a/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java b/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java index db735453dda..6ad4dd4304a 100644 --- a/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java +++ b/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java @@ -143,7 +143,10 @@ public NextAction apply(Packet packet) { V1Lifecycle lifecycle = new V1Lifecycle(); V1Handler preStopHandler = new V1Handler(); V1ExecAction lifecycleExecAction = new V1ExecAction(); - lifecycleExecAction.addCommandItem("/shared/domain/" + weblogicDomainName + "/nodemgr_home/stopServer.sh"); + lifecycleExecAction.addCommandItem("/weblogic-operator/scripts/stopServer.sh"); + lifecycleExecAction.addCommandItem(weblogicDomainUID); + lifecycleExecAction.addCommandItem(spec.getAsName()); + lifecycleExecAction.addCommandItem(weblogicDomainName); preStopHandler.setExec(lifecycleExecAction); lifecycle.setPreStop(preStopHandler); container.setLifecycle(lifecycle); @@ -165,9 +168,10 @@ public NextAction apply(Packet packet) { volumeMountScripts.setReadOnly(true); container.addVolumeMountsItem(volumeMountScripts); - container.addCommandItem("/shared/domain/" + weblogicDomainName + "/nodemgr_home/startServer.sh"); + container.addCommandItem("/weblogic-operator/scripts/startServer.sh"); container.addCommandItem(weblogicDomainUID); container.addCommandItem(spec.getAsName()); + container.addCommandItem(weblogicDomainName); V1Probe readinessProbe = new V1Probe(); V1ExecAction readinessAction = new V1ExecAction(); @@ -536,9 +540,10 @@ public NextAction apply(Packet packet) { V1Lifecycle lifecycle = new V1Lifecycle(); V1Handler preStop = new V1Handler(); V1ExecAction exec = new V1ExecAction(); - exec.addCommandItem("/shared/domain/" + weblogicDomainName + "/nodemgr_home/stopServer.sh"); + exec.addCommandItem("/weblogic-operator/scripts/stopServer.sh"); exec.addCommandItem(weblogicDomainUID); exec.addCommandItem(weblogicServerName); + exec.addCommandItem(weblogicDomainName); exec.addCommandItem(spec.getAsName()); exec.addCommandItem(String.valueOf(spec.getAsPort())); preStop.setExec(exec); @@ -562,9 +567,10 @@ public NextAction apply(Packet packet) { volumeMountScripts.setReadOnly(true); container.addVolumeMountsItem(volumeMountScripts); - container.addCommandItem("/shared/domain/" + weblogicDomainName + "/nodemgr_home/startServer.sh"); + container.addCommandItem("/weblogic-operator/scripts/startServer.sh"); container.addCommandItem(weblogicDomainUID); container.addCommandItem(weblogicServerName); + container.addCommandItem(weblogicDomainName); container.addCommandItem(spec.getAsName()); container.addCommandItem(String.valueOf(spec.getAsPort())); diff --git a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetriever.java b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetriever.java index fb54e2ca4e5..3ac7668de46 100644 --- a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetriever.java +++ b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetriever.java @@ -64,16 +64,6 @@ public class WlsConfigRetriever { // REST URL for canceling a WebLogic edit session private static final String CANCEL_EDIT_SESSION_URL = "/management/weblogic/latest/edit/changeManager/cancelEdit"; - // REST URL for creating a WebLogic named edit session - private static final String CREATE_EDIT_SESSION = "/management/weblogic/latest/domainRuntime/editSessionConfigurationManager/editSessionConfigurations"; - - // Name of WebLogic named edit session to be used by weblogic operator - private static final String EDIT_SESSION_NAME = "weblogic-operator"; - - // Payload for creating a WebLogic named edit session - private static final String CREATE_EDIT_SESSION_PAYLOAD = - " { name: '" + EDIT_SESSION_NAME + "' , description: 'edit session for weblogic operator' } "; - /** * Constructor. * @@ -493,9 +483,6 @@ private static boolean updateDynamicClusterSizeWithServiceURL(final WlsClusterCo boolean result = false; try { - // create WebLogic named edit session -// httpClient.executePostUrlOnServiceClusterIP(CREATE_EDIT_SESSION, serviceURL, CREATE_EDIT_SESSION_PAYLOAD); - // start a WebLogic edit session // httpClient.executePostUrlOnServiceClusterIP(START_EDIT_SESSION_URL, serviceURL, ""); // LOGGER.info(MessageKeys.WLS_EDIT_SESSION_STARTED); From cd1b3a818ea121e89c005a20bcd19059a1c419aa Mon Sep 17 00:00:00 2001 From: anthony_lai Date: Thu, 15 Mar 2018 10:23:18 -0700 Subject: [PATCH 017/186] updated message IDs, minor code cleanup --- .../operator/logging/MessageKeys.java | 17 ++++---- .../wlsconfig/WlsConfigRetriever.java | 42 +++++++------------ src/main/resources/Operator.properties | 17 ++++---- 3 files changed, 28 insertions(+), 48 deletions(-) diff --git a/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java b/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java index 633285ac71e..a170612beca 100644 --- a/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java +++ b/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java @@ -139,14 +139,11 @@ private MessageKeys() { public static final String SERVER_SERVICE_DELETED = "WLSKO-0123"; public static final String CLUSTER_SERVICE_DELETED = "WLSKO-0124"; public static final String INGRESS_DELETED = "WLSKO-0125"; - public static final String WLS_UPDATE_CLUSTER_SIZE_STARTING = "WLSKO-0201"; - public static final String WLS_UPDATE_CLUSTER_SIZE_FAILED = "WLSKO-0202"; - public static final String WLS_UPDATE_CLUSTER_SIZE_TIMED_OUT = "WLSKO-0203"; - public static final String WLS_UPDATE_CLUSTER_SIZE_INVALID_CLUSTER = "WLSKO-0204"; - public static final String WLS_CLUSTER_SIZE_UPDATED = "WLSKO-0205"; - public static final String WLS_SERVER_TEMPLATE_NOT_FOUND = "WLSKO-0206"; - public static final String WLS_CREATING_MACHINE = "WLSKO-0207"; - public static final String WLS_EDIT_SESSION_STARTED = "WLSKO-0208"; - public static final String WLS_EDIT_SESSION_ACTIVATED = "WLSKO-0209"; - public static final String WLS_EDIT_SESSION_CANCELLED = "WLSKO-0210"; + public static final String WLS_UPDATE_CLUSTER_SIZE_STARTING = "WLSKO-0126"; + public static final String WLS_UPDATE_CLUSTER_SIZE_FAILED = "WLSKO-0127"; + public static final String WLS_UPDATE_CLUSTER_SIZE_TIMED_OUT = "WLSKO-0128"; + public static final String WLS_UPDATE_CLUSTER_SIZE_INVALID_CLUSTER = "WLSKO-0129"; + public static final String WLS_CLUSTER_SIZE_UPDATED = "WLSKO-0130"; + public static final String WLS_SERVER_TEMPLATE_NOT_FOUND = "WLSKO-0131"; + public static final String WLS_CREATING_MACHINE = "WLSKO-0132"; } diff --git a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetriever.java b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetriever.java index 3ac7668de46..063d7f4b5cb 100644 --- a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetriever.java +++ b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsConfigRetriever.java @@ -482,35 +482,21 @@ private static boolean updateDynamicClusterSizeWithServiceURL(final WlsClusterCo LOGGER.entering(); boolean result = false; - try { - // start a WebLogic edit session -// httpClient.executePostUrlOnServiceClusterIP(START_EDIT_SESSION_URL, serviceURL, ""); -// LOGGER.info(MessageKeys.WLS_EDIT_SESSION_STARTED); - - // Create machine(s) - String newMachineNames[] = wlsClusterConfig.getMachineNamesForDynamicServers(machineNamePrefix, targetClusterSize); - for (String machineName: newMachineNames) { - LOGGER.info(MessageKeys.WLS_CREATING_MACHINE, machineName); - httpClient.executePostUrlOnServiceClusterIP(WlsMachineConfig.getCreateUrl(), - serviceURL, WlsMachineConfig.getCreatePayload(machineName), true); - } - - // Update the dynamic cluster size of the WebLogic cluster - String jsonResult = httpClient.executePostUrlOnServiceClusterIP( - wlsClusterConfig.getUpdateDynamicClusterSizeUrl(), - serviceURL, - wlsClusterConfig.getUpdateDynamicClusterSizePayload(targetClusterSize), true).getResponse(); - -// // activate the WebLogic edit session -// httpClient.executePostUrlOnServiceClusterIP(ACTIVATE_EDIT_SESSION_URL, serviceURL, "", true); - - result = wlsClusterConfig.checkUpdateDynamicClusterSizeJsonResult(jsonResult); -// LOGGER.info(MessageKeys.WLS_EDIT_SESSION_ACTIVATED); - } catch (HTTPException httpException) { - // cancel the WebLogic edit session -// httpClient.executePostUrlOnServiceClusterIP(CANCEL_EDIT_SESSION_URL, serviceURL, ""); -// LOGGER.info(MessageKeys.WLS_EDIT_SESSION_CANCELLED); + // Create machine(s) + String newMachineNames[] = wlsClusterConfig.getMachineNamesForDynamicServers(machineNamePrefix, targetClusterSize); + for (String machineName: newMachineNames) { + LOGGER.info(MessageKeys.WLS_CREATING_MACHINE, machineName); + httpClient.executePostUrlOnServiceClusterIP(WlsMachineConfig.getCreateUrl(), + serviceURL, WlsMachineConfig.getCreatePayload(machineName)); } + + // Update the dynamic cluster size of the WebLogic cluster + String jsonResult = httpClient.executePostUrlOnServiceClusterIP( + wlsClusterConfig.getUpdateDynamicClusterSizeUrl(), + serviceURL, + wlsClusterConfig.getUpdateDynamicClusterSizePayload(targetClusterSize)).getResponse(); + + result = wlsClusterConfig.checkUpdateDynamicClusterSizeJsonResult(jsonResult); LOGGER.exiting(result); return result; } diff --git a/src/main/resources/Operator.properties b/src/main/resources/Operator.properties index 9ef13f39e72..cb65d1b0d8a 100644 --- a/src/main/resources/Operator.properties +++ b/src/main/resources/Operator.properties @@ -124,13 +124,10 @@ WLSKO-0122=Pod for domain with domainUID {0} in namespace {1} and with server na WLSKO-0123=Service for domain with domainUID {0} in namespace {1} and with server name {2} deleted; validating domain WLSKO-0124=Service for domain with domainUID {0} in namespace {1} and with cluster name {2} deleted; validating domain WLSKO-0125=Ingress for domain with domainUID {0} in namespace {1} and with cluster name {2} deleted; validating domain -WLSKO-0201=Updating cluster size for WebLogic dynamic cluster {0} to {1} -WLSKO-0202=Failed to update WebLogic dynamic cluster size for cluster {0} due to exception: {1} -WLSKO-0203=Failed to update WebLogic dynamic cluster size for cluster {0} within timeout of {1} milliseconds -WLSKO-0204=Failed to update WebLogic dynamic cluster size for cluster {0}. Cluster is not a dynamic cluster -WLSKO-0205=Updated cluster size for WebLogic dynamic cluster {0} to {1}. Time taken {2} ms -WLSKO-0206=Cannot find WebLogic server template with name {0} which is referenced by WebLogic cluster {1} -WLSKO-0207=Creating WebLogic machine named {0} -WLSKO-0208=WebLogic edit session started -WLSKO-0209=WebLogic edit session activated -WLSKO-0210=WebLogic edit session cancelled +WLSKO-0126=Updating cluster size for WebLogic dynamic cluster {0} to {1} +WLSKO-0127=Failed to update WebLogic dynamic cluster size for cluster {0} due to exception: {1} +WLSKO-0128=Failed to update WebLogic dynamic cluster size for cluster {0} within timeout of {1} milliseconds +WLSKO-0129=Failed to update WebLogic dynamic cluster size for cluster {0}. Cluster is not a dynamic cluster +WLSKO-0130=Updated cluster size for WebLogic dynamic cluster {0} to {1}. Time taken {2} ms +WLSKO-0131=Cannot find WebLogic server template with name {0} which is referenced by WebLogic cluster {1} +WLSKO-0132=Creating WebLogic machine named {0} From 3e40c46a137f2cacd6c9e220205baf19aca604ce Mon Sep 17 00:00:00 2001 From: anthony_lai Date: Mon, 19 Mar 2018 17:44:49 -0700 Subject: [PATCH 018/186] minor fix to RestBackendImpl verifyWLSConfiguredClusterCapacity, update domain-job-template.yaml to not set listenaddress in NodeManagerMBean created for configured servers --- kubernetes/internal/domain-job-template.yaml | 1 - .../java/oracle/kubernetes/operator/rest/RestBackendImpl.java | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/kubernetes/internal/domain-job-template.yaml b/kubernetes/internal/domain-job-template.yaml index 40eebb24271..b822b144b57 100644 --- a/kubernetes/internal/domain-job-template.yaml +++ b/kubernetes/internal/domain-job-template.yaml @@ -197,7 +197,6 @@ data: cd('Machine/%s' % machineName) create(machineName, 'NodeManager') cd('NodeManager/%s' % machineName) - set('ListenAddress', '%DOMAIN_UID%-%MANAGED_SERVER_NAME_BASE%%s' % msIndex) set('NMType', 'Plain') # Create a cluster diff --git a/src/main/java/oracle/kubernetes/operator/rest/RestBackendImpl.java b/src/main/java/oracle/kubernetes/operator/rest/RestBackendImpl.java index 1d068c7470c..a6caae0c43b 100644 --- a/src/main/java/oracle/kubernetes/operator/rest/RestBackendImpl.java +++ b/src/main/java/oracle/kubernetes/operator/rest/RestBackendImpl.java @@ -382,7 +382,7 @@ private void verifyWLSConfiguredClusterCapacity(ClientHolder client, String name // Verify the current configured cluster size int clusterSize = wlsClusterConfig.getClusterSize(); if (wlsClusterConfig.hasDynamicServers()) { - clusterSize += wlsClusterConfig.getDynamicClusterSize(); + clusterSize += wlsClusterConfig.getMaxDynamicClusterSize(); } if (managedServerCount > clusterSize) { throw createWebApplicationException(Status.BAD_REQUEST, MessageKeys.SCALE_COUNT_GREATER_THAN_CONFIGURED, managedServerCount, clusterSize, cluster, cluster); From 88c97341d1231c8776630383a08758179222ce4d Mon Sep 17 00:00:00 2001 From: Anthony Lai Date: Thu, 22 Mar 2018 12:23:03 -0700 Subject: [PATCH 019/186] fixed previous merge of create-weblogic-domain.sh. fixed unit test in CreateDomainGeneratedFilesBaseTest to reflect changes for moving files to configMap --- kubernetes/internal/create-weblogic-domain.sh | 2 ++ .../CreateDomainGeneratedFilesBaseTest.java | 30 +++++++++---------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/kubernetes/internal/create-weblogic-domain.sh b/kubernetes/internal/create-weblogic-domain.sh index abf4cf445a6..97394c077b5 100755 --- a/kubernetes/internal/create-weblogic-domain.sh +++ b/kubernetes/internal/create-weblogic-domain.sh @@ -414,6 +414,7 @@ function createYamlFiles { sed -i -e "s:%T3_CHANNEL_PORT%:${t3ChannelPort}:g" ${jobOutput} sed -i -e "s:%T3_PUBLIC_ADDRESS%:${t3PublicAddress}:g" ${jobOutput} sed -i -e "s:%CLUSTER_NAME%:${clusterName}:g" ${jobOutput} + sed -i -e "s:%CLUSTER_TYPE%:${clusterType}:g" ${jobOutput} # Generate the yaml to create the domain custom resource echo Generating ${dcrOutput} @@ -440,6 +441,7 @@ function createYamlFiles { sed -i -e "s:%INITIAL_MANAGED_SERVER_REPLICAS%:${initialManagedServerReplicas}:g" ${dcrOutput} sed -i -e "s:%EXPOSE_T3_CHANNEL_PREFIX%:${exposeAdminT3ChannelPrefix}:g" ${dcrOutput} sed -i -e "s:%CLUSTER_NAME%:${clusterName}:g" ${dcrOutput} + sed -i -e "s:%CLUSTER_TYPE%:${clusterType}:g" ${dcrOutput} sed -i -e "s:%EXPOSE_ADMIN_PORT_PREFIX%:${exposeAdminNodePortPrefix}:g" ${dcrOutput} sed -i -e "s:%ADMIN_NODE_PORT%:${adminNodePort}:g" ${dcrOutput} sed -i -e "s:%JAVA_OPTIONS%:${javaOptions}:g" ${dcrOutput} diff --git a/src/test/java/oracle/kubernetes/operator/create/CreateDomainGeneratedFilesBaseTest.java b/src/test/java/oracle/kubernetes/operator/create/CreateDomainGeneratedFilesBaseTest.java index 18bcb23a713..9693b64b778 100644 --- a/src/test/java/oracle/kubernetes/operator/create/CreateDomainGeneratedFilesBaseTest.java +++ b/src/test/java/oracle/kubernetes/operator/create/CreateDomainGeneratedFilesBaseTest.java @@ -189,7 +189,7 @@ protected V1Job getExpectedCreateWeblogicDomainJob() { .secretName(getInputs().getWeblogicCredentialsSecretName())))))); } - //@Test TODO- INVESTIGATE! + @Test public void generatesCorrect_createWeblogicDomainConfigMap() throws Exception { // The config map contains several properties that contain shell and wlst scripts // that we don't want to duplicate in the test. However, part of their text @@ -261,27 +261,24 @@ protected void assertThatActualReadDomainSecretPyIsCorrect(String actualReadDoma protected void assertThatActualCreateDomainScriptShIsCorrect(String actualCreateDomainScriptSh) { /* create-domain-script.sh: |- + #!/bin/bash + # + + # Include common utility functions + source /u01/weblogic/utility.sh + export DOMAIN_HOME=${SHARED_PATH}/domain/%DOMAIN_NAME% - echo "AdminURL=http\://$3\:%ADMIN_PORT%" >> ${startProp} - nmConnect(admin_username, admin_password, '$1-$2', '5556', '%DOMAIN_NAME%', '${DOMAIN_HOME}', 'plain') - nmConnect(admin_username, admin_password, '$1-$2', '5556', '%DOMAIN_NAME%', '${DOMAIN_HOME}', 'plain') - createNodeMgrHome %DOMAIN_UID% %ADMIN_SERVER_NAME% - createStartScript %DOMAIN_UID% %ADMIN_SERVER_NAME% - createStopScript %DOMAIN_UID% %ADMIN_SERVER_NAME% - while [ $index -lt %NUMBER_OF_MS% ] - createNodeMgrHome %DOMAIN_UID% %MANAGED_SERVER_NAME_BASE%${index} %DOMAIN_UID%-%ADMIN_SERVER_NAME% - createStartScript %DOMAIN_UID% %MANAGED_SERVER_NAME_BASE%${index} - createStopScript %DOMAIN_UID% %MANAGED_SERVER_NAME_BASE%${index} + + # Create the domain + wlst.sh -skipWLSModuleScanning /u01/weblogic/create-domain.py + + echo "Successfully Completed" */ assertThat( actualCreateDomainScriptSh, containsRegexps( - getInputs().getDomainUID(), getInputs().getDomainName(), - getInputs().getAdminServerName(), - getInputs().getManagedServerNameBase(), - getInputs().getDomainUID() + "-" + getInputs().getAdminServerName(), - "index -lt " + getInputs().getConfiguredManagedServerCount())); + "wlst.sh -skipWLSModuleScanning /u01/weblogic/create-domain.py")); } protected void assertThatActualCreateDomainPyIsCorrect(String actualCreateDomainPy) { @@ -324,6 +321,7 @@ protected void assertThatActualCreateDomainPyIsCorrect(String actualCreateDomain containsRegexps( getInputs().getDomainName(), getInputs().getClusterName(), + getInputs().getClusterType(), getInputs().getAdminServerName(), getInputs().getManagedServerNameBase(), getInputs().getDomainUID() + "-" + getInputs().getAdminServerName(), From 5659a6c863e08239aa992be4becaf9219297c1a2 Mon Sep 17 00:00:00 2001 From: anthony_lai Date: Fri, 23 Mar 2018 13:25:10 -0700 Subject: [PATCH 020/186] no longer reset NodeManagerMBean listenAddress in server shutdown script --- .../operator/helpers/ConfigMapHelper.java | 42 ++----------------- .../operator/helpers/PodHelper.java | 2 - 2 files changed, 4 insertions(+), 40 deletions(-) diff --git a/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java b/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java index f49a9a306c3..586bf26188a 100644 --- a/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java +++ b/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java @@ -215,7 +215,7 @@ public class ConfigMapHelper { " print 'Updating listen address of machine %s' % machineName\n" + " try:\n" + " edit()\n" + - " startEdit(60000, 120000, 'true')\n" + + " startEdit(120000, 120000, 'true')\n" + " cd('/')\n" + " machine=cmo.lookupMachine(machineName)\n" + " print 'Machine is %s' % machine\n" + @@ -227,6 +227,7 @@ public class ConfigMapHelper { " print 'Updated listen address of machine %s to %s' % (machineName, service_name)\n" + " except:\n" + " cancelEdit('y')\n" + + " disconnect()\n" + "\n" + "# Connect to nodemanager and start server\n" + "try:\n" + @@ -244,12 +245,7 @@ public class ConfigMapHelper { "\n" + "echo \"Stop the server\"\n" + "\n" + - "admin_server_t3_url=\n" + - "if [ -n \"$4\" ]; then\n" + - " admin_server_t3_url=t3://$1-$4:$5\n" + - "fi\n" + - "\n" + - "wlst.sh -skipWLSModuleScanning /weblogic-operator/scripts/stop-server.py $1 $2 $3 $admin_server_t3_url\n" + + "wlst.sh -skipWLSModuleScanning /weblogic-operator/scripts/stop-server.py $1 $2 $3\n" + "\n" + "# Return status of 2 means failed to stop a server through the NodeManager.\n" + "# Look to see if there is a server process that can be killed.\n" + @@ -280,36 +276,6 @@ public class ConfigMapHelper { "domain_uid = sys.argv[1]\n" + "server_name = sys.argv[2]\n" + "domain_name = sys.argv[3]\n" + - "if (len(sys.argv) == 5):\n" + - " admin_server_url = sys.argv[4]\n" + - "else:\n" + - " admin_server_url = None\n" + - "\n" + - "print 'admin_server_url is %s' % admin_server_url\n" + - "\n" + - "# Update node manager listen address to None\n" + - "if admin_server_url is not None:\n" + - " connect(admin_username, admin_password, admin_server_url)\n" + - " serverConfig()\n" + - " server=cmo.lookupServer(server_name)\n" + - " machineName=server.getMachine().getName()\n" + - " print 'Name of machine assigned to server %s is %s' % (server_name, machineName)\n" + - "\n" + - " if machineName is not None:\n" + - " print 'Updating listen address of machine %s' % machineName\n" + - " try:\n" + - " edit()\n" + - " startEdit(60000, 120000, 'true')\n" + - " cd('/')\n" + - " machine=cmo.lookupMachine(machineName)\n" + - " print 'Machine is %s' % machine\n" + - " nm=machine.getNodeManager()\n" + - " nm.setListenAddress(None)\n" + - " save()\n" + - " activate()\n" + - " print 'Updated listen address of machine %s to None' % machineName\n" + - " except:\n" + - " cancelEdit('y')\n" + "\n" + "service_name = domain_uid + \"-\" + server_name\n" + "domain_path='/shared/domain/%s' % domain_name\n" + @@ -323,7 +289,7 @@ public class ConfigMapHelper { "\n" + "# Kill the server\n" + "try:\n" + - " nmKill('$2')\n" + + " nmKill(server_name)\n" + "except:\n" + " print('Connected to the NodeManager, but failed to stop the server')\n" + " exit(exitcode=2)\n" + diff --git a/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java b/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java index 6ad4dd4304a..7dec5e6344a 100644 --- a/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java +++ b/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java @@ -544,8 +544,6 @@ public NextAction apply(Packet packet) { exec.addCommandItem(weblogicDomainUID); exec.addCommandItem(weblogicServerName); exec.addCommandItem(weblogicDomainName); - exec.addCommandItem(spec.getAsName()); - exec.addCommandItem(String.valueOf(spec.getAsPort())); preStop.setExec(exec); lifecycle.setPreStop(preStop); container.setLifecycle(lifecycle); From 349efe1bc0de8ce6e18910246a61ba5b9163c794 Mon Sep 17 00:00:00 2001 From: anthony_lai Date: Mon, 26 Mar 2018 14:56:52 -0700 Subject: [PATCH 021/186] clusterType create domain job input param - update valid values to upper case, add validation and unit tests --- kubernetes/create-weblogic-domain-inputs.yaml | 5 ++-- .../create-weblogic-domain-job-template.yaml | 4 +-- kubernetes/internal/create-weblogic-domain.sh | 22 ++++++++++++-- src/integration-tests/bash/run.sh | 12 ++++---- .../CreateDomainGeneratedFilesBaseTest.java | 1 + ...ratedFilesOptionalFeaturesEnabledTest.java | 1 + .../operator/create/CreateDomainInputs.java | 3 ++ .../create/CreateDomainInputsFileTest.java | 2 +- .../CreateDomainInputsValidationTest.java | 30 +++++++++++++++++++ 9 files changed, 67 insertions(+), 13 deletions(-) diff --git a/kubernetes/create-weblogic-domain-inputs.yaml b/kubernetes/create-weblogic-domain-inputs.yaml index ed274c126c6..1665590d460 100644 --- a/kubernetes/create-weblogic-domain-inputs.yaml +++ b/kubernetes/create-weblogic-domain-inputs.yaml @@ -14,8 +14,9 @@ domainName: base_domain # This id must be lowercase and unique across all domains in a Kubernetes cluster. #domainUID: domain1 -# Type of WebLogic Cluster (configured or dynamic) -clusterType: dynamic +# Type of WebLogic Cluster +# Legal values are "CONFIGURED" or "DYNAMIC" +clusterType: DYNAMIC # Determines which WebLogic Servers the Operator will start up # Legal values are "NONE", "ALL", "ADMIN", "SPECIFIED", or "AUTO" diff --git a/kubernetes/internal/create-weblogic-domain-job-template.yaml b/kubernetes/internal/create-weblogic-domain-job-template.yaml index f17980798a0..723daf1b714 100644 --- a/kubernetes/internal/create-weblogic-domain-job-template.yaml +++ b/kubernetes/internal/create-weblogic-domain-job-template.yaml @@ -186,7 +186,7 @@ data: # Configure machines # ====================== - if cluster_type == "configured": + if cluster_type == "CONFIGURED": for index in range(0, number_of_ms): @@ -203,7 +203,7 @@ data: cd('/') cl=create(cluster_name, 'Cluster') - if cluster_type == "configured": + if cluster_type == "CONFIGURED": # Create managed servers for index in range(0, number_of_ms): diff --git a/kubernetes/internal/create-weblogic-domain.sh b/kubernetes/internal/create-weblogic-domain.sh index 423259efd6d..935ba0ed983 100755 --- a/kubernetes/internal/create-weblogic-domain.sh +++ b/kubernetes/internal/create-weblogic-domain.sh @@ -258,7 +258,25 @@ function validateStartupControl { "AUTO") ;; *) - validationError "Invalid valid for startupControl: ${startupControl}. Valid values are 'NONE', 'ALL', 'ADMIN', 'SPECIFIED', and 'AUTO'." + validationError "Invalid value for startupControl: ${startupControl}. Valid values are 'NONE', 'ALL', 'ADMIN', 'SPECIFIED', and 'AUTO'." + ;; + esac + fi +} + +# +# Function to validate the cluster type value +# +function validateClusterType { + validateInputParamsSpecified clusterType + if [ ! -z "${clusterType}" ]; then + case ${clusterType} in + "CONFIGURED") + ;; + "DYNAMIC") + ;; + *) + validationError "Invalid value for clusterType: ${clusterType}. Valid values are 'CONFIGURED' and 'DYNAMIC'." ;; esac fi @@ -362,6 +380,7 @@ function initialize { validateLoadBalancer initAndValidateOutputDir validateStartupControl + validateClusterType failIfValidationErrors } @@ -466,7 +485,6 @@ function createYamlFiles { sed -i -e "s:%INITIAL_MANAGED_SERVER_REPLICAS%:${initialManagedServerReplicas}:g" ${dcrOutput} sed -i -e "s:%EXPOSE_T3_CHANNEL_PREFIX%:${exposeAdminT3ChannelPrefix}:g" ${dcrOutput} sed -i -e "s:%CLUSTER_NAME%:${clusterName}:g" ${dcrOutput} - sed -i -e "s:%CLUSTER_TYPE%:${clusterType}:g" ${dcrOutput} sed -i -e "s:%EXPOSE_ADMIN_PORT_PREFIX%:${exposeAdminNodePortPrefix}:g" ${dcrOutput} sed -i -e "s:%ADMIN_NODE_PORT%:${adminNodePort}:g" ${dcrOutput} sed -i -e "s:%JAVA_OPTIONS%:${javaOptions}:g" ${dcrOutput} diff --git a/src/integration-tests/bash/run.sh b/src/integration-tests/bash/run.sh index f390bb114f6..779bf73f2e1 100755 --- a/src/integration-tests/bash/run.sh +++ b/src/integration-tests/bash/run.sh @@ -2625,12 +2625,12 @@ function test_suite { op_define oper2 weblogic-operator-2 test2 32001 # DOM_KEY OP_KEY NAMESPACE DOMAIN_UID STARTUP_CONTROL WL_CLUSTER_NAME WL_CLUSTER_TYPE MS_BASE_NAME ADMIN_PORT ADMIN_WLST_PORT ADMIN_NODE_PORT MS_PORT LOAD_BALANCER_WEB_PORT LOAD_BALANCER_DASHBOARD_PORT - dom_define domain1 oper1 default domain1 AUTO cluster-1 dynamic managed-server 7001 30012 30701 8001 30305 30315 - dom_define domain2 oper1 default domain2 AUTO cluster-1 dynamic managed-server 7011 30031 30702 8021 30306 30316 - dom_define domain3 oper1 test1 domain3 AUTO cluster-1 dynamic managed-server 7021 30041 30703 8031 30307 30317 - dom_define domain4 oper2 test2 domain4 AUTO cluster-1 configured managed-server 7041 30051 30704 8041 30308 30318 - dom_define domain5 oper1 default domain5 ADMIN cluster-1 dynamic managed-server 7051 30061 30705 8051 30309 30319 - dom_define domain6 oper1 default domain6 AUTO cluster-1 dynamic managed-server 7061 30071 30706 8061 30310 30320 + dom_define domain1 oper1 default domain1 AUTO cluster-1 DYNAMIC managed-server 7001 30012 30701 8001 30305 30315 + dom_define domain2 oper1 default domain2 AUTO cluster-1 DYNAMIC managed-server 7011 30031 30702 8021 30306 30316 + dom_define domain3 oper1 test1 domain3 AUTO cluster-1 DYNAMIC managed-server 7021 30041 30703 8031 30307 30317 + dom_define domain4 oper2 test2 domain4 AUTO cluster-1 CONFIGURED managed-server 7041 30051 30704 8041 30308 30318 + dom_define domain5 oper1 default domain5 ADMIN cluster-1 DYNAMIC managed-server 7051 30061 30705 8051 30309 30319 + dom_define domain6 oper1 default domain6 AUTO cluster-1 DYNAMIC managed-server 7061 30071 30706 8061 30310 30320 # create namespaces for domains (the operator job creates a namespace if needed) # TODO have the op_define commands themselves create target namespace if it doesn't already exist, or test if the namespace creation is needed in the first place, and if so, ask MikeG to create them as part of domain create job diff --git a/src/test/java/oracle/kubernetes/operator/create/CreateDomainGeneratedFilesBaseTest.java b/src/test/java/oracle/kubernetes/operator/create/CreateDomainGeneratedFilesBaseTest.java index 089f4cb4e80..d6002810d56 100644 --- a/src/test/java/oracle/kubernetes/operator/create/CreateDomainGeneratedFilesBaseTest.java +++ b/src/test/java/oracle/kubernetes/operator/create/CreateDomainGeneratedFilesBaseTest.java @@ -290,6 +290,7 @@ protected void assertThatActualCreateDomainPyIsCorrect(String actualCreateDomain server_port = %MANAGED_SERVER_PORT% cluster_name = "%CLUSTER_NAME%" number_of_ms = %NUMBER_OF_MS% + cluster_type = "%CLUSTER_TYPE%" print('domain_name : [%DOMAIN_NAME%]'); print('admin_port : [%ADMIN_PORT%]'); set('Name', '%DOMAIN_NAME%') diff --git a/src/test/java/oracle/kubernetes/operator/create/CreateDomainGeneratedFilesOptionalFeaturesEnabledTest.java b/src/test/java/oracle/kubernetes/operator/create/CreateDomainGeneratedFilesOptionalFeaturesEnabledTest.java index ccd90a3ffb5..6882b279d50 100644 --- a/src/test/java/oracle/kubernetes/operator/create/CreateDomainGeneratedFilesOptionalFeaturesEnabledTest.java +++ b/src/test/java/oracle/kubernetes/operator/create/CreateDomainGeneratedFilesOptionalFeaturesEnabledTest.java @@ -25,6 +25,7 @@ public static void setup() throws Exception { CreateDomainInputs.newInputs() .exposeAdminNodePort("true") .exposeAdminT3Channel("true") + .clusterType("CONFIGURED") .weblogicImagePullSecretName("test-weblogic-image-pull-secret-name") .loadBalancer(LOAD_BALANCER_TRAEFIK) .weblogicDomainStorageType(STORAGE_TYPE_NFS) diff --git a/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputs.java b/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputs.java index 9b8740d6854..82fd7d1d821 100644 --- a/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputs.java +++ b/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputs.java @@ -39,6 +39,8 @@ public class CreateDomainInputs { public static final String STARTUP_CONTROL_ADMIN = "ADMIN"; public static final String STARTUP_CONTROL_SPECIFIED = "SPECIFIED"; public static final String STARTUP_CONTROL_AUTO = "AUTO"; + public static final String CLUSTER_TYPE_CONFIGURED = "CONFIGURED"; + public static final String CLUSTER_TYPE_DYNAMIC = "DYNAMIC"; private static final String DEFAULT_INPUTS = "kubernetes/create-weblogic-domain-inputs.yaml"; @@ -79,6 +81,7 @@ public static CreateDomainInputs newInputs() throws Exception { .adminPort("7002") .adminServerName("TestAdminServer") .clusterName("TestCluster") + .clusterType(CLUSTER_TYPE_DYNAMIC) .domainName("TestDomain") .domainUID("test-domain-uid") .javaOptions("TestJavaOptions") diff --git a/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputsFileTest.java b/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputsFileTest.java index 99d90514535..43b872a0060 100644 --- a/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputsFileTest.java +++ b/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputsFileTest.java @@ -47,7 +47,7 @@ public void defaultInputsFile_hasCorrectContents() throws Exception { .adminPort("7001") .adminServerName("admin-server") .clusterName("cluster-1") - .clusterType("dynamic") + .clusterType("DYNAMIC") .domainName("base_domain") .domainUID("") .exposeAdminNodePort("false") diff --git a/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputsValidationTest.java b/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputsValidationTest.java index d5be85dca55..e115a8261e7 100644 --- a/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputsValidationTest.java +++ b/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputsValidationTest.java @@ -26,6 +26,7 @@ public class CreateDomainInputsValidationTest { private static final String PARAM_DOMAIN_UID = "domainUID"; private static final String PARAM_STARTUP_CONTROL = "startupControl"; private static final String PARAM_CLUSTER_NAME = "clusterName"; + private static final String PARAM_CLUSTER_TYPE = "clusterType"; private static final String PARAM_CONFIGURED_MANAGED_SERVER_COUNT = "configuredManagedServerCount"; private static final String PARAM_INITIAL_MANAGED_SERVER_REPLICAS = "initialManagedServerReplicas"; private static final String PARAM_MANAGED_SERVER_NAME_BASE = "managedServerNameBase"; @@ -155,6 +156,31 @@ public void createDomain_with_invalidStartupControl_failsAndReturnsError() throw failsAndPrints(invalidEnumParamValueError(PARAM_STARTUP_CONTROL, val))); } + @Test + public void createDomain_with_missingClusterType_failsAndReturnsError() throws Exception { + assertThat( + execCreateDomain(newInputs().clusterType("")), + failsAndPrints(paramMissingError(PARAM_CLUSTER_TYPE))); + } + + @Test + public void createDomain_with_clusterTypeConfigured_succeeds() throws Exception { + createDomain_with_validClusterType_succeeds(CLUSTER_TYPE_CONFIGURED); + } + + @Test + public void createDomain_with_clusterTypeDynamic_succeeds() throws Exception { + createDomain_with_validClusterType_succeeds(CLUSTER_TYPE_DYNAMIC); + } + + @Test + public void createDomain_with_invalidClusterType_failsAndReturnsError() throws Exception { + String val = "Invalid-cluster-type"; + assertThat( + execCreateDomain(newInputs().clusterType(val)), + failsAndPrints(invalidEnumParamValueError(PARAM_CLUSTER_TYPE, val))); + } + @Test public void createDomain_with_missingClusterName_failsAndReturnsError() throws Exception { assertThat( @@ -487,6 +513,10 @@ private void createDomain_with_validStartupControl_succeeds(String startupContro createDomain_with_validInputs_succeeds(newInputs().startupControl(startupControl)); } + private void createDomain_with_validClusterType_succeeds(String clusterType) throws Exception { + createDomain_with_validInputs_succeeds(newInputs().clusterType(clusterType)); + } + private void createDomain_with_validInputs_succeeds(CreateDomainInputs inputs) throws Exception { // throws an error if the inputs are not valid, succeeds otherwise: GeneratedDomainYamlFiles.generateDomainYamlFiles(inputs).remove(); From 57d8fb808de6282c42405482746fdf094469a19c Mon Sep 17 00:00:00 2001 From: Anthony Lai Date: Wed, 28 Mar 2018 09:17:37 -0700 Subject: [PATCH 022/186] removed unused constants from WlsRetriever --- .../kubernetes/operator/wlsconfig/WlsRetriever.java | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsRetriever.java b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsRetriever.java index d00da5e3ef0..f1051db27bf 100644 --- a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsRetriever.java +++ b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsRetriever.java @@ -64,15 +64,6 @@ public class WlsRetriever { // wait time before retrying to read server configured for the cluster from admin server - default is 1s private static final int READ_CONFIG_RETRY_MILLIS = Integer.getInteger("read.config.retry.ms", 1000); - // REST URL for starting a WebLogic edit session - private static final String START_EDIT_SESSION_URL = "/management/weblogic/latest/edit/changeManager/startEdit"; - - // REST URL for activating a WebLogic edit session - private static final String ACTIVATE_EDIT_SESSION_URL = "/management/weblogic/latest/edit/changeManager/activate"; - - // REST URL for canceling a WebLogic edit session - private static final String CANCEL_EDIT_SESSION_URL = "/management/weblogic/latest/edit/changeManager/cancelEdit"; - /** * Constructor. * @@ -615,6 +606,5 @@ String connectAndGetServiceURL() { } } - - + } From 8a78ddd78c2be469d57459ce04295930cd2abc23 Mon Sep 17 00:00:00 2001 From: anthony_lai Date: Wed, 28 Mar 2018 16:51:21 -0700 Subject: [PATCH 023/186] No need to expose nm port for admin server. update PodHelper unit tests --- .../kubernetes/operator/helpers/PodHelper.java | 7 ------- .../operator/create/PodHelperConfigTest.java | 17 ++++++++++++++--- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java b/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java index eff4c074362..68d7af71f9b 100644 --- a/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java +++ b/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java @@ -223,15 +223,8 @@ protected V1Pod computeAdminPodConfig(Packet packet) { V1ContainerPort containerPort = new V1ContainerPort(); containerPort.setContainerPort(spec.getAsPort()); containerPort.setProtocol("TCP"); - containerPort.setName("weblogic"); container.addPortsItem(containerPort); - V1ContainerPort nmPort = new V1ContainerPort(); - nmPort.setContainerPort(5556); - nmPort.setProtocol("TCP"); - nmPort.setName("node-manager"); - container.addPortsItem(nmPort); - V1Lifecycle lifecycle = new V1Lifecycle(); V1Handler preStopHandler = new V1Handler(); V1ExecAction lifecycleExecAction = new V1ExecAction(); diff --git a/src/test/java/oracle/kubernetes/operator/create/PodHelperConfigTest.java b/src/test/java/oracle/kubernetes/operator/create/PodHelperConfigTest.java index 9befe79b724..5ff3d171b41 100644 --- a/src/test/java/oracle/kubernetes/operator/create/PodHelperConfigTest.java +++ b/src/test/java/oracle/kubernetes/operator/create/PodHelperConfigTest.java @@ -5,6 +5,7 @@ import java.util.List; +import io.kubernetes.client.models.V1ContainerPort; import io.kubernetes.client.models.V1PersistentVolumeClaimList; import io.kubernetes.client.models.V1Pod; import static oracle.kubernetes.operator.KubernetesConstants.*; @@ -263,7 +264,7 @@ public V1Pod computeManagedPodConfig(Domain domain, V1PersistentVolumeClaimList packet.put( ProcessingConstants.SERVER_SCAN, // no listen address, no network access points since PodHelper doesn't use them: - new WlsServerConfig(MANAGED_SERVER_NAME, MANAGED_SERVER_PORT, null, null)); + new WlsServerConfig(MANAGED_SERVER_NAME, MANAGED_SERVER_PORT, null, null, false, null, null)); packet.put( ProcessingConstants.CLUSTER_SCAN, // don't attach WlsServerConfigs for the managed server to the WlsClusterConfig @@ -366,6 +367,9 @@ private V1Pod getDesiredManagedServerPodConfigForDefaults(String image, String i .value(MANAGED_OPTION2_VALUE)); pod.getMetadata() .putLabelsItem("weblogic.clusterName", CLUSTER_NAME); + pod.getSpec().getContainers().get(0).addCommandItem(ADMIN_SERVER_NAME).addCommandItem(String.valueOf(ADMIN_SERVER_PORT)); + pod.getSpec().getContainers().get(0).getPorts().get(0).setName("weblogic"); + pod.getSpec().getContainers().get(0).addPortsItem(new V1ContainerPort().protocol("TCP").containerPort(5556).name("node-manager")); return pod; } @@ -397,11 +401,18 @@ private V1Pod getDesiredBaseServerPodConfigForDefaults(String image, String imag .name("weblogic-server") .image(image) .imagePullPolicy(imagePullPolicy) - .addCommandItem("/shared/domain/" + DOMAIN_NAME + "/servers/" + serverName + "/nodemgr_home/startServer.sh") + .addCommandItem("/weblogic-operator/scripts/startServer.sh") + .addCommandItem(DOMAIN_UID) + .addCommandItem(serverName) + .addCommandItem(DOMAIN_NAME) .lifecycle(newLifecycle() .preStop(newHandler() .exec(newExecAction() - .addCommandItem("/shared/domain/" + DOMAIN_NAME + "/servers/" + serverName + "/nodemgr_home/stopServer.sh")))) + .addCommandItem("/weblogic-operator/scripts/stopServer.sh") + .addCommandItem(DOMAIN_UID) + .addCommandItem(serverName) + .addCommandItem(DOMAIN_NAME) + ))) .livenessProbe(newProbe() .initialDelaySeconds(10) .periodSeconds(10) From 4e0da65da59cd377f6afd876a0ef4077576193b0 Mon Sep 17 00:00:00 2001 From: Anthony Lai Date: Thu, 29 Mar 2018 09:05:04 -0700 Subject: [PATCH 024/186] remove leftover merge conflict indicator --- .../oracle/kubernetes/operator/wlsconfig/WlsRetriever.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsRetriever.java b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsRetriever.java index 6fbfc09b729..947028e57ac 100644 --- a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsRetriever.java +++ b/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsRetriever.java @@ -64,14 +64,8 @@ public class WlsRetriever { /** * Constructor. * -<<<<<<< HEAD - * @param clientHelper The ClientHelper to be used to obtain the Kubernetes API Client. * @param namespace The Namespace in which the target Domain is located. * @param asServiceName The name of the Kubernetes Service which provides access to the Admin Server. -======= - * @param namespace The Namespace in which the target Domain is located. - * @param asServiceName The name of the Kubernetes Service which provides access to the Admin Server. ->>>>>>> master * @param adminSecretName The name of the Kubernetes Secret which contains the credentials to authenticate to the Admin Server. * @return The WlsRetriever object for the specified inputs. */ From ca54677693508be85b584c8e1ff9945ab95c49ad Mon Sep 17 00:00:00 2001 From: Ryan Eberhard Date: Fri, 30 Mar 2018 10:48:57 -0400 Subject: [PATCH 025/186] Move files --- .../main/java/oracle/kubernetes/operator/http/HTTPException.java | 0 .../src}/main/java/oracle/kubernetes/operator/http/Result.java | 0 .../java/oracle/kubernetes/operator/wlsconfig/ConfigUpdate.java | 0 .../oracle/kubernetes/operator/wlsconfig/MacroSubstitutor.java | 0 .../src}/main/java/oracle/kubernetes/operator/wlsconfig/Util.java | 0 .../kubernetes/operator/wlsconfig/WlsDynamicServerConfig.java | 0 .../kubernetes/operator/wlsconfig/WlsDynamicServersConfig.java | 0 .../oracle/kubernetes/operator/wlsconfig/WlsMachineConfig.java | 0 .../kubernetes/operator/wlsconfig/MacroSubstitutorTest.java | 0 .../test/java/oracle/kubernetes/operator/wlsconfig/UtilTest.java | 0 .../kubernetes/operator/wlsconfig/WlsDynamicServerConfigTest.java | 0 11 files changed, 0 insertions(+), 0 deletions(-) rename {src => operator/src}/main/java/oracle/kubernetes/operator/http/HTTPException.java (100%) rename {src => operator/src}/main/java/oracle/kubernetes/operator/http/Result.java (100%) rename {src => operator/src}/main/java/oracle/kubernetes/operator/wlsconfig/ConfigUpdate.java (100%) rename {src => operator/src}/main/java/oracle/kubernetes/operator/wlsconfig/MacroSubstitutor.java (100%) rename {src => operator/src}/main/java/oracle/kubernetes/operator/wlsconfig/Util.java (100%) rename {src => operator/src}/main/java/oracle/kubernetes/operator/wlsconfig/WlsDynamicServerConfig.java (100%) rename {src => operator/src}/main/java/oracle/kubernetes/operator/wlsconfig/WlsDynamicServersConfig.java (100%) rename {src => operator/src}/main/java/oracle/kubernetes/operator/wlsconfig/WlsMachineConfig.java (100%) rename {src => operator/src}/test/java/oracle/kubernetes/operator/wlsconfig/MacroSubstitutorTest.java (100%) rename {src => operator/src}/test/java/oracle/kubernetes/operator/wlsconfig/UtilTest.java (100%) rename {src => operator/src}/test/java/oracle/kubernetes/operator/wlsconfig/WlsDynamicServerConfigTest.java (100%) diff --git a/src/main/java/oracle/kubernetes/operator/http/HTTPException.java b/operator/src/main/java/oracle/kubernetes/operator/http/HTTPException.java similarity index 100% rename from src/main/java/oracle/kubernetes/operator/http/HTTPException.java rename to operator/src/main/java/oracle/kubernetes/operator/http/HTTPException.java diff --git a/src/main/java/oracle/kubernetes/operator/http/Result.java b/operator/src/main/java/oracle/kubernetes/operator/http/Result.java similarity index 100% rename from src/main/java/oracle/kubernetes/operator/http/Result.java rename to operator/src/main/java/oracle/kubernetes/operator/http/Result.java diff --git a/src/main/java/oracle/kubernetes/operator/wlsconfig/ConfigUpdate.java b/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/ConfigUpdate.java similarity index 100% rename from src/main/java/oracle/kubernetes/operator/wlsconfig/ConfigUpdate.java rename to operator/src/main/java/oracle/kubernetes/operator/wlsconfig/ConfigUpdate.java diff --git a/src/main/java/oracle/kubernetes/operator/wlsconfig/MacroSubstitutor.java b/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/MacroSubstitutor.java similarity index 100% rename from src/main/java/oracle/kubernetes/operator/wlsconfig/MacroSubstitutor.java rename to operator/src/main/java/oracle/kubernetes/operator/wlsconfig/MacroSubstitutor.java diff --git a/src/main/java/oracle/kubernetes/operator/wlsconfig/Util.java b/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/Util.java similarity index 100% rename from src/main/java/oracle/kubernetes/operator/wlsconfig/Util.java rename to operator/src/main/java/oracle/kubernetes/operator/wlsconfig/Util.java diff --git a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDynamicServerConfig.java b/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDynamicServerConfig.java similarity index 100% rename from src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDynamicServerConfig.java rename to operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDynamicServerConfig.java diff --git a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDynamicServersConfig.java b/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDynamicServersConfig.java similarity index 100% rename from src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDynamicServersConfig.java rename to operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDynamicServersConfig.java diff --git a/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsMachineConfig.java b/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsMachineConfig.java similarity index 100% rename from src/main/java/oracle/kubernetes/operator/wlsconfig/WlsMachineConfig.java rename to operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsMachineConfig.java diff --git a/src/test/java/oracle/kubernetes/operator/wlsconfig/MacroSubstitutorTest.java b/operator/src/test/java/oracle/kubernetes/operator/wlsconfig/MacroSubstitutorTest.java similarity index 100% rename from src/test/java/oracle/kubernetes/operator/wlsconfig/MacroSubstitutorTest.java rename to operator/src/test/java/oracle/kubernetes/operator/wlsconfig/MacroSubstitutorTest.java diff --git a/src/test/java/oracle/kubernetes/operator/wlsconfig/UtilTest.java b/operator/src/test/java/oracle/kubernetes/operator/wlsconfig/UtilTest.java similarity index 100% rename from src/test/java/oracle/kubernetes/operator/wlsconfig/UtilTest.java rename to operator/src/test/java/oracle/kubernetes/operator/wlsconfig/UtilTest.java diff --git a/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsDynamicServerConfigTest.java b/operator/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsDynamicServerConfigTest.java similarity index 100% rename from src/test/java/oracle/kubernetes/operator/wlsconfig/WlsDynamicServerConfigTest.java rename to operator/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsDynamicServerConfigTest.java From eea61ec716133a1c610a23f6666c4f900fe1fd9c Mon Sep 17 00:00:00 2001 From: Anthony Lai Date: Fri, 30 Mar 2018 11:52:05 -0700 Subject: [PATCH 026/186] add comment to never reduce dynamicClusterSize in mbean --- .../oracle/kubernetes/operator/wlsconfig/WlsClusterConfig.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfig.java b/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfig.java index 31768361e19..7e1b8eccee0 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfig.java +++ b/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfig.java @@ -223,6 +223,8 @@ public void validateReplicas(Integer replicas, String machineNamePrefix, // updates, ie, suggestedConfigUpdates is not null, and if replicas value is larger than // the current dynamic cluster size, or if some of the machines to be used for the dynamic // servers are not yet configured. + // + // Note: Never reduce the value of dynamicClusterSize even during scale down if (suggestedConfigUpdates != null) { if (hasDynamicServers()) { if (replicas > getDynamicClusterSize() || !verifyMachinesConfigured(machineNamePrefix, replicas)) { From d886140b4b834af1137ecb747091d7f997f0ae0d Mon Sep 17 00:00:00 2001 From: Anthony Lai Date: Mon, 2 Apr 2018 16:03:23 -0700 Subject: [PATCH 027/186] remove executorservice shutdown --- .../java/oracle/kubernetes/operator/wlsconfig/WlsRetriever.java | 1 - 1 file changed, 1 deletion(-) diff --git a/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsRetriever.java b/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsRetriever.java index 466b3988b91..93aa46b488a 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsRetriever.java +++ b/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsRetriever.java @@ -498,7 +498,6 @@ public boolean updateDynamicClusterSize(final WlsClusterConfig wlsClusterConfig, ScheduledExecutorService executorService = ContainerResolver.getInstance().getContainer().getSPI(ScheduledExecutorService.class); long startTime = System.currentTimeMillis(); Future future = executorService.submit(() -> doUpdateDynamicClusterSize(wlsClusterConfig, machineNamePrefix, targetClusterSize)); - executorService.shutdown(); boolean result = false; try { result = future.get(timeout, TimeUnit.MILLISECONDS); From 73134939023ae9fdfdc522f719692efa91cb0bd4 Mon Sep 17 00:00:00 2001 From: anthony_lai Date: Fri, 6 Apr 2018 11:31:34 -0700 Subject: [PATCH 028/186] remove machines configuration --- .../create-weblogic-domain-job-template.yaml | 8 ++++---- .../operator/helpers/ConfigMapHelper.java | 8 +++++--- .../kubernetes/operator/helpers/PodHelper.java | 13 +++++++------ .../operator/wlsconfig/WlsClusterConfig.java | 3 ++- .../kubernetes/operator/wlsconfig/WlsRetriever.java | 13 +++++++------ .../create/CreateDomainGeneratedFilesBaseTest.java | 4 +++- .../operator/create/PodHelperConfigTest.java | 2 -- .../operator/wlsconfig/WlsClusterConfigTest.java | 4 ++++ 8 files changed, 32 insertions(+), 23 deletions(-) diff --git a/kubernetes/internal/create-weblogic-domain-job-template.yaml b/kubernetes/internal/create-weblogic-domain-job-template.yaml index ca0363cff08..e31ba1ad756 100644 --- a/kubernetes/internal/create-weblogic-domain-job-template.yaml +++ b/kubernetes/internal/create-weblogic-domain-job-template.yaml @@ -188,7 +188,7 @@ data: # Create a cluster # ====================== cd('/') - create(cluster_name, 'Cluster') + cl=create(cluster_name, 'Cluster') if cluster_type == "CONFIGURED": @@ -232,10 +232,10 @@ data: set('ServerNamePrefix', "%MANAGED_SERVER_NAME_BASE%") set('DynamicClusterSize', number_of_ms) set('MaxDynamicClusterSize', 800) - set('CalculatedMachineNames', true) + #set('CalculatedMachineNames', true) set('CalculatedListenPorts', false) - machineNameExpression = '%DOMAIN_UID%-%s-machine*' % cluster_name - set('MachineNameMatchExpression', machineNameExpression) + #machineNameExpression = '%DOMAIN_UID%-%s-machine*' % cluster_name + #set('MachineNameMatchExpression', machineNameExpression) set('Id', 1) print('Done setting attributes for Dynamic Cluster: %s' % cluster_name); diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java index 3e73e3e76f3..21f1824cf82 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java @@ -66,7 +66,7 @@ public class ConfigMapHelper { " cp /shared/domain/$3/bin/startNodeManager.sh ${srvr_nmdir}\n" + "\n" + " # Edit the start nodemanager script to use the home for the server\n" + - " sed -i -e \"s:/shared/domain/$3/nodemanager:/u01/nodemanager:g\" /startNodeManager.sh\n" + + " sed -i -e \"s:/shared/domain/$3/nodemanager:/u01/nodemanager:g\" ${srvr_nmdir}/startNodeManager.sh\n" + "\n" + " # Create startup.properties file\n" + " datadir=${DOMAIN_HOME}/servers/$2/data/nodemanager\n" + @@ -99,8 +99,8 @@ public class ConfigMapHelper { "}\n" + "\n" + "# Check for stale state file and remove if found\"\n" + - "if [ -f ${stateFile} ]; then\n" + - " echo \"Removing stale file ${stateFile}\"\n" + + "if [ -f \"$stateFile\" ]; then\n" + + " echo \"Removing stale file $stateFile\"\n" + " rm ${stateFile}\n" + "fi\n" + "\n" + @@ -205,6 +205,7 @@ public class ConfigMapHelper { "\n" + "service_name = domain_uid + \"-\" + server_name\n" + "\n" + + /* Commented out as we are not configuring machines for servers "# Update node manager listen address\n" + "if admin_server_url is not None:\n" + " connect(admin_username, admin_password, admin_server_url)\n" + @@ -231,6 +232,7 @@ public class ConfigMapHelper { " cancelEdit('y')\n" + " disconnect()\n" + "\n" + + */ "# Connect to nodemanager and start server\n" + "try:\n" + " nmConnect(admin_username, admin_password, service_name, '5556', domain_name, domain_path, 'plain')\n" + diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java index 40d9d5e5224..2540f5ad5e3 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java @@ -639,14 +639,15 @@ protected V1Pod computeManagedPodConfig(TuningParameters configMapHelper, Packet V1ContainerPort containerPort = new V1ContainerPort(); containerPort.setContainerPort(scan.getListenPort()); containerPort.setProtocol("TCP"); - containerPort.setName("weblogic"); +// containerPort.setName("weblogic"); // commented out as we are not exposing node manager port container.addPortsItem(containerPort); - V1ContainerPort nmPort = new V1ContainerPort(); - nmPort.setContainerPort(5556); - nmPort.setProtocol("TCP"); - nmPort.setName("node-manager"); - container.addPortsItem(nmPort); + // Commented out as we are not exposing node manager port +// V1ContainerPort nmPort = new V1ContainerPort(); +// nmPort.setContainerPort(5556); +// nmPort.setProtocol("TCP"); +// nmPort.setName("node-manager"); +// container.addPortsItem(nmPort); V1Lifecycle lifecycle = new V1Lifecycle(); V1Handler preStop = new V1Handler(); diff --git a/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfig.java b/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfig.java index 9b2beda3f5f..af6dc14960d 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfig.java +++ b/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfig.java @@ -228,7 +228,8 @@ public void validateReplicas(Integer replicas, String machineNamePrefix, // Note: Never reduce the value of dynamicClusterSize even during scale down if (suggestedConfigUpdates != null) { if (hasDynamicServers()) { - if (replicas > getDynamicClusterSize() || !verifyMachinesConfigured(machineNamePrefix, replicas)) { +// if (replicas > getDynamicClusterSize() || !verifyMachinesConfigured(machineNamePrefix, replicas)) { + if (replicas > getDynamicClusterSize() ) { suggestedConfigUpdates.add(new DynamicClusterSizeConfigUpdate(this, Math.max(replicas, getDynamicClusterSize()))); } } diff --git a/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsRetriever.java b/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsRetriever.java index 93aa46b488a..b94b56f3603 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsRetriever.java +++ b/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsRetriever.java @@ -562,12 +562,13 @@ private static boolean updateDynamicClusterSizeWithServiceURL(final WlsClusterCo boolean result = false; // Create machine(s) - String newMachineNames[] = wlsClusterConfig.getMachineNamesForDynamicServers(machineNamePrefix, targetClusterSize); - for (String machineName: newMachineNames) { - LOGGER.info(MessageKeys.WLS_CREATING_MACHINE, machineName); - httpClient.executePostUrlOnServiceClusterIP(WlsMachineConfig.getCreateUrl(), - serviceURL, WlsMachineConfig.getCreatePayload(machineName)); - } + // Commented out as we are not configuring machines for servers +// String newMachineNames[] = wlsClusterConfig.getMachineNamesForDynamicServers(machineNamePrefix, targetClusterSize); +// for (String machineName: newMachineNames) { +// LOGGER.info(MessageKeys.WLS_CREATING_MACHINE, machineName); +// httpClient.executePostUrlOnServiceClusterIP(WlsMachineConfig.getCreateUrl(), +// serviceURL, WlsMachineConfig.getCreatePayload(machineName)); +// } // Update the dynamic cluster size of the WebLogic cluster String jsonResult = httpClient.executePostUrlOnServiceClusterIP( diff --git a/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainGeneratedFilesBaseTest.java b/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainGeneratedFilesBaseTest.java index 3765f1a8c6f..353beb104b0 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainGeneratedFilesBaseTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainGeneratedFilesBaseTest.java @@ -317,6 +317,7 @@ protected void assertThatActualCreateDomainPyIsCorrect(String actualCreateDomain cmo.setProductionModeEnabled(%PRODUCTION_MODE_ENABLED%) asbpFile=open('%s/servers/%ADMIN_SERVER_NAME%/security/boot.properties' % domain_path, 'w+') secdir='%s/servers/%MANAGED_SERVER_NAME_BASE%%s/security' % (domain_path, index+1) + set('ServerNamePrefix', '%MANAGED_SERVER_NAME_BASE%") */ assertThat( actualCreateDomainPy, @@ -332,7 +333,8 @@ protected void assertThatActualCreateDomainPyIsCorrect(String actualCreateDomain "number_of_ms *= " + getInputs().getConfiguredManagedServerCount(), "set\\('ListenPort', " + getInputs().getAdminPort() + "\\)", "set\\('PublicPort', " + getInputs().getT3ChannelPort() + "\\)", - "set\\('PublicAddress', '" + getInputs().getT3PublicAddress() + "'\\)")); + "set\\('PublicAddress', '" + getInputs().getT3PublicAddress() + "'\\)", + "set\\('ServerNamePrefix', \"" + getInputs().getManagedServerNameBase() + "\"\\)")); // TBD should we check anything else? } diff --git a/operator/src/test/java/oracle/kubernetes/operator/create/PodHelperConfigTest.java b/operator/src/test/java/oracle/kubernetes/operator/create/PodHelperConfigTest.java index 9682c1373c8..a2ddbc1a458 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/create/PodHelperConfigTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/create/PodHelperConfigTest.java @@ -404,8 +404,6 @@ private V1Pod getDesiredManagedServerPodConfigForDefaults(String image, String i pod.getMetadata() .putLabelsItem(CLUSTERNAME_LABEL, CLUSTER_NAME); pod.getSpec().getContainers().get(0).addCommandItem(ADMIN_SERVER_NAME).addCommandItem(String.valueOf(ADMIN_SERVER_PORT)); - pod.getSpec().getContainers().get(0).getPorts().get(0).setName("weblogic"); - pod.getSpec().getContainers().get(0).addPortsItem(new V1ContainerPort().protocol("TCP").containerPort(5556).name("node-manager")); return pod; } diff --git a/operator/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfigTest.java b/operator/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfigTest.java index d9e9a4d0983..810f3367dea 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfigTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfigTest.java @@ -11,6 +11,7 @@ import oracle.kubernetes.operator.logging.LoggingFactory; import org.junit.After; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import java.lang.reflect.Field; @@ -234,6 +235,7 @@ public void verifyValidateClusterStartupDoNotSuggestsUpdateToDynamicClusterIfRep assertEquals(0, suggestedConfigUpdates.size()); } + @Ignore // we are currently not suggesting updates based on number of machines @Test public void verifyValidateClusterStartupSuggestsUpdateToDynamicClusterIfNotEnoughMachines() throws Exception { WlsDynamicServersConfig wlsDynamicServersConfig = createDynamicServersConfig(2, 1, "ms-", "cluster1"); @@ -259,6 +261,7 @@ public void verifyValidateClusterStartupSuggestsUpdateToDynamicClusterIfNotEnoug assertEquals(2, configUpdate.targetClusterSize); } + @Ignore // we are currently not suggesting updates based on number of machines @Test public void verifyValidateClusterStartupDoNotLowerClusterSizeIfNotEnoughMachines() throws Exception { WlsDynamicServersConfig wlsDynamicServersConfig = createDynamicServersConfig(2, 1, "ms-", "cluster1"); @@ -285,6 +288,7 @@ public void verifyValidateClusterStartupDoNotLowerClusterSizeIfNotEnoughMachines assertEquals(2, configUpdate.targetClusterSize); } + @Ignore // we are currently not suggesting updates based on number of machines @Test public void verifyValidateClusterStartupDoNotSuggestUpdateToDynamicClusterIfEnoughMachines() throws Exception { WlsDynamicServersConfig wlsDynamicServersConfig = createDynamicServersConfig(1, 1, "ms-", "cluster1"); From 24b3d0e5816e77089b14b6889bc71cab5e6a847e Mon Sep 17 00:00:00 2001 From: anthony_lai Date: Fri, 6 Apr 2018 12:24:37 -0700 Subject: [PATCH 029/186] update test with changes in dynamic cluster branch --- .../operator/create/ConfigMapHelperConfigTest.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/operator/src/test/java/oracle/kubernetes/operator/create/ConfigMapHelperConfigTest.java b/operator/src/test/java/oracle/kubernetes/operator/create/ConfigMapHelperConfigTest.java index eb6eb58f577..6d4aa54c124 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/create/ConfigMapHelperConfigTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/create/ConfigMapHelperConfigTest.java @@ -24,6 +24,10 @@ public class ConfigMapHelperConfigTest { private static final String PROPERTY_LIVENESS_PROBE_SH = "livenessProbe.sh"; private static final String PROPERTY_READINESS_PROBE_SH = "readinessProbe.sh"; private static final String PROPERTY_READ_STATE_SH = "readState.sh"; + private static final String PROPERTY_START_SERVER_SH = "startServer.sh"; + private static final String PROPERTY_START_SERVER_PY = "start-server.py"; + private static final String PROPERTY_STOP_SERVER_SH = "stopServer.sh"; + private static final String PROPERTY_STOP_SERVER_PY = "stop-server.py"; @Test public void computedDomainConfigMap_isCorrect() throws Exception { @@ -36,6 +40,10 @@ public void computedDomainConfigMap_isCorrect() throws Exception { assertThat(getThenEmptyConfigMapDataValue(actual, PROPERTY_LIVENESS_PROBE_SH), not(isEmptyOrNullString())); assertThat(getThenEmptyConfigMapDataValue(actual, PROPERTY_READINESS_PROBE_SH), not(isEmptyOrNullString())); assertThat(getThenEmptyConfigMapDataValue(actual, PROPERTY_READ_STATE_SH), not(isEmptyOrNullString())); + assertThat(getThenEmptyConfigMapDataValue(actual, PROPERTY_START_SERVER_SH), not(isEmptyOrNullString())); + assertThat(getThenEmptyConfigMapDataValue(actual, PROPERTY_START_SERVER_PY), not(isEmptyOrNullString())); + assertThat(getThenEmptyConfigMapDataValue(actual, PROPERTY_STOP_SERVER_SH), not(isEmptyOrNullString())); + assertThat(getThenEmptyConfigMapDataValue(actual, PROPERTY_STOP_SERVER_PY), not(isEmptyOrNullString())); assertThat( actual, yamlEqualTo(getDesiredDomainConfigMap())); @@ -52,6 +60,11 @@ private V1ConfigMap getDesiredDomainConfigMap() { .putAnnotationsItem(FORMAT_ANNOTATION, FORMAT_VERSION)) .putDataItem(PROPERTY_LIVENESS_PROBE_SH, "") .putDataItem(PROPERTY_READINESS_PROBE_SH, "") + .putDataItem(PROPERTY_READ_STATE_SH, "") + .putDataItem(PROPERTY_START_SERVER_SH, "") + .putDataItem(PROPERTY_START_SERVER_PY, "") + .putDataItem(PROPERTY_STOP_SERVER_SH, "") + .putDataItem(PROPERTY_STOP_SERVER_PY, "") .putDataItem(PROPERTY_READ_STATE_SH, ""); } From a9d1f0361e90fecf3d222371a24820de54180f00 Mon Sep 17 00:00:00 2001 From: Rosemary Marano Date: Mon, 9 Apr 2018 16:30:12 -0400 Subject: [PATCH 030/186] review, minor edits --- site/name-changes.md | 191 +++++++++++++++++++++---------------------- 1 file changed, 95 insertions(+), 96 deletions(-) diff --git a/site/name-changes.md b/site/name-changes.md index bd67d960686..aa3b10bd718 100644 --- a/site/name-changes.md +++ b/site/name-changes.md @@ -2,15 +2,15 @@ The initial version of the WebLogic Server Kubernetes Operator did not use consistent conventions for many customer visible names. -We've recently addressed this by: -* standardizing the names of scripts and input files for creating operators and domains -* standardizing the input parameter names and values -* standardizing the names of the generated YAML files -* requiring that the customer create and specify a directory that the operators' and domains' generated YAML files will be stored in +We addressed this by: +* Standardizing the names of scripts and input files for creating operators and domains +* Standardizing the input parameter names and values +* Standardizing the names of the generated YAML files +* Requiring that the customer create and specify a directory that the operators' and domains' generated YAML files will be stored in This changes how operators and domains are created, and also changes the generated artifacts for operators and domains. -We're not providing an upgrade tool or backwards compatibility with the previous names. Instead, customers need to use the new script, inputs file and parameter names to recreate their operators and domains. +We're not providing an upgrade tool or backwards compatibility with the previous names. Instead, customers need to use the new script, inputs file, and parameter names to recreate their operators and domains. This document lists the customer visible naming changes. Also, the WebLogic Server Kubernetes Operator documentation has been updated. @@ -22,78 +22,78 @@ The following files are used to create the operator and to create and delete dom | Previous File Name | New File Name | | --- | --- | -| kubernetes/create-domain-job.sh | kubernetes/create-weblogic-domain.sh | -| kubernetes/create-domain-job-inputs.yaml | kubernetes/create-weblogic-domain-inputs.yaml | -| kubernetes/create-operator-inputs.yaml | kubernetes/create-weblogic-operator-inputs.yaml | -| kubernetes/create-weblogic-operator.sh | same | -| kubernetes/delete-domain.sh | kubernetes/delete-weblogic-operator-resources.sh | +| `kubernetes/create-domain-job.sh` | `kubernetes/create-weblogic-domain.sh` | +| `kubernetes/create-domain-job-inputs.yaml` | `kubernetes/create-weblogic-domain-inputs.yaml` | +| `kubernetes/create-operator-inputs.yaml` | `kubernetes/create-weblogic-operator-inputs.yaml` | +| `kubernetes/create-weblogic-operator.sh` | same | +| `kubernetes/delete-domain.sh` | `kubernetes/delete-weblogic-operator-resources.sh` | ### Generated YAML Files for Operators and Domains The create scripts generate a number of YAML files that are used to configure the corresponding Kubernetes artifacts for the operator and the domains. -Normally, customers do not use these YAML files. However, customers can look at them. They can also change the operator and domain configuration by editing these files and reapplying them. +Typically, customers do not use these YAML files. However, customers can look at them. They can also change the operator and domain configuration by editing these files and reapplying them. #### Directory for the Generated YAML Files -Previously, these files were placed in the kubernetes directory (e.g. kubernetes/weblogic-operator.yaml). Now, they are placed in per-operator and per-domain directories (since a Kubernetes cluster can have more than one operator and an operator can manage more than one domain). +Previously, these files were placed in the `kubernetes` directory (for example, `kubernetes/weblogic-operator.yaml`). Now, they are placed in per-operator and per-domain directories (since a Kubernetes cluster can have more than one operator and an operator can manage more than one domain). -The customer must create a directory that will parent the per-operator and per-domain directories, and use the -o option to pass the name of that directory to the create script, for example: - mkdir /scratch/my-user-projects - create-weblogic-operator.sh -o /scratch/my-user-projects -The pathname can either be a full path name, or a relative path name. If it's a relative pathname, then it's relative to the directory of the shell invoking the create script. +The customer must create a directory that will parent the per-operator and per-domain directories, and use the `-o` option to pass the name of that directory to the create script, for example: + `mkdir /scratch/my-user-projects + create-weblogic-operator.sh -o /scratch/my-user-projects`. +The pathname can be either a full path name or a relative path name. If it's a relative pathname, then it's relative to the directory of the shell invoking the create script. The per-operator directory name is: - /weblogic-operators/ + `/weblogic-operators/`. Similarly, the per-domain directory name is: - /weblogic-domains/ + `/weblogic-domains/`. #### What If I Mess Up Creating a Domain or Operator And Want To Do It Again? * Remove the resources that were created for the domain: - * kubernetes/delete-weblogic-domain-resources.sh yourDomainUID + * `kubernetes/delete-weblogic-domain-resources.sh yourDomainUID` * Remove the resources that were created for the operator: - * kubectl delete -f weblogic-operator.yaml + * `kubectl delete -f weblogic-operator.yaml` * kubectl delete -f weblogic-operator-security.yaml -* either remove the directory that was generated for that operator / domain, or remove the generated YAML files and the copy of the input file from it -* make whatever changes you need in your inputs file -* re-run the create script +* Either remove the directory that was generated for that operator or domain, or remove the generated YAML files and the copy of the input file from it. +* Make whatever changes you need in your inputs file. +* Re-run the create script. If you run the create script without cleaning up the previously generated directory, the create script will tell you about the offending files and then exit without creating anything. #### Location of the Input YAML Files -The create scripts support a -i option for specifying the location of the inputs file. Similar to the -o option, the path can either be a full path name or a relative path name. Relative path names are relative to the directory of the shell invoking the create script. +The create scripts support an `-i` option for specifying the location of the inputs file. Similar to the `-o` option, the path can be either a full path name or a relative path name. Relative path names are relative to the directory of the shell invoking the create script. -If -i is not specified, kubernetes/create-weblogic-operator.sh uses kubernetes/create-weblogic-operator-inputs.yaml. +If `-i` is not specified, `kubernetes/create-weblogic-operator.sh` uses `kubernetes/create-weblogic-operator-inputs.yaml`. -Previously, kubernetes/create-domain-job.sh used kubernetes/create-domain-job-inputs.yaml as the input file if -i was not specified. This behavior has been changed. The customer must select a Kubernetes cluster wide unique id for the domain and set the domainUid property in the inputs file to that value. This means that the customer must always modify the inputs file. +Previously, `kubernetes/create-domain-job.sh` used `kubernetes/create-domain-job-inputs.yaml` as the input file if `-i` was not specified. This behavior has been changed. The customer must select a Kubernetes cluster-wide unique ID for the domain and set the `domainUid` property in the inputs file to that value. This means that the customer must always modify the inputs file. -Also, we do not want the customer to have to change files in the operator's install directory. Because of this, the -i option MUST be specified when calling kubernetes/create-weblogic-operator.sh. The basic flow is: +Also, we do not want the customer to have to change files in the operator's install directory. Because of this, the `-i` option MUST be specified when calling `kubernetes/create-weblogic-operator.sh`. The basic flow is: -* pick a user projects directory, e.g. /scratch/my-user-projects -* mkdir /scratch/my-user-projects -* pick a unique id for the domain, e.g. foo.com -* cp kubernetes/create-weblogic-domain-inputs.yaml my-inputs.yaml -* set the domainUid in my-inputs.yaml to foo.com -* kubernetes/create-weblogic-operator.sh -i my-inputs.yaml -o /scratch/my-user-projects +* Pick a user projects directory, for example, `/scratch/my-user-projects` +* `mkdir /scratch/my-user-projects` +* Pick a unique ID for the domain, for example, `foo.com` +* `cp kubernetes/create-weblogic-domain-inputs.yaml my-inputs.yaml` +* Set the domainUid in `my-inputs.yaml` to `foo.com` +* `kubernetes/create-weblogic-operator.sh -i my-inputs.yaml -o /scratch/my-user-projects` -Note: my-inputs.yaml will be copied to /scratch/my-user-projects/weblogic-domains/foo.com/create-weblogic-domain-inputs.yaml +**Note:** `my-inputs.yaml` will be copied to `/scratch/my-user-projects/weblogic-domains/foo.com/create-weblogic-domain-inputs.yaml` -#### File Names of the Generated YAML File +#### File Names of the Generated YAML Files The names of several of the generated YAML files have changed. | Previous File Name | New File Name | | --- | --- | -| domain-custom-resource.yaml | same | -| domain-job.yaml | create-weblogic-domain-job.yaml | -| persistent-volume.yaml | weblogic-domain-pv.yaml | -| persistent-volume-claim.yaml | weblogic-domain-pvc.yaml | -| rbac.yaml | weblogic-operator-security.yaml | -| traefik-deployment.yaml | weblogic-domain-traefik-${clusterName, lower case}.yaml | -| traefik-rbac.yaml | weblogic-domain-traefik-security-${clusterName, lower case}.yaml | -| weblogic-operator.yaml | same | +| `domain-custom-resource.yaml` | same | +| `domain-job.yaml` | `create-weblogic-domain-job.yaml` | +| `persistent-volume.yaml` | `weblogic-domain-pv.yaml` | +| `persistent-volume-claim.yaml` | `weblogic-domain-pvc.yaml` | +| `rbac.yaml` | `weblogic-operator-security.yaml` | +| `traefik-deployment.yaml` | `weblogic-domain-traefik-${clusterName, lower case}.yaml` | +| `traefik-rbac.yaml` | `weblogic-domain-traefik-security-${clusterName, lower case}.yaml` | +| `weblogic-operator.yaml` | same | ## Input File Contents Some of the contents of the inputs files have changed: @@ -108,17 +108,17 @@ Some of the contents of the inputs files have changed: | Previous Property Name | New Property Name | | --- | --- | -| image | weblogicOperatorImage | -| imagePullPolicy | weblogicOperatorImagePullPolicy | -| imagePullSecretName | weblogicOperatorImagePullSecretName | +| `image` | `weblogicOperatorImage` | +| `imagePullPolicy` | `weblogicOperatorImagePullPolicy` | +| `imagePullSecretName` | `weblogicOperatorImagePullSecretName` | #### Property Values | Previous Property Name | Property Name | Old Property Value | New Property Value | | --- | --- | --- | --- | -| externalRestOption | externalRestOption | none | NONE | -| externalRestOption | externalRestOption | custom-cert | CUSTOM_CERT | -| externalRestOption | externalRestOption | self-signed-cert | SELF_SIGNED_CERT | +| `externalRestOption` | `externalRestOption` | `none` | `NONE` | +| `externalRestOption` | `externalRestOption` | `custom-cert` | `CUSTOM_CERT` | +| `externalRestOption` | `externalRestOption` | `self-signed-cert` | `SELF_SIGNED_CERT` | ### create-weblogic-domain-inputs.yaml @@ -126,63 +126,62 @@ Some of the contents of the inputs files have changed: | Previous Property Name | New Property Name | | --- | --- | -| createDomainScript | This property has been removed | -| domainUid | domainUID | -| imagePullSecretName | weblogicImagePullSecretName | -| loadBalancerAdminPort | loadBalancerDashboardPort | -| managedServerCount | configuredManagedServerCount | -| managedServerStartCount | initialManagedServerReplicas | -| nfsServer | weblogicDomainStorageNFSServer | -| persistencePath | weblogicDomainStoragePath | -| persistenceSize | weblogicDomainStorageSize | -| persistenceType | weblogicDomainStorageType | -| persistenceStorageClass | This property has been removed | -| persistenceVolumeClaimName | This property has been removed | -| persistenceVolumeName | This property has been removed | -| secretName | weblogicCredentialsSecretName | - -#### Properties that must be customized +| `createDomainScript` | This property has been removed | +| `domainUid` | `domainUID` | +| `imagePullSecretName` | `weblogicImagePullSecretName` | +| `loadBalancerAdminPort` | `loadBalancerDashboardPort` | +| `managedServerCount` | `configuredManagedServerCount` | +| `managedServerStartCount` | `initialManagedServerReplicas` | +| `nfsServer` | `weblogicDomainStorageNFSServer` | +| `persistencePath` | `weblogicDomainStoragePath` | +| `persistenceSize` | `weblogicDomainStorageSize` | +| `persistenceType` | `weblogicDomainStorageType` | +| `persistenceStorageClass` | This property has been removed | +| `persistenceVolumeClaimName` | This property has been removed | +| `persistenceVolumeName` | This property has been removed | +| `secretName` | `weblogicCredentialsSecretName` | + +#### Properties That Must be Customized The following input properties, which used to have default values, now must be uncommented and customized. | Previous Property Name | New Property Name | Previous Default Value | Notes | | --- | --- | --- | --- | -| domainUid | domainUID | domain1 | Since the domain UID is supposed to be unique across the Kubernetes cluster, the customer must choose one. | -| persistencePath | weblogicDomainStoragePath | /scratch/k8s_dir/persistentVolume001 | The customer must select a directory for the domain's storage. | -| nfsServer | weblogicDomainStorageNFSServer | nfsServer | If weblogicDomainStorageType is NFS, then the customer must specify the name or IP of the NFS server. | +| `domainUid` | `domainUID` | `domain1` | Because the domain UID is supposed to be unique across the Kubernetes cluster, the customer must choose one. | +| `persistencePath` | `weblogicDomainStoragePath` | `/scratch/k8s_dir/persistentVolume001` | The customer must select a directory for the domain's storage. | +| `nfsServer` | `weblogicDomainStorageNFSServer` | `nfsServer` | If `weblogicDomainStorageType` is NFS, then the customer must specify the name or IP of the NFS server. | #### Property Values | Previous Property Name | New Property Name | Old Property Value | New Property Value | | --- | --- | --- | --- | -| loadBalancer | loadBalancer | none | NONE | -| loadBalancer | loadBalancer | traefik | TRAEFIK | -| persistenceType | weblogicDomainStorageType | hostPath | HOST_PATH | -| persistenceType | weblogicDomainStorageType | nfs | NFS | +| `loadBalancer` | `loadBalancer` | `none` | `NONE` | +| `loadBalancer` | `loadBalancer` | `traefik` | `TRAEFIK` | +| `persistenceType` | `weblogicDomainStorageType` | `hostPath` | `HOST_PATH` | +| `persistenceType` | `weblogicDomainStorageType` | `nfs` | `NFS` | ## Kubernetes Artifact Names | Artifact Type | Previous Name | New Name | | --- | --- | --- | -| persistent volume | ${domainUid}-${persistenceVolume} or ${persistenceVolume} | ${domainUID}-weblogic-domain-pv | -| persistent volume claim | ${domainUid}-${persistenceVolumeClaim} or ${persistenceVolumeClaim} | ${domainUID}-weblogic-domain-pvc | -| storage class name | ${domainUid} or ${persistenceStorageClass} | ${domainUID-weblogic-domain-storage-class | -| job | domain-${domainUid}-job | ${domainUID}-create-weblogic-domain-job | -| container | domain-job | create-weblogic-domain-job | -| container | ${d.domainUID}-${d.clusterName, lower case}-traefik | traefik | -| config map | operator-config-map | weblogic-operator-cm | -| config map | ${domainUid}-${clusterName, lower case}-traefik | ${domainUID}-${clusterName, lower case}-traefik-cm | -| config map | domain-${domainUid}-scripts | ${domainUID}-create-weblogic-domain-job-cm | -| config map | weblogic-domain-config-map | weblogic-domain-cm | -| secret | operator-secrets | weblogic-operator-secrets | -| port | rest-https | rest | -| service | external-weblogic-operator-service | external-weblogic-operator-srv | -| service | internal-weblogic-operator-service | internal-weblogic-operator-srv | -| volume & mount | operator-config-volume | weblogic-operator-cm-volume | -| volume & mount | operator-secrets-volume | weblogic-operator-secrets-volume | -| volume & mount | config-map-scripts | create-weblogic-domain-job-cm-volume | -| volume & mount | pv-storage | weblogic-domain-storage-volume | -| volume & mount | secrets | weblogic-credentials-volume | -| volume & mount | scripts | weblogic-domain-cm-volume | - -Note: The input properties for controlling the domain's persistent volume, persistent volume claim and storage class names have been removed. These names are now derived from the domain uid. - +| persistent volume | `${domainUid}-${persistenceVolume}` or `${persistenceVolume}` | `${domainUID}-weblogic-domain-pv` | +| persistent volume claim | `${domainUid}-${persistenceVolumeClaim}` or `${persistenceVolumeClaim}` | `${domainUID}-weblogic-domain-pvc` | +| storage class name | `${domainUid}` or `${persistenceStorageClass}` | `${domainUID-weblogic-domain-storage-class` | +| job | `domain-${domainUid}-job` | `${domainUID}-create-weblogic-domain-job` | +| container | `domain-job` | `create-weblogic-domain-job` | +| container | `${d.domainUID}-${d.clusterName, lower case}-traefik` | `traefik` | +| config map | `operator-config-map` | `weblogic-operator-cm` | +| config map | `${domainUid}-${clusterName, lower case}-traefik` | `${domainUID}-${clusterName, lower case}-traefik-cm` | +| config map | `domain-${domainUid}-scripts` | `${domainUID}-create-weblogic-domain-job-cm` | +| config map | `weblogic-domain-config-map` | `weblogic-domain-cm` | +| secret | `operator-secrets` | `weblogic-operator-secrets` | +| port | `rest-https` | `rest` | +| service | `external-weblogic-operator-service` | `external-weblogic-operator-srv` | +| service | `internal-weblogic-operator-service` | `internal-weblogic-operator-srv` | +| volume & mount | `operator-config-volume` | `weblogic-operator-cm-volume` | +| volume & mount | `operator-secrets-volume` | `weblogic-operator-secrets-volume` | +| volume & mount | `config-map-scripts` | `create-weblogic-domain-job-cm-volume` | +| volume & mount | `pv-storage` | `weblogic-domain-storage-volume` | +| volume & mount | `secrets` | `weblogic-credentials-volume` | +| volume & mount | `scripts` | `weblogic-domain-cm-volume` | + +**Note:** The input properties for controlling the domain's persistent volume, persistent volume claim, and storage class names have been removed. These names are now derived from the domain UID. From 20d2a387a4d8e95c97193558ab8633c54ab9f580 Mon Sep 17 00:00:00 2001 From: Rosemary Marano Date: Tue, 10 Apr 2018 11:43:26 -0400 Subject: [PATCH 031/186] minor edits --- site/recent-changes.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/site/recent-changes.md b/site/recent-changes.md index 1957b1a132f..942ae6e0574 100644 --- a/site/recent-changes.md +++ b/site/recent-changes.md @@ -1,8 +1,8 @@ # Recent Changes to the Oracle WebLogic Server Kubernetes Operator -This page track recent changes to the operator, especially ones that introduce backwards incompatibilities. +This page track recent changes to the operator, especially ones that introduce backward incompatibilities. -| Date | Introduces Backwards Incompatibilities | Change | +| Date | Introduces Backward Incompatibilities | Change | | --- | --- | --- | -| March 20, 2018 | yes | See [Name Changes](name-changes.md). Several of files and input parameters have been renamed. This affects how operators an domains are created. It also changes generated Kubernetes artifacts, therefore customers must recreate their operators and domains. -| April 4, 2018 | yes | See [Name Changes](name-changes.md). Many kubernetes artifact names and labels have changed. Also, the names of generated yaml files for creating a domain's pv and pvc have changed. Because of these changes, customers must recreate their operators and domains. +| March 20, 2018 | yes | See [Name Changes](name-changes.md). Several files and input parameters have been renamed. This affects how operators and domains are created. It also changes generated Kubernetes artifacts, therefore customers must recreate their operators and domains. +| April 4, 2018 | yes | See [Name Changes](name-changes.md). Many Kubernetes artifact names and labels have changed. Also, the names of generated YAML files for creating a domain's PV and PVC have changed. Because of these changes, customers must recreate their operators and domains. From f2c44ecd45a9f6678c9024b5a27526e707400618 Mon Sep 17 00:00:00 2001 From: Anthony Lai Date: Wed, 11 Apr 2018 10:08:37 -0700 Subject: [PATCH 032/186] fix issue in create domain job using domain name other than base_domain --- kubernetes/internal/create-weblogic-domain-job-template.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/internal/create-weblogic-domain-job-template.yaml b/kubernetes/internal/create-weblogic-domain-job-template.yaml index e31ba1ad756..24ecb3d755a 100644 --- a/kubernetes/internal/create-weblogic-domain-job-template.yaml +++ b/kubernetes/internal/create-weblogic-domain-job-template.yaml @@ -181,7 +181,7 @@ data: set('DomainsDirRemoteSharingEnabled', 'true') # Set the Node Manager user name and password (domain name will change after writeDomain) - cd('/SecurityConfiguration/%DOMAIN_NAME%') + cd('/SecurityConfiguration/base_domain') set('NodeManagerUsername', admin_username) set('NodeManagerPasswordEncrypted', admin_password) From f978388aebfb67bdb1afc82f2460ec5cd4a13f27 Mon Sep 17 00:00:00 2001 From: Lily He Date: Thu, 12 Apr 2018 05:50:54 -0700 Subject: [PATCH 033/186] add voyager loader balance --- kubernetes/internal/create-weblogic-domain.sh | 81 ++++++++++++++----- .../internal/voyager-ingress-template.yaml | 36 +++++++++ src/integration-tests/bash/run.sh | 14 ++-- 3 files changed, 104 insertions(+), 27 deletions(-) create mode 100644 kubernetes/internal/voyager-ingress-template.yaml diff --git a/kubernetes/internal/create-weblogic-domain.sh b/kubernetes/internal/create-weblogic-domain.sh index 06f9c295d94..26d061beef2 100755 --- a/kubernetes/internal/create-weblogic-domain.sh +++ b/kubernetes/internal/create-weblogic-domain.sh @@ -183,10 +183,12 @@ function validateLoadBalancer { case ${loadBalancer} in "TRAEFIK") ;; + "VOYAGER") + ;; "NONE") ;; *) - validationError "Invalid value for loadBalancer: ${loadBalancer}. Valid values are TRAEFIK and NONE." + validationError "Invalid value for loadBalancer: ${loadBalancer}. Valid values are TRAEFIK, VOYAGER and NONE." ;; esac fi @@ -321,6 +323,11 @@ function initialize { validationError "The template file ${traefikInput} for generating the traefik deployment was not found" fi + voyagerInput="${scriptDir}/voyager-ingress-template.yaml" + if [ ! -f ${voyagerInput} ]; then + validationError "The template file ${voyagerInput} for generating the voyager ingress was not found" + fi + failIfValidationErrors # Parse the commonn inputs file @@ -388,6 +395,7 @@ function createYamlFiles { dcrOutput="${domainOutputDir}/domain-custom-resource.yaml" traefikSecurityOutput="${domainOutputDir}/weblogic-domain-traefik-security-${clusterNameLC}.yaml" traefikOutput="${domainOutputDir}/weblogic-domain-traefik-${clusterNameLC}.yaml" + voyagerOutput="${domainOutputDir}/voyager-ingress.yaml" enabledPrefix="" # uncomment the feature disabledPrefix="# " # comment out the feature @@ -473,25 +481,38 @@ function createYamlFiles { sed -i -e "s:%JAVA_OPTIONS%:${javaOptions}:g" ${dcrOutput} sed -i -e "s:%STARTUP_CONTROL%:${startupControl}:g" ${dcrOutput} - # Traefik file - cp ${traefikInput} ${traefikOutput} - echo Generating ${traefikOutput} - sed -i -e "s:%NAMESPACE%:$namespace:g" ${traefikOutput} - sed -i -e "s:%DOMAIN_UID%:${domainUID}:g" ${traefikOutput} - sed -i -e "s:%DOMAIN_NAME%:${domainName}:g" ${traefikOutput} - sed -i -e "s:%CLUSTER_NAME%:${clusterName}:g" ${traefikOutput} - sed -i -e "s:%CLUSTER_NAME_LC%:${clusterNameLC}:g" ${traefikOutput} - sed -i -e "s:%LOAD_BALANCER_WEB_PORT%:$loadBalancerWebPort:g" ${traefikOutput} - sed -i -e "s:%LOAD_BALANCER_DASHBOARD_PORT%:$loadBalancerDashboardPort:g" ${traefikOutput} - - # Traefik security file - cp ${traefikSecurityInput} ${traefikSecurityOutput} - echo Generating ${traefikSecurityOutput} - sed -i -e "s:%NAMESPACE%:$namespace:g" ${traefikSecurityOutput} - sed -i -e "s:%DOMAIN_UID%:${domainUID}:g" ${traefikSecurityOutput} - sed -i -e "s:%DOMAIN_NAME%:${domainName}:g" ${traefikSecurityOutput} - sed -i -e "s:%CLUSTER_NAME%:${clusterName}:g" ${traefikSecurityOutput} - sed -i -e "s:%CLUSTER_NAME_LC%:${clusterNameLC}:g" ${traefikSecurityOutput} + if [ "${loadBalancer}" = "TRAEFIK" ]; then + # Traefik file + cp ${traefikInput} ${traefikOutput} + echo Generating ${traefikOutput} + sed -i -e "s:%NAMESPACE%:$namespace:g" ${traefikOutput} + sed -i -e "s:%DOMAIN_UID%:${domainUID}:g" ${traefikOutput} + sed -i -e "s:%DOMAIN_NAME%:${domainName}:g" ${traefikOutput} + sed -i -e "s:%CLUSTER_NAME%:${clusterName}:g" ${traefikOutput} + sed -i -e "s:%CLUSTER_NAME_LC%:${clusterNameLC}:g" ${traefikOutput} + sed -i -e "s:%LOAD_BALANCER_WEB_PORT%:$loadBalancerWebPort:g" ${traefikOutput} + sed -i -e "s:%LOAD_BALANCER_DASHBOARD_PORT%:$loadBalancerDashboardPort:g" ${traefikOutput} + + # Traefik security file + cp ${traefikSecurityInput} ${traefikSecurityOutput} + echo Generating ${traefikSecurityOutput} + sed -i -e "s:%NAMESPACE%:$namespace:g" ${traefikSecurityOutput} + sed -i -e "s:%DOMAIN_UID%:${domainUID}:g" ${traefikSecurityOutput} + sed -i -e "s:%DOMAIN_NAME%:${domainName}:g" ${traefikSecurityOutput} + sed -i -e "s:%CLUSTER_NAME%:${clusterName}:g" ${traefikSecurityOutput} + sed -i -e "s:%CLUSTER_NAME_LC%:${clusterNameLC}:g" ${traefikSecurityOutput} + fi + + if [ "${loadBalancer}" = "VOYAGER" ]; then + # Voyager ingress file + cp ${voyagerInput} ${voyagerOutput} + echo Generating ${voyagerOutput} + sed -i -e "s:%DOMAIN_UID%:${domainUID}:g" ${voyagerOutput} + sed -i -e "s:%CLUSTER_NAME%:${clusterName}:g" ${voyagerOutput} + sed -i -e "s:%MANAGED_SERVER_PORT%:${managedServerPort}:g" ${voyagerOutput} + sed -i -e "s:%LOAD_BALANCER_WEB_PORT%:$loadBalancerWebPort:g" ${voyagerOutput} + sed -i -e "s:%LOAD_BALANCER_DASHBOARD_PORT%:$loadBalancerDashboardPort:g" ${voyagerOutput} + fi # Remove any "...yaml-e" files left over from running sed rm -f ${domainOutputDir}/*.yaml-e @@ -584,6 +605,20 @@ function createDomain { } +# +# Deploy voyager/HAProxy load balancer +# +function setupVoyagerLoadBalancer { + # deploy voyager ingress controller + kubectl create namespace voyager + curl -fsSL https://raw.githubusercontent.com/appscode/voyager/6.0.0/hack/deploy/voyager.sh \ + | bash -s -- --provider=baremetal --namespace=voyager + + # deploy voyager ingress resource + kubectl create -f ${voyagerOutput} + + # TODO verify the result +} # # Deploy traefik load balancer # @@ -682,7 +717,7 @@ function outputJobSummary { if [ "${exposeAdminT3Channel}" = true ]; then echo "T3 access is available at t3:${K8S_IP}:${t3ChannelPort}" fi - if [ "${loadBalancer}" = "TRAEFIK" ]; then + if [ "${loadBalancer}" = "TRAEFIK" ]|| [ "${loadBalancer}" = "VOYAGER" ]; then echo "The load balancer for cluster '${clusterName}' is available at http:${K8S_IP}:${loadBalancerWebPort}/ (add the application path to the URL)" echo "The load balancer dashboard for cluster '${clusterName}' is available at http:${K8S_IP}:${loadBalancerDashboardPort}" echo "" @@ -696,6 +731,8 @@ function outputJobSummary { if [ "${loadBalancer}" = "TRAEFIK" ]; then echo " ${traefikSecurityOutput}" echo " ${traefikOutput}" + elif [ "${loadBalancer}" = "VOYAGER" ]; then + echo " ${voyagerOutput}" fi } @@ -726,6 +763,8 @@ if [ "${generateOnly}" = false ]; then # Setup load balancer if [ "${loadBalancer}" = "TRAEFIK" ]; then setupTraefikLoadBalancer + elif [ "${loadBalancer}" = "VOYAGER" ]; then + setupVoyagerLoadBalancer fi # Create the domain custom resource diff --git a/kubernetes/internal/voyager-ingress-template.yaml b/kubernetes/internal/voyager-ingress-template.yaml new file mode 100644 index 00000000000..88fb5c63558 --- /dev/null +++ b/kubernetes/internal/voyager-ingress-template.yaml @@ -0,0 +1,36 @@ +apiVersion: voyager.appscode.com/v1beta1 +kind: Ingress +metadata: + name: voyager-ingress + namespace: default + annotations: + ingress.appscode.com/type: 'NodePort' + ingress.appscode.com/stats: 'true' +spec: + rules: + - host: %DOMAIN_UID%.%CLUSTER_NAME% + http: + nodePort: '%LOAD_BALANCER_WEB_PORT%' + paths: + - backend: + serviceName: %DOMAIN_UID%-cluster-%CLUSTER_NAME% + servicePort: '%MANAGED_SERVER_PORT%' + +--- +apiVersion: v1 +kind: Service +metadata: + name: voyager-stats + labels: + app: voyager-stats +spec: + type: NodePort + ports: + - name: client + protocol: TCP + port: 56789 + targetPort: 56789 + nodePort: %LOAD_BALANCER_DASHBOARD_PORT% + selector: + origin: voyager + origin-name: voyager-ingress diff --git a/src/integration-tests/bash/run.sh b/src/integration-tests/bash/run.sh index 08dcd88832e..17e20bf6bb6 100755 --- a/src/integration-tests/bash/run.sh +++ b/src/integration-tests/bash/run.sh @@ -1074,6 +1074,7 @@ function verify_webapp_load_balancing { local NAMESPACE="`dom_get $1 NAMESPACE`" local DOMAIN_UID="`dom_get $1 DOMAIN_UID`" + local WL_CLUSTER_NAME="`dom_get $1 WL_CLUSTER_NAME`" local MS_BASE_NAME="`dom_get $1 MS_BASE_NAME`" local LOAD_BALANCER_WEB_PORT="`dom_get $1 LOAD_BALANCER_WEB_PORT`" local TMP_DIR="`dom_get $1 TMP_DIR`" @@ -1104,13 +1105,14 @@ function verify_webapp_load_balancing { local max_count=30 local wait_time=6 local count=0 + local vheader="host: $DOMAIN_UID.$WL_CLUSTER_NAME" # this is only useful to voyager but it does no harm to traefik etc while [ "${HTTP_RESPONSE}" != "200" -a $count -lt $max_count ] ; do local count=`expr $count + 1` echo "NO_DATA" > $CURL_RESPONSE_BODY - local HTTP_RESPONSE=$(curl --silent --show-error --noproxy ${NODEPORT_HOST} ${TEST_APP_URL} \ - --write-out "%{http_code}" \ - -o ${CURL_RESPONSE_BODY} \ + local HTTP_RESPONSE=$(eval "curl --silent --show-error -H '${vheader}' --noproxy ${NODEPORT_HOST} ${TEST_APP_URL} \ + --write-out '%{http_code}' \ + -o ${CURL_RESPONSE_BODY}" \ ) if [ "${HTTP_RESPONSE}" != "200" ]; then @@ -1141,9 +1143,9 @@ function verify_webapp_load_balancing { do echo "NO_DATA" > $CURL_RESPONSE_BODY - local HTTP_RESPONSE=$(curl --silent --show-error --noproxy ${NODEPORT_HOST} ${TEST_APP_URL} \ - --write-out "%{http_code}" \ - -o ${CURL_RESPONSE_BODY} \ + local HTTP_RESPONSE=$(eval "curl --silent --show-error -H '${vheader}' --noproxy ${NODEPORT_HOST} ${TEST_APP_URL} \ + --write-out '%{http_code}' \ + -o ${CURL_RESPONSE_BODY}" \ ) echo $HTTP_RESPONSE | sed 's/^/+/' From 3636756002295a929cae782f0de5fa372eec061a Mon Sep 17 00:00:00 2001 From: Lily He Date: Thu, 12 Apr 2018 06:23:05 -0700 Subject: [PATCH 034/186] add verification after deploy voyager res --- kubernetes/internal/create-weblogic-domain.sh | 14 ++++++++++++-- src/integration-tests/bash/run.sh | 2 +- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/kubernetes/internal/create-weblogic-domain.sh b/kubernetes/internal/create-weblogic-domain.sh index 26d061beef2..b85124dd651 100755 --- a/kubernetes/internal/create-weblogic-domain.sh +++ b/kubernetes/internal/create-weblogic-domain.sh @@ -617,7 +617,17 @@ function setupVoyagerLoadBalancer { # deploy voyager ingress resource kubectl create -f ${voyagerOutput} - # TODO verify the result + echo Checking voyager deploy + vdep=`kubectl get deploy | grep voyager | wc | awk ' { print $1; } '` + if [ "$vdep" != "1" ]; then + fail "The deployment of voyager ingress was not created" + fi + +echo Checking voyager service + vscv=`kubectl get service voyager-stats | grep voyager-stats | wc | awk ' { print $1; } '` + if [ "$vscv" != "1" ]; then + fail "The service ${voyager-stats} was not created" + fi } # # Deploy traefik load balancer @@ -717,7 +727,7 @@ function outputJobSummary { if [ "${exposeAdminT3Channel}" = true ]; then echo "T3 access is available at t3:${K8S_IP}:${t3ChannelPort}" fi - if [ "${loadBalancer}" = "TRAEFIK" ]|| [ "${loadBalancer}" = "VOYAGER" ]; then + if [ "${loadBalancer}" = "TRAEFIK" ] || [ "${loadBalancer}" = "VOYAGER" ]; then echo "The load balancer for cluster '${clusterName}' is available at http:${K8S_IP}:${loadBalancerWebPort}/ (add the application path to the URL)" echo "The load balancer dashboard for cluster '${clusterName}' is available at http:${K8S_IP}:${loadBalancerDashboardPort}" echo "" diff --git a/src/integration-tests/bash/run.sh b/src/integration-tests/bash/run.sh index 17e20bf6bb6..26542bc04f9 100755 --- a/src/integration-tests/bash/run.sh +++ b/src/integration-tests/bash/run.sh @@ -1105,7 +1105,7 @@ function verify_webapp_load_balancing { local max_count=30 local wait_time=6 local count=0 - local vheader="host: $DOMAIN_UID.$WL_CLUSTER_NAME" # this is only useful to voyager but it does no harm to traefik etc + local vheader="host: $DOMAIN_UID.$WL_CLUSTER_NAME" # this is only needed for voyager but it does no harm to traefik etc while [ "${HTTP_RESPONSE}" != "200" -a $count -lt $max_count ] ; do local count=`expr $count + 1` From 2cb71f2a8d166fa7aadf79fedcbec851b1a1fced Mon Sep 17 00:00:00 2001 From: Lily He Date: Thu, 12 Apr 2018 06:52:07 -0700 Subject: [PATCH 035/186] fix typo --- kubernetes/internal/create-weblogic-domain.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/internal/create-weblogic-domain.sh b/kubernetes/internal/create-weblogic-domain.sh index b85124dd651..f6c04c49143 100755 --- a/kubernetes/internal/create-weblogic-domain.sh +++ b/kubernetes/internal/create-weblogic-domain.sh @@ -626,7 +626,7 @@ function setupVoyagerLoadBalancer { echo Checking voyager service vscv=`kubectl get service voyager-stats | grep voyager-stats | wc | awk ' { print $1; } '` if [ "$vscv" != "1" ]; then - fail "The service ${voyager-stats} was not created" + fail "The service voyager-stats was not created" fi } # From 40f8a7e32382e138cde00fc9fbdc849c539d6f94 Mon Sep 17 00:00:00 2001 From: Simon Meng Date: Sun, 15 Apr 2018 22:10:12 -0700 Subject: [PATCH 036/186] Add NFS test case to integration test suite --- src/integration-tests/bash/run.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/integration-tests/bash/run.sh b/src/integration-tests/bash/run.sh index 08dcd88832e..039e2d77524 100755 --- a/src/integration-tests/bash/run.sh +++ b/src/integration-tests/bash/run.sh @@ -809,6 +809,10 @@ function run_create_domain_job { sed -i -e "s/^exposeAdminT3Channel:.*/exposeAdminT3Channel: true/" $inputs # Customize more configuration + if [ "$DOMAIN_UID" == "domain5" ] && [ "$JENKINS" = "true" ] ; then + sed -i -e "s/^weblogicDomainStorageType:.*/weblogicDomainStorageType: NFS/" $inputs + sed -i -e "s/^#weblogicDomainStorageNFSServer:.*/weblogicDomainStorageNFSServer: $NODEPORT_HOST/" $inputs + fi sed -i -e "s;^#weblogicDomainStoragePath:.*;weblogicDomainStoragePath: $PV_ROOT/acceptance_test_pv/$DOMAIN_STORAGE_DIR;" $inputs sed -i -e "s/^#domainUID:.*/domainUID: $DOMAIN_UID/" $inputs sed -i -e "s/^clusterName:.*/clusterName: $WL_CLUSTER_NAME/" $inputs @@ -2697,6 +2701,7 @@ function test_suite { test_domain_lifecycle domain1 domain4 # create domain5 in the default namespace with startupControl="ADMIN", and verify that only admin server is created + # on Jenkins, this domain will also test NFS instead of HOSTPATH PV storage (search for [ "$DOMAIN_UID" == "domain5" ]) test_create_domain_startup_control_admin domain5 # create domain6 in the default namespace with pvReclaimPolicy="Recycle", and verify that the PV is deleted once the domain and PVC are deleted From 11984e24c7651b953a79e99be1f340dda82d8f66 Mon Sep 17 00:00:00 2001 From: Ryan Eberhard Date: Mon, 16 Apr 2018 08:00:40 -0400 Subject: [PATCH 037/186] Work in progress --- Dockerfile | 1 + .../create-weblogic-domain-job-template.yaml | 7 + .../internal/generate-security-policy.sh | 12 +- .../operator/KubernetesConstants.java | 1 + .../operator/ProcessingConstants.java | 1 + .../operator/helpers/CallBuilder.java | 346 +++++++++- .../operator/helpers/ConfigMapHelper.java | 633 +++++++++++++++++- .../operator/helpers/DomainHomeHelper.java | 200 ++++++ .../src/main/resources/Operator.properties | 6 +- .../CreateOperatorGeneratedFilesBaseTest.java | 14 +- pom.xml | 17 + wercker.yml | 1 + 12 files changed, 1196 insertions(+), 43 deletions(-) create mode 100644 operator/src/main/java/oracle/kubernetes/operator/helpers/DomainHomeHelper.java diff --git a/Dockerfile b/Dockerfile index 36b6c070e02..61555a75d81 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,6 +9,7 @@ ENV PATH=$PATH:/operator COPY src/scripts/* /operator/ COPY operator/target/weblogic-kubernetes-operator-0.2.jar /operator/weblogic-kubernetes-operator.jar +COPY target/weblogic-deploy.zip /operator/weblogic-deploy.zip COPY operator/target/lib/*.jar /operator/lib/ HEALTHCHECK --interval=1m --timeout=10s \ diff --git a/kubernetes/internal/create-weblogic-domain-job-template.yaml b/kubernetes/internal/create-weblogic-domain-job-template.yaml index 36d7d7e26b0..b3862981421 100644 --- a/kubernetes/internal/create-weblogic-domain-job-template.yaml +++ b/kubernetes/internal/create-weblogic-domain-job-template.yaml @@ -169,6 +169,13 @@ data: cat << EOF > ${scriptFile} #!/bin/bash + # Base64 decode binary support files + mkdir -p /tmp/weblogic-operator/bin + shopt -s nullglob + for f in /weblogic-operator/scripts/*.base64; do + cat "\$f" | base64 --decode > /tmp/weblogic-operator/bin/\$(basename "\$f" .base64) + done + # Check for stale state file and remove if found" if [ -f ${stateFile} ]; then echo "Removing stale file ${stateFile}" diff --git a/kubernetes/internal/generate-security-policy.sh b/kubernetes/internal/generate-security-policy.sh index a92185d0dcb..4095f38e24c 100755 --- a/kubernetes/internal/generate-security-policy.sh +++ b/kubernetes/internal/generate-security-policy.sh @@ -94,8 +94,11 @@ metadata: weblogic.operatorName: ${NAMESPACE} rules: - apiGroups: [""] - resources: ["namespaces", "persistentvolumes"] + resources: ["namespaces"] verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete", "deletecollection"] - apiGroups: ["apiextensions.k8s.io"] resources: ["customresourcedefinitions"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete", "deletecollection"] @@ -197,13 +200,13 @@ metadata: weblogic.operatorName: ${NAMESPACE} rules: - apiGroups: [""] - resources: ["secrets", "persistentvolumeclaims"] + resources: ["secrets"] verbs: ["get", "list", "watch"] - apiGroups: ["storage.k8s.io"] resources: ["storageclasses"] verbs: ["get", "list", "watch"] - apiGroups: [""] - resources: ["services", "configmaps", "pods", "jobs", "events"] + resources: ["services", "configmaps", "pods", "podtemplates", "events", "persistentvolumeclaims"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete", "deletecollection"] - apiGroups: [""] resources: ["pods/logs"] @@ -211,6 +214,9 @@ rules: - apiGroups: [""] resources: ["pods/exec"] verbs: ["create"] +- apiGroups: ["batch"] + resources: ["jobs", "cronjobs"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete", "deletecollection"] - apiGroups: ["settings.k8s.io"] resources: ["podpresets"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete", "deletecollection"] diff --git a/operator/src/main/java/oracle/kubernetes/operator/KubernetesConstants.java b/operator/src/main/java/oracle/kubernetes/operator/KubernetesConstants.java index 0e12b245032..89b384655d5 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/KubernetesConstants.java +++ b/operator/src/main/java/oracle/kubernetes/operator/KubernetesConstants.java @@ -26,5 +26,6 @@ public interface KubernetesConstants { public static final String CONTAINER_NAME = "weblogic-server"; public static final String DOMAIN_CONFIG_MAP_NAME = "weblogic-domain-cm"; + public static final String DOMAIN_HOME_CONFIG_MAP_NAME = "weblogic-domain-home-cm"; } diff --git a/operator/src/main/java/oracle/kubernetes/operator/ProcessingConstants.java b/operator/src/main/java/oracle/kubernetes/operator/ProcessingConstants.java index fec5c18378a..e6649b54079 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/ProcessingConstants.java +++ b/operator/src/main/java/oracle/kubernetes/operator/ProcessingConstants.java @@ -31,6 +31,7 @@ public interface ProcessingConstants { public static final String EXPLICIT_RESTART_CLUSTERS = "explicitRestartClusters"; public static final String SCRIPT_CONFIG_MAP = "scriptConfigMap"; + public static final String DOMAIN_HOME_CONFIG_MAP = "domainHomeConfigMap"; public static final String SERVER_STATE_MAP = "serverStateMap"; public static final String SERVER_HEALTH_MAP = "serverHealthMap"; diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java index 760121c94a1..c95f04e0357 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java @@ -20,6 +20,7 @@ import io.kubernetes.client.apis.ApiextensionsV1beta1Api; import io.kubernetes.client.apis.AuthenticationV1Api; import io.kubernetes.client.apis.AuthorizationV1Api; +import io.kubernetes.client.apis.BatchV1Api; import io.kubernetes.client.apis.CoreV1Api; import io.kubernetes.client.apis.ExtensionsV1beta1Api; import io.kubernetes.client.apis.VersionApi; @@ -28,9 +29,12 @@ import io.kubernetes.client.models.V1DeleteOptions; import io.kubernetes.client.models.V1Event; import io.kubernetes.client.models.V1EventList; +import io.kubernetes.client.models.V1Job; +import io.kubernetes.client.models.V1JobList; import io.kubernetes.client.models.V1ListMeta; import io.kubernetes.client.models.V1Namespace; import io.kubernetes.client.models.V1NamespaceList; +import io.kubernetes.client.models.V1PersistentVolume; import io.kubernetes.client.models.V1PersistentVolumeClaimList; import io.kubernetes.client.models.V1PersistentVolumeList; import io.kubernetes.client.models.V1Pod; @@ -879,6 +883,187 @@ public Step deleteCollectionPodAsync(String namespace, ResponseStep re return createRequestAsync(responseStep, new RequestParams("deleteCollection", namespace, null, null), DELETECOLLECTION_POD); } + /* Jobs */ + + /** + * List jobs + * @param namespace Namespace + * @return List of jobs + * @throws ApiException API Exception + */ + public V1JobList listJob(String namespace) throws ApiException { + String _continue = ""; + ApiClient client = helper.take(); + try { + return new BatchV1Api(client).listNamespacedJob(namespace, pretty, _continue, fieldSelector, + includeUninitialized, labelSelector, limit, resourceVersion, timeoutSeconds, watch); + } finally { + helper.recycle(client); + } + } + + private com.squareup.okhttp.Call listJobAsync(ApiClient client, String namespace, String _continue, ApiCallback callback) throws ApiException { + return new BatchV1Api(client).listNamespacedJobAsync(namespace, pretty, _continue, + fieldSelector, includeUninitialized, labelSelector, limit, resourceVersion, timeoutSeconds, watch, callback); + } + + private final CallFactory LIST_JOB = (requestParams, usage, cont, callback) -> { + return listJobAsync(usage, requestParams.namespace, cont, callback); + }; + + /** + * Asynchronous step for listing jobs + * @param namespace Namespace + * @param responseStep Response step for when call completes + * @return Asynchronous step + */ + public Step listJobsAsync(String namespace, ResponseStep responseStep) { + return createRequestAsync(responseStep, new RequestParams("listJob", namespace, null, null), LIST_JOB); + } + + /** + * Read job + * @param name Name + * @param namespace Namespace + * @return Read service + * @throws ApiException API Exception + */ + public V1Job readJob(String name, String namespace) throws ApiException { + ApiClient client = helper.take(); + try { + return new BatchV1Api(client).readNamespacedJob(name, namespace, pretty, exact, export); + } finally { + helper.recycle(client); + } + } + + private com.squareup.okhttp.Call readJobAsync(ApiClient client, String name, String namespace, ApiCallback callback) throws ApiException { + return new BatchV1Api(client).readNamespacedJobAsync(name, namespace, pretty, exact, export, callback); + } + + private final CallFactory READ_JOB = (requestParams, usage, cont, callback) -> { + return readJobAsync(usage, requestParams.name, requestParams.namespace, callback); + }; + + /** + * Asynchronous step for reading job + * @param name Name + * @param namespace Namespace + * @param responseStep Response step for when call completes + * @return Asynchronous step + */ + public Step readJobAsync(String name, String namespace, ResponseStep responseStep) { + return createRequestAsync(responseStep, new RequestParams("readJob", namespace, name, null), READ_JOB); + } + + /** + * Create job + * @param namespace Namespace + * @param body Body + * @return Created job + * @throws ApiException API Exception + */ + public V1Job createJob(String namespace, V1Job body) throws ApiException { + ApiClient client = helper.take(); + try { + return new BatchV1Api(client).createNamespacedJob(namespace, body, pretty); + } finally { + helper.recycle(client); + } + } + + private com.squareup.okhttp.Call createJobAsync(ApiClient client, String namespace, V1Job body, ApiCallback callback) throws ApiException { + return new BatchV1Api(client).createNamespacedJobAsync(namespace, body, pretty, callback); + } + + private final CallFactory CREATE_JOB = (requestParams, usage, cont, callback) -> { + return createJobAsync(usage, requestParams.namespace, (V1Job) requestParams.body, callback); + }; + + /** + * Asynchronous step for creating job + * @param namespace Namespace + * @param body Body + * @param responseStep Response step for when call completes + * @return Asynchronous step + */ + public Step createJobAsync(String namespace, V1Job body, ResponseStep responseStep) { + return createRequestAsync(responseStep, new RequestParams("createJob", namespace, null, body), CREATE_JOB); + } + + /** + * Replace job + * @param name Name + * @param namespace Namespace + * @param body Body + * @return Replaced job + * @throws ApiException API Exception + */ + public V1Job replaceJob(String name, String namespace, V1Job body) throws ApiException { + ApiClient client = helper.take(); + try { + return new BatchV1Api(client).replaceNamespacedJob(name, namespace, body, pretty); + } finally { + helper.recycle(client); + } + } + + private com.squareup.okhttp.Call replaceJobAsync(ApiClient client, String name, String namespace, V1Job body, ApiCallback callback) throws ApiException { + return new BatchV1Api(client).replaceNamespacedJobAsync(name, namespace, body, pretty, callback); + } + + private final CallFactory REPLACE_JOB = (requestParams, usage, cont, callback) -> { + return replaceJobAsync(usage, requestParams.name, requestParams.namespace, (V1Job) requestParams.body, callback); + }; + + /** + * Asynchronous step for replacing job + * @param name Name + * @param namespace Namespace + * @param body Body + * @param responseStep Response step for when call completes + * @return Asynchronous step + */ + public Step replaceJobAsync(String name, String namespace, V1Job body, ResponseStep responseStep) { + return createRequestAsync(responseStep, new RequestParams("replaceJob", namespace, name, body), REPLACE_JOB); + } + + /** + * Delete job + * @param name Name + * @param namespace Namespace + * @param body Delete options + * @return Status of deletion + * @throws ApiException API Exception + */ + public V1Status deleteJob(String name, String namespace, V1DeleteOptions body) throws ApiException { + ApiClient client = helper.take(); + try { + return new BatchV1Api(client).deleteNamespacedJob(name, namespace, body, pretty, gracePeriodSeconds, orphanDependents, propagationPolicy); + } finally { + helper.recycle(client); + } + } + + private com.squareup.okhttp.Call deleteJobAsync(ApiClient client, String name, String namespace,V1DeleteOptions body, ApiCallback callback) throws ApiException { + return new BatchV1Api(client).deleteNamespacedJobAsync(name, namespace, body, pretty, gracePeriodSeconds, orphanDependents, propagationPolicy, callback); + } + + private final CallFactory DELETE_JOB = (requestParams, usage, cont, callback) -> { + return deleteJobAsync(usage, requestParams.name, requestParams.namespace, (V1DeleteOptions) requestParams.body, callback); + }; + + /** + * Asynchronous step for deleting job + * @param name Name + * @param namespace Namespace + * @param responseStep Response step for when call completes + * @return Asynchronous step + */ + public Step deleteJobAsync(String name, String namespace, ResponseStep responseStep) { + return createRequestAsync(responseStep, new RequestParams("deleteJob", namespace, name, null), DELETE_JOB); + } + /* Services */ /** @@ -1067,7 +1252,7 @@ public Step deleteServiceAsync(String name, String namespace, ResponseStep responseStep) { + public Step createEventAsync(String namespace, V1Event body, ResponseStep responseStep) { return createRequestAsync(responseStep, new RequestParams("createEvent", namespace, null, body), CREATE_EVENT); } @@ -1203,79 +1388,180 @@ private com.squareup.okhttp.Call deleteEventAsync(ApiClient client, String name, public Step deleteEventAsync(String name, String namespace, ResponseStep responseStep) { return createRequestAsync(responseStep, new RequestParams("deleteEvent", namespace, name, null), DELETE_EVENT); } - - /* Persistent Volume Claims */ + + /* Persistent Volumes */ /** - * List persistent volume claims - * @param namespace Namespace - * @return List of persistent volume claims + * List persistent volumes + * @return List of persistent volumes * @throws ApiException API Exception */ - public V1PersistentVolumeClaimList listPersistentVolumeClaim(String namespace) throws ApiException { + public V1PersistentVolumeList listPersistentVolume() throws ApiException { String _continue = ""; ApiClient client = helper.take(); try { - return new CoreV1Api(client).listNamespacedPersistentVolumeClaim(namespace, pretty, _continue, fieldSelector, + return new CoreV1Api(client).listPersistentVolume(pretty, _continue, fieldSelector, includeUninitialized, labelSelector, limit, resourceVersion, timeoutSeconds, watch); } finally { helper.recycle(client); } } - private com.squareup.okhttp.Call listPersistentVolumeClaimAsync(ApiClient client, String namespace, String _continue, ApiCallback callback) throws ApiException { - return new CoreV1Api(client).listNamespacedPersistentVolumeClaimAsync(namespace, pretty, _continue, + private com.squareup.okhttp.Call listPersistentVolumeAsync(ApiClient client, String _continue, ApiCallback callback) throws ApiException { + return new CoreV1Api(client).listPersistentVolumeAsync(pretty, _continue, fieldSelector, includeUninitialized, labelSelector, limit, resourceVersion, timeoutSeconds, watch, callback); } - private final CallFactory LIST_PERSISTENTVOLUMECLAIM = (requestParams, usage, cont, callback) -> { - return listPersistentVolumeClaimAsync(usage, requestParams.namespace, cont, callback); + private final CallFactory LIST_PERSISTENT_VOLUME = (requestParams, usage, cont, callback) -> { + return listPersistentVolumeAsync(usage, cont, callback); }; /** - * Asynchronous step for listing persistent volume claims - * @param namespace Namespace + * Asynchronous step for listing persistent volumes * @param responseStep Response step for when call completes * @return Asynchronous step */ - public Step listPersistentVolumeClaimAsync(String namespace, ResponseStep responseStep) { - return createRequestAsync(responseStep, new RequestParams("listPersistentVolumeClaim", namespace, null, null), LIST_PERSISTENTVOLUMECLAIM); + public Step listPersistentVolumeAsync(String namespace, ResponseStep responseStep) { + return createRequestAsync(responseStep, new RequestParams("listPersistentVolume", null, null, null), LIST_PERSISTENT_VOLUME); } - /* Persistent Volumes */ + /** + * Read persistent volume + * @param name Name + * @return Read persistent volume + * @throws ApiException API Exception + */ + public V1PersistentVolume readEvent(String name) throws ApiException { + ApiClient client = helper.take(); + try { + return new CoreV1Api(client).readPersistentVolume(name, pretty, exact, export); + } finally { + helper.recycle(client); + } + } + + private com.squareup.okhttp.Call readPersistentVolumeAsync(ApiClient client, String name, ApiCallback callback) throws ApiException { + return new CoreV1Api(client).readPersistentVolumeAsync(name, pretty, exact, export, callback); + } + + private final CallFactory READ_PERSISTENT_VOLUME = (requestParams, usage, cont, callback) -> { + return readPersistentVolumeAsync(usage, requestParams.name, callback); + }; /** - * List persistent volumes - * @return List of persistent volumes + * Asynchronous step for reading persistent volume + * @param name Name + * @param responseStep Response step for when call completes + * @return Asynchronous step + */ + public Step readPersistentVolumeAsync(String name, ResponseStep responseStep) { + return createRequestAsync(responseStep, new RequestParams("readEvent", null, name, null), READ_PERSISTENT_VOLUME); + } + + /** + * Create persistent volume + * @param body Body + * @return Created persistent volume * @throws ApiException API Exception */ - public V1PersistentVolumeList listPersistentVolume() throws ApiException { + public V1PersistentVolume createPersistentVolume(V1PersistentVolume body) throws ApiException { + ApiClient client = helper.take(); + try { + return new CoreV1Api(client).createPersistentVolume(body, pretty); + } finally { + helper.recycle(client); + } + } + + private com.squareup.okhttp.Call createPersistentVolumeAsync(ApiClient client, V1PersistentVolume body, ApiCallback callback) throws ApiException { + return new CoreV1Api(client).createPersistentVolumeAsync(body, pretty, callback); + } + + private final CallFactory CREATE_PERSISTENT_VOLUME = (requestParams, usage, cont, callback) -> { + return createPersistentVolumeAsync(usage, (V1PersistentVolume) requestParams.body, callback); + }; + + /** + * Asynchronous step for creating persistent volume + * @param body Body + * @param responseStep Response step for when call completes + * @return Asynchronous step + */ + public Step createPersistentVolumeAsync(V1PersistentVolume body, ResponseStep responseStep) { + return createRequestAsync(responseStep, new RequestParams("createPersistentVolume", null, null, body), CREATE_PERSISTENT_VOLUME); + } + + /** + * Delete persistent volume + * @param name Name + * @param deleteOptions Deletion options + * @return Status of deletion + * @throws ApiException API Exception + */ + public V1Status deletePersistentVolume(String name, V1DeleteOptions deleteOptions) throws ApiException { + ApiClient client = helper.take(); + try { + return new CoreV1Api(client).deletePersistentVolume(name, deleteOptions, pretty, gracePeriodSeconds, + orphanDependents, propagationPolicy); + } finally { + helper.recycle(client); + } + } + + private com.squareup.okhttp.Call deletePersistentVolumeAsync(ApiClient client, String name, V1DeleteOptions deleteOptions, ApiCallback callback) throws ApiException { + return new CoreV1Api(client).deletePersistentVolumeAsync(name, deleteOptions, pretty, gracePeriodSeconds, orphanDependents, propagationPolicy, callback); + } + + private final CallFactory DELETE_PERSISTENT_VOLUME = (requestParams, usage, cont, callback) -> { + return deletePersistentVolumeAsync(usage, requestParams.name, (V1DeleteOptions) requestParams.body, callback); + }; + + /** + * Asynchronous step for deleting persistent volume + * @param name Name + * @param responseStep Response step for when call completes + * @return Asynchronous step + */ + public Step deletePersistentVolumeAsync(String name, ResponseStep responseStep) { + return createRequestAsync(responseStep, new RequestParams("deletePersistentVolume", null, name, null), DELETE_PERSISTENT_VOLUME); + } + + /* Persistent Volume Claims */ + + /** + * List persistent volume claims + * @param namespace Namespace + * @return List of persistent volume claims + * @throws ApiException API Exception + */ + public V1PersistentVolumeClaimList listPersistentVolumeClaim(String namespace) throws ApiException { String _continue = ""; ApiClient client = helper.take(); try { - return new CoreV1Api(client).listPersistentVolume(pretty, _continue, fieldSelector, includeUninitialized, - labelSelector, limit, resourceVersion, timeoutSeconds, watch); + return new CoreV1Api(client).listNamespacedPersistentVolumeClaim(namespace, pretty, _continue, fieldSelector, + includeUninitialized, labelSelector, limit, resourceVersion, timeoutSeconds, watch); } finally { helper.recycle(client); } } - private com.squareup.okhttp.Call listPersistentVolumeAsync(ApiClient client, String _continue, ApiCallback callback) throws ApiException { - return new CoreV1Api(client).listPersistentVolumeAsync(pretty, _continue, + private com.squareup.okhttp.Call listPersistentVolumeClaimAsync(ApiClient client, String namespace, String _continue, ApiCallback callback) throws ApiException { + return new CoreV1Api(client).listNamespacedPersistentVolumeClaimAsync(namespace, pretty, _continue, fieldSelector, includeUninitialized, labelSelector, limit, resourceVersion, timeoutSeconds, watch, callback); } - private final CallFactory LIST_PERSISTENTVOLUME = (requestParams, usage, cont, callback) -> { - return listPersistentVolumeAsync(usage, cont, callback); + private final CallFactory LIST_PERSISTENTVOLUMECLAIM = (requestParams, usage, cont, callback) -> { + return listPersistentVolumeClaimAsync(usage, requestParams.namespace, cont, callback); }; /** - * Asynchronous step for listing persistent volumes + * Asynchronous step for listing persistent volume claims + * @param namespace Namespace * @param responseStep Response step for when call completes * @return Asynchronous step */ - public Step listPersistentVolumeAsync(ResponseStep responseStep) { - return createRequestAsync(responseStep, new RequestParams("listPersistentVolume", null, null, null), LIST_PERSISTENTVOLUME); + public Step listPersistentVolumeClaimAsync(String namespace, ResponseStep responseStep) { + return createRequestAsync(responseStep, new RequestParams("listPersistentVolumeClaim", namespace, null, null), LIST_PERSISTENTVOLUMECLAIM); } /* Secrets */ diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java index 2b2e0b29ae5..5004ab6ebf2 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java @@ -3,6 +3,11 @@ package oracle.kubernetes.operator.helpers; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.Base64; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -28,7 +33,7 @@ public class ConfigMapHelper { private ConfigMapHelper() {} /** - * Factory for {@link Step} that creates config map containing scripts + * Factory for {@link Step} that creates config map containing scripts for server instances * @param operatorNamespace the operator's namespace * @param domainNamespace the domain's namespace * @param next Next processing step @@ -78,7 +83,7 @@ public NextAction onFailure(Packet packet, ApiException e, int statusCode, public NextAction onSuccess(Packet packet, V1ConfigMap result, int statusCode, Map> responseHeaders) { - LOGGER.info(MessageKeys.CM_CREATED, domainNamespace); + LOGGER.info(MessageKeys.CM_CREATED, KubernetesConstants.DOMAIN_CONFIG_MAP_NAME, domainNamespace); packet.put(ProcessingConstants.SCRIPT_CONFIG_MAP, result); return doNext(packet); } @@ -86,7 +91,7 @@ public NextAction onSuccess(Packet packet, V1ConfigMap result, int statusCode, return doNext(create, packet); } else if (AnnotationHelper.checkFormatAnnotation(result.getMetadata()) && result.getData().entrySet().containsAll(cm.getData().entrySet())) { // existing config map has correct data - LOGGER.fine(MessageKeys.CM_EXISTS, domainNamespace); + LOGGER.fine(MessageKeys.CM_EXISTS, KubernetesConstants.DOMAIN_CONFIG_MAP_NAME, domainNamespace); packet.put(ProcessingConstants.SCRIPT_CONFIG_MAP, result); return doNext(packet); } else { @@ -104,7 +109,7 @@ public NextAction onFailure(Packet packet, ApiException e, int statusCode, @Override public NextAction onSuccess(Packet packet, V1ConfigMap result, int statusCode, Map> responseHeaders) { - LOGGER.info(MessageKeys.CM_REPLACED, domainNamespace); + LOGGER.info(MessageKeys.CM_REPLACED, KubernetesConstants.DOMAIN_CONFIG_MAP_NAME, domainNamespace); packet.put(ProcessingConstants.SCRIPT_CONFIG_MAP, result); return doNext(packet); } @@ -214,4 +219,624 @@ protected V1ConfigMap computeDomainConfigMap() { } } + /** + * Factory for {@link Step} that creates config map containing scripts for domain home creation + * @param operatorNamespace the operator's namespace + * @param domainNamespace the domain's namespace + * @param next Next processing step + * @return Step for creating config map containing scripts + */ + public static Step createDomainHomeConfigMapStep(String operatorNamespace, String domainNamespace, Step next) { + return new DomainHomeConfigMapStep(operatorNamespace, domainNamespace, next); + } + + // Make this public so that it can be unit tested + public static class DomainHomeConfigMapStep extends Step { + private final String operatorNamespace; + private final String domainNamespace; + + public DomainHomeConfigMapStep(String operatorNamespace, String domainNamespace, Step next) { + super(next); + this.operatorNamespace = operatorNamespace; + this.domainNamespace = domainNamespace; + } + + @Override + public NextAction apply(Packet packet) { + V1ConfigMap cm; + try { + cm = computeDomainHomeConfigMap(); + } catch (IOException e1) { + return doTerminate(e1, packet); + } + + CallBuilderFactory factory = ContainerResolver.getInstance().getContainer().getSPI(CallBuilderFactory.class); + Step read = factory.create().readConfigMapAsync(cm.getMetadata().getName(), domainNamespace, new ResponseStep(next) { + @Override + public NextAction onFailure(Packet packet, ApiException e, int statusCode, + Map> responseHeaders) { + if (statusCode == CallBuilder.NOT_FOUND) { + return onSuccess(packet, null, statusCode, responseHeaders); + } + return super.onFailure(packet, e, statusCode, responseHeaders); + } + + @Override + public NextAction onSuccess(Packet packet, V1ConfigMap result, int statusCode, + Map> responseHeaders) { + if (result == null) { + Step create = factory.create().createConfigMapAsync(domainNamespace, cm, new ResponseStep(next) { + @Override + public NextAction onFailure(Packet packet, ApiException e, int statusCode, + Map> responseHeaders) { + return super.onFailure(DomainHomeConfigMapStep.this, packet, e, statusCode, responseHeaders); + } + + @Override + public NextAction onSuccess(Packet packet, V1ConfigMap result, int statusCode, + Map> responseHeaders) { + + LOGGER.info(MessageKeys.CM_CREATED, KubernetesConstants.DOMAIN_HOME_CONFIG_MAP_NAME, domainNamespace); + packet.put(ProcessingConstants.DOMAIN_HOME_CONFIG_MAP, result); + return doNext(packet); + } + }); + return doNext(create, packet); + } else if (AnnotationHelper.checkFormatAnnotation(result.getMetadata()) && result.getData().entrySet().containsAll(cm.getData().entrySet())) { + // existing config map has correct data + LOGGER.fine(MessageKeys.CM_EXISTS, KubernetesConstants.DOMAIN_HOME_CONFIG_MAP_NAME, domainNamespace); + packet.put(ProcessingConstants.DOMAIN_HOME_CONFIG_MAP, result); + return doNext(packet); + } else { + // we need to update the config map + Map updated = result.getData(); + updated.putAll(cm.getData()); + cm.setData(updated); + Step replace = factory.create().replaceConfigMapAsync(cm.getMetadata().getName(), domainNamespace, cm, new ResponseStep(next) { + @Override + public NextAction onFailure(Packet packet, ApiException e, int statusCode, + Map> responseHeaders) { + return super.onFailure(DomainHomeConfigMapStep.this, packet, e, statusCode, responseHeaders); + } + + @Override + public NextAction onSuccess(Packet packet, V1ConfigMap result, int statusCode, + Map> responseHeaders) { + LOGGER.info(MessageKeys.CM_REPLACED, KubernetesConstants.DOMAIN_CONFIG_MAP_NAME, domainNamespace); + packet.put(ProcessingConstants.DOMAIN_HOME_CONFIG_MAP, result); + return doNext(packet); + } + }); + return doNext(replace, packet); + } + } + }); + + return doNext(read, packet); + } + + // Make this protected so that it can be unit tested + protected V1ConfigMap computeDomainHomeConfigMap() throws IOException { + String name = KubernetesConstants.DOMAIN_HOME_CONFIG_MAP_NAME; + V1ConfigMap cm = new V1ConfigMap(); + cm.setApiVersion("v1"); + cm.setKind("ConfigMap"); + + V1ObjectMeta metadata = new V1ObjectMeta(); + metadata.setName(name); + metadata.setNamespace(domainNamespace); + + AnnotationHelper.annotateWithFormat(metadata); + + Map labels = new HashMap<>(); + labels.put(LabelConstants.OPERATORNAME_LABEL, operatorNamespace); + labels.put(LabelConstants.CREATEDBYOPERATOR_LABEL, "true"); + metadata.setLabels(labels); + + cm.setMetadata(metadata); + + Map data = new HashMap<>(); + + // HERE + data.put("weblogic-deploy.zip.base64", toBase64("/operator/weblogic-deploy.zip")); + + data.put("utility.sh", + "#!/bin/bash\n" + + "#\n" + + "\n" + + "#\n" + + "# Report an error and fail the job\n" + + "# $1 - text of error\n" + + "function fail {\n" + + " echo ERROR: $1\n" + + " exit 1\n" + + "}\n" + + "\n" + + "#\n" + + "# Create a folder\n" + + "# $1 - path of folder to create\n" + + "function createFolder {\n" + + " mkdir -m 777 -p $1\n" + + " if [ ! -d $1 ]; then\n" + + " fail \"Unable to create folder $1\"\n" + + " fi\n" + + "}\n" + + "\n" + + "#\n" + + "# Check a file exists\n" + + "# $1 - path of file to check\n" + + "function checkFileExists {\n" + + " if [ ! -f $1 ]; then\n" + + " fail \"The file $1 does not exist\"\n" + + " fi\n" + + "}\n"); + + data.put("create-domain-job.sh", + "#!/bin/bash\n" + + "#\n" + + "\n" + + "# Include common utility functions\n" + + "source /u01/weblogic/utility.sh\n" + + "\n" + + "# Verify the script to create the domain exists\n" + + "script='/u01/weblogic/create-domain-script.sh'\n" + + "if [ -f $script ]; then\n" + + " echo The domain will be created using the script $script\n" + + "else\n" + + " fail \"Could not locate the domain creation script ${script}\"\n" + + "fi\n" + + "\n" + + "# Validate the domain secrets exist before proceeding.\n" + + "if [ ! -f /weblogic-operator/secrets/username ]; then\n" + + " fail \"The domain secret /weblogic-operator/secrets/username was not found\"\n" + + "fi\n" + + "if [ ! -f /weblogic-operator/secrets/password ]; then\n" + + " fail \"The domain secret /weblogic-operator/secrets/password was not found\"\n" + + "fi\n" + + "\n" + + "# Do not proceed if the domain already exists\n" + + "domainFolder=${SHARED_PATH}/domain/%DOMAIN_NAME%\n" + + "if [ -d ${domainFolder} ]; then\n" + + " fail \"The create domain job will not overwrite an existing domain. The domain folder ${domainFolder} already exists\"\n" + + "fi\n" + + "\n" + + "# Create the base folders\n" + + "createFolder ${SHARED_PATH}/domain\n" + + "createFolder ${SHARED_PATH}/applications\n" + + "createFolder ${SHARED_PATH}/logs\n" + + "createFolder ${SHARED_PATH}/stores\n" + + "\n" + + "# Execute the script to create the domain\n" + + "source $script\n"); + + data.put("read-domain-secret.py", + "#\n" + + "# +++ Start of common code for reading domain secrets\n" + + "\n" + + "# Read username secret\n" + + "file = open('/weblogic-operator/secrets/username', 'r')\n" + + "admin_username = file.read()\n" + + "file.close()\n" + + "\n" + + "# Read password secret\n" + + "file = open('/weblogic-operator/secrets/password', 'r')\n" + + "admin_password = file.read()\n" + + "file.close()\n" + + "\n" + + "# +++ End of common code for reading domain secrets\n" + + "#\n"); + + data.put("create-domain-script.sh", + "#!/bin/bash\n" + + "#\n" + + "\n" + + "# Include common utility functions\n" + + "source /u01/weblogic/utility.sh\n" + + "\n" + + "export DOMAIN_HOME=${SHARED_PATH}/domain/%DOMAIN_NAME%\n" + + "\n" + + "# Function to create node manager home for a server\n" + + "# $1 - Domain UID\n" + + "# $2 - Server Name\n" + + "# $3 - Admin Server Hostname (only passed for managed servers)\n" + + "function createNodeMgrHome() {\n" + + "\n" + + " # Create startup.properties file\n" + + " datadir=${DOMAIN_HOME}/servers/$2/data/nodemanager\n" + + " startProp=${datadir}/startup.properties\n" + + " createFolder ${datadir}\n" + + " echo \"# Server startup properties\" > ${startProp}\n" + + " echo \"AutoRestart=true\" >> ${startProp}\n" + + " if [ -n \"$3\" ]; then\n" + + " echo \"AdminURL=http\\://$3\\:%ADMIN_PORT%\" >> ${startProp}\n" + + " fi\n" + + " echo \"RestartMax=2\" >> ${startProp}\n" + + " echo \"RotateLogOnStartup=false\" >> ${startProp}\n" + + " echo \"RotationType=bySize\" >> ${startProp}\n" + + " echo \"RotationTimeStart=00\\:00\" >> ${startProp}\n" + + " echo \"RotatedFileCount=100\" >> ${startProp}\n" + + " echo \"RestartDelaySeconds=0\" >> ${startProp}\n" + + " echo \"FileSizeKB=5000\" >> ${startProp}\n" + + " echo \"FileTimeSpanFactor=3600000\" >> ${startProp}\n" + + " echo \"RestartInterval=3600\" >> ${startProp}\n" + + " echo \"NumberOfFilesLimited=true\" >> ${startProp}\n" + + " echo \"FileTimeSpan=24\" >> ${startProp}\n" + + " echo \"NMHostName=$1-$2\" >> ${startProp}\n" + + "\n" + + " # Create nodemanager home for the server\n" + + " nmdir=${DOMAIN_HOME}/servers/$2/nodemgr_home\n" + + " createFolder ${nmdir}\n" + + " prop=${nmdir}/nodemanager.properties\n" + + " cp ${DOMAIN_HOME}/nodemanager/nodemanager.properties ${nmdir}\n" + + " cp ${DOMAIN_HOME}/nodemanager/nodemanager.domains ${nmdir}\n" + + " cp ${DOMAIN_HOME}/bin/startNodeManager.sh ${nmdir}\n" + + "\n" + + " # Edit the start nodemanager script to use the home for the server\n" + + " sed -i -e \"s:/nodemanager:/servers/$2/nodemgr_home:g\" ${nmdir}/startNodeManager.sh\n" + + "\n" + + " # Edit the nodemanager properties file to use the home for the server\n" + + " sed -i -e \"s:DomainsFile=.*:DomainsFile=${nmdir}/nodemanager.domains:g\" ${prop}\n" + + " sed -i -e \"s:NodeManagerHome=.*:NodeManagerHome=${nmdir}:g\" ${prop}\n" + + " sed -i -e \"s:ListenAddress=.*:ListenAddress=$1-$2:g\" ${prop}\n" + + " sed -i -e \"s:LogFile=.*:LogFile=/shared/logs/nodemanager-$2.log:g\" ${prop}\n" + + "\n" + + "}\n" + + "\n" + + "# Function to create script for starting a server\n" + + "# $1 - Domain UID\n" + + "# $2 - Server Name\n" + + "# $3 - Flag (only passed for admin server)\n" + + "function createStartScript() {\n" + + "\n" + + " nmdir=${DOMAIN_HOME}/servers/$2/nodemgr_home\n" + + " stateFile=${DOMAIN_HOME}/servers/$2/data/nodemanager/$2.state\n" + + " scriptFile=${nmdir}/startServer.sh\n" + + " pyFile=${nmdir}/start-server.py\n" + + " argsFile=${nmdir}/set-ms-args.py\n" + + "\n" + + " # Create a script that starts the node manager, then uses wlst to connect\n" + + " # to the nodemanager and start the server.\n" + + " # The script and 'EOF' on the following lines must not be indented!\n" + + " cat << EOF > ${scriptFile}\n" + + "#!/bin/bash\n" + + "\n" + + "# Base64 decode binary support files\n" + + "mkdir -p /tmp/weblogic-operator/bin\n" + + "shopt -s nullglob\n" + + "for f in /weblogic-operator/scripts/*.base64; do\n" + + " cat \"\\$f\" | base64 --decode > /tmp/weblogic-operator/bin/\\$(basename \"\\$f\" .base64)\n" + + "done\n" + + "\n" + + "# Check for stale state file and remove if found\"\n" + + "if [ -f ${stateFile} ]; then\n" + + " echo \"Removing stale file ${stateFile}\"\n" + + " rm ${stateFile}\n" + + "fi\n" + + "\n" + + "echo \"Start the nodemanager\"\n" + + ". ${nmdir}/startNodeManager.sh &\n" + + "\n" + + "echo \"Allow the nodemanager some time to start before attempting to connect\"\n" + + "sleep 15\n" + + "echo \"Finished waiting for the nodemanager to start\"\n" + + "\n" + + "echo \"Update JVM arguments\"\n" + + "if [ $# -eq 3 ]\n" + + "then\n" + + " echo \"Update JVM arguments for admin server\"\n" + + " echo \"Arguments=\\${USER_MEM_ARGS} -XX\\:+UnlockExperimentalVMOptions -XX\\:+UseCGroupMemoryLimitForHeap \\${JAVA_OPTIONS}\" >> ${startProp}\n" + + "else\n" + + " echo \"Update JVM arguments for managed server\"\n" + + " wlst.sh ${argsFile} $1 $2 ${startProp}\n" + + "fi\n" + + "\n" + + "echo \"Start the server\"\n" + + "wlst.sh -skipWLSModuleScanning ${pyFile}\n" + + "\n" + + "echo \"Wait indefinitely so that the Kubernetes pod does not exit and try to restart\"\n" + + "while true; do sleep 60; done\n" + + "EOF\n" + + "\n" + + " checkFileExists ${scriptFile}\n" + + " chmod +x ${scriptFile}\n" + + "\n" + + " # Create a python script to execute the wlst commands.\n" + + " # The script and 'EOF' on the following lines must not be indented!\n" + + " cat /u01/weblogic/read-domain-secret.py > ${pyFile}\n" + + " cat << EOF >> ${pyFile}\n" + + "\n" + + "# Connect to nodemanager and start server\n" + + "nmConnect(admin_username, admin_password, '$1-$2', '5556', '%DOMAIN_NAME%', '${DOMAIN_HOME}', 'plain')\n" + + "nmStart('$2')\n" + + "\n" + + "# Exit WLST\n" + + "nmDisconnect()\n" + + "exit()\n" + + "EOF\n" + + "\n" + + " checkFileExists ${pyFile}\n" + + "\n" + + " # Create a python script to set JVM arguments for managed server.\n" + + " # The script and 'EOF' on the following lines must not be indented!\n" + + " cat << EOF > ${argsFile}\n" + + "\n" + + "import os\n" + + "import sys\n" + + "EOF\n" + + "\n" + + " cat /u01/weblogic/read-domain-secret.py >> ${argsFile}\n" + + " cat << EOF >> ${argsFile}\n" + + "\n" + + "mem_args=os.environ['USER_MEM_ARGS']\n" + + "java_opt=os.environ['JAVA_OPTIONS']\n" + + "admin_server=os.environ['ADMIN_NAME']\n" + + "admin_port=os.environ['ADMIN_PORT']\n" + + "\n" + + "domain_UID=sys.argv[1]\n" + + "server_name=sys.argv[2]\n" + + "startup_file=sys.argv[3]\n" + + "\n" + + "adminUrl='t3://' + domain_UID + '-' + admin_server + ':' + admin_port\n" + + "dirStr='Servers/managed-server1/ServerStart/' + server_name\n" + + "\n" + + "# Connect to admin server to get startup arguments of this server\n" + + "connect(admin_username, admin_password, adminUrl)\n" + + "cd(dirStr)\n" + + "args=get('Arguments')\n" + + "disconnect()\n" + + "\n" + + "f = open(startup_file, 'a')\n" + + "s=str(\"Arguments=\"+ mem_args + \" -XX\\:+UnlockExperimentalVMOptions -XX\\:+UseCGroupMemoryLimitForHeap \" + java_opt )\n" + + "if not (args is None):\n" + + " s=str(s + \" \" + args + \"\\n\")\n" + + "else:\n" + + " s=str(s + \"\\n\")\n" + + "\n" + + "f.write(s)\n" + + "f.close()\n" + + "EOF\n" + + "\n" + + " checkFileExists ${argsFile}\n" + + "\n" + + "}\n" + + "\n" + + "# Function to create script for stopping a server\n" + + "# $1 - Domain UID\n" + + "# $2 - Server Name\n" + + "function createStopScript() {\n" + + "\n" + + " nmdir=${DOMAIN_HOME}/servers/$2/nodemgr_home\n" + + " scriptFile=${nmdir}/stopServer.sh\n" + + " pyFile=${nmdir}/stop-server.py\n" + + "\n" + + " # Create a script that stops the server.\n" + + " # The script and 'EOF' on the following lines must not be indented!\n" + + " cat << EOF > ${scriptFile}\n" + + "#!/bin/bash\n" + + "\n" + + "echo \"Stop the server\"\n" + + "wlst.sh -skipWLSModuleScanning ${pyFile}\n" + + "\n" + + "# Return status of 2 means failed to stop a server through the NodeManager.\n" + + "# Look to see if there is a server process that can be killed.\n" + + "if [ \\$? -eq 2 ]; then\n" + + " pid=\\$(jps -v | grep '[D]weblogic.Name=$2' | awk '{print \\$1}')\n" + + " if [ ! -z \\$pid ]; then\n" + + " echo \"Killing the server process \\$pid\"\n" + + " kill -15 \\$pid\n" + + " fi\n" + + "fi\n" + + "\n" + + "EOF\n" + + "\n" + + " checkFileExists ${scriptFile}\n" + + " chmod +x ${scriptFile}\n" + + "\n" + + " # Create a python script to execute the wlst commands.\n" + + " # The script and 'EOF' on the following lines must not be indented!\n" + + " cat /u01/weblogic/read-domain-secret.py > ${pyFile}\n" + + " cat << EOF >> ${pyFile}\n" + + "\n" + + "# Connect to nodemanager and stop server\n" + + "try:\n" + + " nmConnect(admin_username, admin_password, '$1-$2', '5556', '%DOMAIN_NAME%', '${DOMAIN_HOME}', 'plain')\n" + + "except:\n" + + " print('Failed to connect to the NodeManager')\n" + + " exit(exitcode=2)\n" + + "\n" + + "# Kill the server\n" + + "try:\n" + + " nmKill('$2')\n" + + "except:\n" + + " print('Connected to the NodeManager, but failed to stop the server')\n" + + " exit(exitcode=2)\n" + + "\n" + + "# Exit WLST\n" + + "nmDisconnect()\n" + + "exit()\n" + + "EOF\n" + + "}\n" + + "\n" + + "checkFileExists ${pyFile}\n" + + "\n" + + "# Create the domain\n" + + "wlst.sh -skipWLSModuleScanning /u01/weblogic/create-domain.py\n" + + "\n" + + "# Setup admin server\n" + + "createNodeMgrHome %DOMAIN_UID% %ADMIN_SERVER_NAME%\n" + + "createStartScript %DOMAIN_UID% %ADMIN_SERVER_NAME% 'admin'\n" + + "createStopScript %DOMAIN_UID% %ADMIN_SERVER_NAME%\n" + + "\n" + + "# Create the managed servers\n" + + "index=0\n" + + "while [ $index -lt %CONFIGURED_MANAGED_SERVER_COUNT% ]\n" + + "do\n" + + " ((index++))\n" + + " createNodeMgrHome %DOMAIN_UID% %MANAGED_SERVER_NAME_BASE%${index} %DOMAIN_UID%-%ADMIN_SERVER_NAME%\n" + + " createStartScript %DOMAIN_UID% %MANAGED_SERVER_NAME_BASE%${index}\n" + + " createStopScript %DOMAIN_UID% %MANAGED_SERVER_NAME_BASE%${index}\n" + + "done\n" + + "\n" + + "echo \"Successfully Completed\"\n"); + + data.put("create-domain.py", + "# This python script is used to create a WebLogic domain\n" + + "\n" + + "# Read the domain secrets from the common python file\n" + + "execfile(\"/u01/weblogic/read-domain-secret.py\")\n" + + "\n" + + "server_port = %MANAGED_SERVER_PORT%\n" + + "domain_path = os.environ.get(\"DOMAIN_HOME\")\n" + + "cluster_name = \"%CLUSTER_NAME%\"\n" + + "number_of_ms = %CONFIGURED_MANAGED_SERVER_COUNT%\n" + + "\n" + + "print('domain_path : [%s]' % domain_path);\n" + + "print('domain_name : [%DOMAIN_NAME%]');\n" + + "print('admin_username : [%s]' % admin_username);\n" + + "print('admin_port : [%ADMIN_PORT%]');\n" + + "print('cluster_name : [%s]' % cluster_name);\n" + + "print('server_port : [%s]' % server_port);\n" + + "\n" + + "# Open default domain template\n" + + "# ============================\n" + + "readTemplate(\"/u01/oracle/wlserver/common/templates/wls/wls.jar\")\n" + + "\n" + + "set('Name', '%DOMAIN_NAME%')\n" + + "setOption('DomainName', '%DOMAIN_NAME%')\n" + + "create('%DOMAIN_NAME%','Log')\n" + + "cd('/Log/%DOMAIN_NAME%');\n" + + "set('FileName', '/shared/logs/%DOMAIN_NAME%.log')\n" + + "\n" + + "# Configure the Administration Server\n" + + "# ===================================\n" + + "cd('/Servers/AdminServer')\n" + + "set('ListenAddress', '%DOMAIN_UID%-%ADMIN_SERVER_NAME%')\n" + + "set('ListenPort', %ADMIN_PORT%)\n" + + "set('Name', '%ADMIN_SERVER_NAME%')\n" + + "\n" + + "create('T3Channel', 'NetworkAccessPoint')\n" + + "cd('/Servers/%ADMIN_SERVER_NAME%/NetworkAccessPoints/T3Channel')\n" + + "set('PublicPort', %T3_CHANNEL_PORT%)\n" + + "set('PublicAddress', '%T3_PUBLIC_ADDRESS%')\n" + + "set('ListenAddress', '%DOMAIN_UID%-%ADMIN_SERVER_NAME%')\n" + + "set('ListenPort', %T3_CHANNEL_PORT%)\n" + + "\n" + + "cd('/Servers/%ADMIN_SERVER_NAME%')\n" + + "create('%ADMIN_SERVER_NAME%', 'Log')\n" + + "cd('/Servers/%ADMIN_SERVER_NAME%/Log/%ADMIN_SERVER_NAME%')\n" + + "set('FileName', '/shared/logs/%ADMIN_SERVER_NAME%.log')\n" + + "\n" + + "# Set the admin user's username and password\n" + + "# ==========================================\n" + + "cd('/Security/%DOMAIN_NAME%/User/weblogic')\n" + + "cmo.setName(admin_username)\n" + + "cmo.setPassword(admin_password)\n" + + "\n" + + "# Write the domain and close the domain template\n" + + "# ==============================================\n" + + "setOption('OverwriteDomain', 'true')\n" + + "\n" + + "# Configure the node manager\n" + + "# ==========================\n" + + "cd('/NMProperties')\n" + + "set('ListenAddress','0.0.0.0')\n" + + "set('ListenPort',5556)\n" + + "set('CrashRecoveryEnabled', 'true')\n" + + "set('NativeVersionEnabled', 'true')\n" + + "set('StartScriptEnabled', 'false')\n" + + "set('SecureListener', 'false')\n" + + "set('LogLevel', 'FINEST')\n" + + "set('DomainsDirRemoteSharingEnabled', 'true')\n" + + "\n" + + "# Set the Node Manager user name and password (domain name will change after writeDomain)\n" + + "cd('/SecurityConfiguration/base_domain')\n" + + "set('NodeManagerUsername', admin_username)\n" + + "set('NodeManagerPasswordEncrypted', admin_password)\n" + + "\n" + + "# Create a cluster\n" + + "cd('/')\n" + + "create(cluster_name, 'Cluster')\n" + + "\n" + + "# Create managed servers\n" + + "for index in range(0, number_of_ms):\n" + + " cd('/')\n" + + "\n" + + " msIndex = index+1\n" + + " name = '%MANAGED_SERVER_NAME_BASE%%s' % msIndex\n" + + "\n" + + " create(name, 'Server')\n" + + " cd('/Servers/%s/' % name )\n" + + " print('managed server name is %s' % name);\n" + + " set('ListenAddress', '%DOMAIN_UID%-%s' % name)\n" + + " set('ListenPort', server_port)\n" + + " set('NumOfRetriesBeforeMSIMode', 0)\n" + + " set('RetryIntervalBeforeMSIMode', 1)\n" + + " set('Cluster', cluster_name)\n" + + "\n" + + " create(name,'Log')\n" + + " cd('/Servers/%s/Log/%s' % (name, name))\n" + + " set('FileName', '/shared/logs/%s.log' % name)\n" + + "\n" + + "# Write Domain\n" + + "# ============\n" + + "writeDomain(domain_path)\n" + + "closeTemplate()\n" + + "print 'Domain Created'\n" + + "\n" + + "# Update Domain\n" + + "readDomain(domain_path)\n" + + "cd('/')\n" + + "cmo.setProductionModeEnabled(%PRODUCTION_MODE_ENABLED%)\n" + + "updateDomain()\n" + + "closeDomain()\n" + + "print 'Domain Updated'\n" + + "\n" + + "# Encrypt the admin username and password\n" + + "adminUsernameEncrypted=encrypt(admin_username, domain_path)\n" + + "adminPasswordEncrypted=encrypt(admin_password, domain_path)\n" + + "\n" + + "print 'Create boot.properties files for admin and managed servers'\n" + + "\n" + + "asbpFile=open('%s/servers/%ADMIN_SERVER_NAME%/security/boot.properties' % domain_path, 'w+')\n" + + "asbpFile.write(\"username=%s\\n\" % adminUsernameEncrypted)\n" + + "asbpFile.write(\"password=%s\\n\" % adminPasswordEncrypted)\n" + + "asbpFile.close()\n" + + "\n" + + "import os\n" + + "\n" + + "# Create boot.properties file for each managed server\n" + + "for index in range(0, number_of_ms):\n" + + "\n" + + " # Define the folder path\n" + + " secdir='%s/servers/%MANAGED_SERVER_NAME_BASE%%s/security' % (domain_path, index+1)\n" + + "\n" + + " # Create the security folder (if it does not already exist)\n" + + " try:\n" + + " os.makedirs(secdir)\n" + + " except OSError:\n" + + " if not os.path.isdir(secdir):\n" + + " raise\n" + + "\n" + + " bpFile=open('%s/boot.properties' % secdir, 'w+')\n" + + " bpFile.write(\"username=%s\\n\" % adminUsernameEncrypted)\n" + + " bpFile.write(\"password=%s\\n\" % adminPasswordEncrypted)\n" + + " bpFile.close()\n" + + "\n" + + "print 'Done'\n" + + "\n" + + "# Exit WLST\n" + + "# =========\n" + + "exit()\n"); + + cm.setData(data); + + return cm; + } + } + + private static String toBase64(String fileName) throws IOException { + File file = new File(fileName); + byte[] encoded = Base64.getEncoder().encode(Files.readAllBytes(file.toPath())); + return new String(encoded, StandardCharsets.UTF_8); + } } diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/DomainHomeHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/DomainHomeHelper.java new file mode 100644 index 00000000000..897cf6dac75 --- /dev/null +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/DomainHomeHelper.java @@ -0,0 +1,200 @@ +// Copyright 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.helpers; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import io.kubernetes.client.ApiException; +import io.kubernetes.client.models.V1ConfigMapVolumeSource; +import io.kubernetes.client.models.V1Container; +import io.kubernetes.client.models.V1EnvVar; +import io.kubernetes.client.models.V1Job; +import io.kubernetes.client.models.V1JobSpec; +import io.kubernetes.client.models.V1ObjectMeta; +import io.kubernetes.client.models.V1PersistentVolumeClaimVolumeSource; +import io.kubernetes.client.models.V1PodSpec; +import io.kubernetes.client.models.V1PodTemplateSpec; +import io.kubernetes.client.models.V1SecretVolumeSource; +import io.kubernetes.client.models.V1Status; +import io.kubernetes.client.models.V1Volume; +import io.kubernetes.client.models.V1VolumeMount; +import oracle.kubernetes.operator.KubernetesConstants; +import oracle.kubernetes.operator.LabelConstants; +import oracle.kubernetes.operator.logging.LoggingFacade; +import oracle.kubernetes.operator.logging.LoggingFactory; +import oracle.kubernetes.operator.work.Container; +import oracle.kubernetes.operator.work.ContainerResolver; +import oracle.kubernetes.operator.work.NextAction; +import oracle.kubernetes.operator.work.Packet; +import oracle.kubernetes.operator.work.Step; +import oracle.kubernetes.weblogic.domain.v1.Domain; +import oracle.kubernetes.weblogic.domain.v1.DomainSpec; + +public class DomainHomeHelper { + private static final LoggingFacade LOGGER = LoggingFactory.getLogger("Operator", "Operator"); + + private DomainHomeHelper() {} + + public static Step createDomainHomeStep(Step next) { + return new DomainHomeStep(next); + } + + public static class DomainHomeStep extends Step { + public DomainHomeStep(Step next) { + super(next); + } + + @Override + public NextAction apply(Packet packet) { + Container c = ContainerResolver.getInstance().getContainer(); + CallBuilderFactory factory = c.getSPI(CallBuilderFactory.class); + + DomainPresenceInfo info = packet.getSPI(DomainPresenceInfo.class); + Domain dom = info.getDomain(); + V1ObjectMeta meta = dom.getMetadata(); + String namespace = meta.getNamespace(); + + V1Job job = computeDomainHomeJob(info); + Step conflictStep = factory.create().deleteJobAsync(job.getMetadata().getName(), namespace, new ResponseStep(next) { + @Override + public NextAction onFailure(Packet packet, ApiException e, int statusCode, + Map> responseHeaders) { + if (statusCode == CallBuilder.NOT_FOUND) { + return onSuccess(packet, null, statusCode, responseHeaders); + } + return super.onFailure(packet, e, statusCode, responseHeaders); + } + + @Override + public NextAction onSuccess(Packet packet, V1Status result, int statusCode, + Map> responseHeaders) { + return doNext(DomainHomeStep.this, packet); + } + }); + + Step create = factory.create().createJobAsync(namespace, computeDomainHomeJob(info), new ResponseStep(next) { + @Override + public NextAction onFailure(Packet packet, ApiException e, int statusCode, + Map> responseHeaders) { + return super.onFailure(conflictStep, packet, e, statusCode, responseHeaders); + } + + @Override + public NextAction onSuccess(Packet packet, V1Job result, int statusCode, + Map> responseHeaders) { + // TODO: wait for the job to finish or fail + // update domain status condition + return null; + } + }); + + return doNext(create, packet); + } + + protected V1Job computeDomainHomeJob(DomainPresenceInfo info) { + Domain dom = info.getDomain(); + V1ObjectMeta meta = dom.getMetadata(); + DomainSpec spec = dom.getSpec(); + String namespace = meta.getNamespace(); + + String weblogicDomainUID = spec.getDomainUID(); + String weblogicDomainName = spec.getDomainName(); + + String jobName = weblogicDomainUID + "-create-weblogic-domain-job"; + + String imageName = spec.getImage(); + if (imageName == null || imageName.length() == 0) { + imageName = KubernetesConstants.DEFAULT_IMAGE; + } + String imagePullPolicy = spec.getImagePullPolicy(); + if (imagePullPolicy == null || imagePullPolicy.length() == 0) { + imagePullPolicy = (imageName.endsWith(KubernetesConstants.LATEST_IMAGE_SUFFIX)) ? KubernetesConstants.ALWAYS_IMAGEPULLPOLICY : KubernetesConstants.IFNOTPRESENT_IMAGEPULLPOLICY; + } + + V1Job job = new V1Job(); + + V1ObjectMeta metadata = new V1ObjectMeta(); + metadata.setName(jobName); + metadata.setNamespace(namespace); + job.setMetadata(metadata); + + AnnotationHelper.annotateWithFormat(metadata); + + Map labels = new HashMap<>(); + labels.put(LabelConstants.DOMAINUID_LABEL, weblogicDomainUID); + labels.put(LabelConstants.DOMAINNAME_LABEL, weblogicDomainName); + labels.put(LabelConstants.SERVERNAME_LABEL, spec.getAsName()); + labels.put(LabelConstants.CREATEDBYOPERATOR_LABEL, "true"); + metadata.setLabels(labels); + + V1JobSpec jobSpec = new V1JobSpec(); + V1PodTemplateSpec templateSpec = new V1PodTemplateSpec(); + V1PodSpec podSpec = new V1PodSpec(); + podSpec.setRestartPolicy("Never"); + + V1Container container = new V1Container(); + container.setName(weblogicDomainUID + "-create-weblogic-domain-job"); + container.setImage(imageName); + container.setImagePullPolicy(imagePullPolicy); + + V1VolumeMount volumeMount = new V1VolumeMount(); + volumeMount.setName("create-weblogic-domain-job-cm-volume"); + volumeMount.setMountPath("/u01/weblogic"); + container.addVolumeMountsItem(volumeMount); + + V1VolumeMount volumeMountShared = new V1VolumeMount(); + volumeMountShared.setName("weblogic-domain-storage-volume"); + volumeMountShared.setMountPath("/shared"); + container.addVolumeMountsItem(volumeMountShared); + + V1VolumeMount volumeMountSecrets = new V1VolumeMount(); + volumeMountSecrets.setName("weblogic-credentials-volume"); + volumeMountSecrets.setMountPath("/weblogic-operator/secrets"); + container.addVolumeMountsItem(volumeMountSecrets); + + container.setCommand(Arrays.asList("/bin/sh")); + container.addArgsItem("/u01/weblogic/create-domain-job.sh"); + + V1EnvVar envItem = new V1EnvVar(); + envItem.setName("SHARED_PATH"); + envItem.setValue("/shared"); + container.addEnvItem(envItem); + + podSpec.addContainersItem(container); + + V1Volume volumeDomainConfigMap = new V1Volume(); + volumeDomainConfigMap.setName("create-weblogic-domain-job-cm-volume"); + V1ConfigMapVolumeSource cm = new V1ConfigMapVolumeSource(); + cm.setName(KubernetesConstants.DOMAIN_HOME_CONFIG_MAP_NAME); + cm.setDefaultMode(0555); // read and execute + volumeDomainConfigMap.setConfigMap(cm); + podSpec.addVolumesItem(volumeDomainConfigMap); + + if (!info.getClaims().getItems().isEmpty()) { + V1Volume volume = new V1Volume(); + volume.setName("weblogic-domain-storage-volume"); + V1PersistentVolumeClaimVolumeSource pvClaimSource = new V1PersistentVolumeClaimVolumeSource(); + pvClaimSource.setClaimName(info.getClaims().getItems().iterator().next().getMetadata().getName()); + volume.setPersistentVolumeClaim(pvClaimSource); + podSpec.addVolumesItem(volume); + } + + V1Volume volumeSecret = new V1Volume(); + volumeSecret.setName("weblogic-credentials-volume"); + V1SecretVolumeSource secret = new V1SecretVolumeSource(); + secret.setSecretName(spec.getAdminSecret().getName()); + volumeSecret.setSecret(secret); + podSpec.addVolumesItem(volumeSecret); + + templateSpec.setSpec(podSpec); + + jobSpec.setTemplate(templateSpec); + job.setSpec(jobSpec); + return job; + } + } +} diff --git a/operator/src/main/resources/Operator.properties b/operator/src/main/resources/Operator.properties index b5267a29f22..0a666f0874a 100644 --- a/operator/src/main/resources/Operator.properties +++ b/operator/src/main/resources/Operator.properties @@ -54,9 +54,9 @@ WLSKO-0052=Existing managed server Service is correct for WebLogic domain with U WLSKO-0053=Creating cluster Service for WebLogic domain with UID: {0}. Cluster name: {1}. WLSKO-0054=Replacing cluster Service for WebLogic domain with UID: {0}. Cluster name: {1}. WLSKO-0055=Existing cluster Service is correct for WebLogic domain with UID: {0}. Cluster name: {1}. -WLSKO-0056=Creating domain control config map for namespace: {0}. -WLSKO-0057=Replacing domain control config map for namespace: {0}. -WLSKO-0058=Existing domain config map is correct for namespace: {0}. +WLSKO-0056=Creating domain config map, {0}, for namespace: {1}. +WLSKO-0057=Replacing domain config map, {0}, for namespace: {1}. +WLSKO-0058=Existing domain config map, {0}, is correct for namespace: {1}. WLSKO-0059=Cannot create Token Review - not authorized. WLSKO-0060=Cannot create Network Policy - not authorized. WLSKO-0061=Cannot delete Network Policy - not authorized. diff --git a/operator/src/test/java/oracle/kubernetes/operator/create/CreateOperatorGeneratedFilesBaseTest.java b/operator/src/test/java/oracle/kubernetes/operator/create/CreateOperatorGeneratedFilesBaseTest.java index bb4c7934de4..e1ba29517bd 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/create/CreateOperatorGeneratedFilesBaseTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/create/CreateOperatorGeneratedFilesBaseTest.java @@ -321,8 +321,12 @@ protected V1beta1ClusterRole getExpectedWeblogicOperatorClusterRole() { .putLabelsItem(OPERATORNAME_LABEL, getInputs().getNamespace())) .addRulesItem(newPolicyRule() .addApiGroupsItem("") - .resources(asList("namespaces", "persistentvolumes")) + .resources(asList("namespaces")) .verbs(asList("get", "list", "watch"))) + .addRulesItem(newPolicyRule() + .addApiGroupsItem("") + .resources(asList("persistentvolumes")) + .verbs(asList("get", "list", "watch", "create", "update", "patch", "delete", "deletecollection"))) .addRulesItem(newPolicyRule() .addApiGroupsItem("apiextensions.k8s.io") .addResourcesItem("customresourcedefinitions") @@ -482,7 +486,7 @@ protected V1beta1ClusterRole getExpectedWeblogicOperatorNamespaceRole() { .putLabelsItem(OPERATORNAME_LABEL, getInputs().getNamespace())) .addRulesItem(newPolicyRule() .addApiGroupsItem("") - .resources(asList("secrets", "persistentvolumeclaims")) + .resources(asList("secrets")) .verbs(asList("get", "list", "watch"))) .addRulesItem(newPolicyRule() .addApiGroupsItem("storage.k8s.io") @@ -490,7 +494,7 @@ protected V1beta1ClusterRole getExpectedWeblogicOperatorNamespaceRole() { .verbs(asList("get", "list", "watch"))) .addRulesItem(newPolicyRule() .addApiGroupsItem("") - .resources(asList("services", "configmaps", "pods", "jobs", "events")) + .resources(asList("services", "configmaps", "pods", "podtemplates", "events", "persistentvolumeclaims")) .verbs(asList("get", "list", "watch", "create", "update", "patch", "delete", "deletecollection"))) .addRulesItem(newPolicyRule() .addApiGroupsItem("") @@ -500,6 +504,10 @@ protected V1beta1ClusterRole getExpectedWeblogicOperatorNamespaceRole() { .addApiGroupsItem("") .resources(asList("pods/exec")) .verbs(asList("create"))) + .addRulesItem(newPolicyRule() + .addApiGroupsItem("batch") + .resources(asList("jobs", "cronjobs")) + .verbs(asList("get", "list", "watch", "create", "update", "patch", "delete", "deletecollection"))) .addRulesItem(newPolicyRule() .addApiGroupsItem("settings.k8s.io") .addResourcesItem("podpresets") diff --git a/pom.xml b/pom.xml index 61f98f3fdd1..786cc658ee9 100644 --- a/pom.xml +++ b/pom.xml @@ -49,6 +49,23 @@ + + com.googlecode.maven-download-plugin + download-maven-plugin + 1.4.0 + + + install-wdt + + wget + + + https://github.com/oracle/weblogic-deploy-tooling/releases/download/weblogic-deploy-tooling-0.8/weblogic-deploy.zip + ${project.build.directory} + + + + org.apache.maven.plugins maven-release-plugin diff --git a/wercker.yml b/wercker.yml index 6495f12fd9d..673c264ddab 100644 --- a/wercker.yml +++ b/wercker.yml @@ -45,6 +45,7 @@ build: cp -R src/scripts/* /operator/ cp operator/target/weblogic-kubernetes-operator-0.2.jar /operator/weblogic-kubernetes-operator.jar cp operator/target/lib/*.jar /operator/lib/ + cp target/weblogic-deploy.zip /operator/weblogic-deploy.zip export IMAGE_TAG_OPERATOR="${WERCKER_GIT_BRANCH//[_\/]/-}" if [ "$IMAGE_TAG_OPERATOR" = "master" ]; then export IMAGE_TAG_OPERATOR="latest" From bfcb77faacdfea6efe5003eddc345694ab8a5dbf Mon Sep 17 00:00:00 2001 From: Ryan Eberhard Date: Mon, 16 Apr 2018 11:30:25 -0400 Subject: [PATCH 038/186] Move inner classes to their own files --- .../java/oracle/kubernetes/operator/Main.java | 868 +----------------- .../steps/BeforeAdminServiceStep.java | 42 + .../operator/steps/ClusterServicesStep.java | 56 ++ .../operator/steps/ConfigMapAfterStep.java | 47 + .../operator/steps/DeleteDomainStep.java | 111 +++ .../operator/steps/DeleteIngressListStep.java | 68 ++ .../operator/steps/DeleteServiceListStep.java | 60 ++ .../operator/steps/DomainPrescenceStep.java | 44 + .../ExternalAdminChannelIteratorStep.java | 37 + .../steps/ExternalAdminChannelsStep.java | 118 +++ .../steps/ListPersistentVolumeClaimStep.java | 64 ++ .../steps/ManagedServerUpAfterStep.java | 52 ++ .../steps/ManagedServerUpIteratorStep.java | 83 ++ .../operator/steps/ManagedServersUpStep.java | 259 ++++++ .../steps/ServerDownFinalizeStep.java | 22 + .../steps/ServerDownIteratorStep.java | 56 ++ .../operator/steps/ServerDownStep.java | 28 + .../steps/WatchPodReadyAdminStep.java | 35 + .../operator/ExternalChannelTest.java | 11 +- 19 files changed, 1206 insertions(+), 855 deletions(-) create mode 100644 operator/src/main/java/oracle/kubernetes/operator/steps/BeforeAdminServiceStep.java create mode 100644 operator/src/main/java/oracle/kubernetes/operator/steps/ClusterServicesStep.java create mode 100644 operator/src/main/java/oracle/kubernetes/operator/steps/ConfigMapAfterStep.java create mode 100644 operator/src/main/java/oracle/kubernetes/operator/steps/DeleteDomainStep.java create mode 100644 operator/src/main/java/oracle/kubernetes/operator/steps/DeleteIngressListStep.java create mode 100644 operator/src/main/java/oracle/kubernetes/operator/steps/DeleteServiceListStep.java create mode 100644 operator/src/main/java/oracle/kubernetes/operator/steps/DomainPrescenceStep.java create mode 100644 operator/src/main/java/oracle/kubernetes/operator/steps/ExternalAdminChannelIteratorStep.java create mode 100644 operator/src/main/java/oracle/kubernetes/operator/steps/ExternalAdminChannelsStep.java create mode 100644 operator/src/main/java/oracle/kubernetes/operator/steps/ListPersistentVolumeClaimStep.java create mode 100644 operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServerUpAfterStep.java create mode 100644 operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServerUpIteratorStep.java create mode 100644 operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServersUpStep.java create mode 100644 operator/src/main/java/oracle/kubernetes/operator/steps/ServerDownFinalizeStep.java create mode 100644 operator/src/main/java/oracle/kubernetes/operator/steps/ServerDownIteratorStep.java create mode 100644 operator/src/main/java/oracle/kubernetes/operator/steps/ServerDownStep.java create mode 100644 operator/src/main/java/oracle/kubernetes/operator/steps/WatchPodReadyAdminStep.java diff --git a/operator/src/main/java/oracle/kubernetes/operator/Main.java b/operator/src/main/java/oracle/kubernetes/operator/Main.java index 408ba89a793..9d1e1b32823 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/Main.java +++ b/operator/src/main/java/oracle/kubernetes/operator/Main.java @@ -7,7 +7,6 @@ import java.io.InputStream; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -28,18 +27,15 @@ import io.kubernetes.client.ApiException; import io.kubernetes.client.JSON; import io.kubernetes.client.models.V1ConfigMap; -import io.kubernetes.client.models.V1DeleteOptions; import io.kubernetes.client.models.V1EnvVar; import io.kubernetes.client.models.V1Event; import io.kubernetes.client.models.V1EventList; import io.kubernetes.client.models.V1ObjectMeta; import io.kubernetes.client.models.V1ObjectReference; -import io.kubernetes.client.models.V1PersistentVolumeClaimList; import io.kubernetes.client.models.V1Pod; import io.kubernetes.client.models.V1PodList; import io.kubernetes.client.models.V1Service; import io.kubernetes.client.models.V1ServiceList; -import io.kubernetes.client.models.V1Status; import io.kubernetes.client.models.V1beta1Ingress; import io.kubernetes.client.models.V1beta1IngressList; import io.kubernetes.client.util.Watch; @@ -55,13 +51,10 @@ import oracle.kubernetes.operator.helpers.CallBuilderFactory; import oracle.kubernetes.operator.helpers.ConfigMapHelper; import oracle.kubernetes.operator.helpers.DomainPresenceInfo; -import oracle.kubernetes.operator.helpers.DomainPresenceInfo.ServerStartupInfo; import oracle.kubernetes.operator.helpers.HealthCheckHelper.KubernetesVersion; import oracle.kubernetes.operator.helpers.HealthCheckHelper; -import oracle.kubernetes.operator.helpers.IngressHelper; import oracle.kubernetes.operator.helpers.PodHelper; import oracle.kubernetes.operator.helpers.ResponseStep; -import oracle.kubernetes.operator.helpers.RollingHelper; import oracle.kubernetes.operator.helpers.ServerKubernetesObjects; import oracle.kubernetes.operator.helpers.ServerKubernetesObjectsFactory; import oracle.kubernetes.operator.helpers.ServiceHelper; @@ -70,12 +63,16 @@ import oracle.kubernetes.operator.logging.MessageKeys; import oracle.kubernetes.operator.rest.RestConfigImpl; import oracle.kubernetes.operator.rest.RestServer; +import oracle.kubernetes.operator.steps.BeforeAdminServiceStep; +import oracle.kubernetes.operator.steps.ConfigMapAfterStep; +import oracle.kubernetes.operator.steps.DeleteDomainStep; +import oracle.kubernetes.operator.steps.DomainPrescenceStep; +import oracle.kubernetes.operator.steps.ExternalAdminChannelsStep; +import oracle.kubernetes.operator.steps.ListPersistentVolumeClaimStep; +import oracle.kubernetes.operator.steps.ManagedServersUpStep; +import oracle.kubernetes.operator.steps.WatchPodReadyAdminStep; import oracle.kubernetes.operator.utils.ConcurrentWeakHashMap; -import oracle.kubernetes.operator.wlsconfig.NetworkAccessPoint; -import oracle.kubernetes.operator.wlsconfig.WlsClusterConfig; import oracle.kubernetes.operator.wlsconfig.WlsRetriever; -import oracle.kubernetes.operator.wlsconfig.WlsDomainConfig; -import oracle.kubernetes.operator.wlsconfig.WlsServerConfig; import oracle.kubernetes.operator.work.Component; import oracle.kubernetes.operator.work.Container; import oracle.kubernetes.operator.work.Engine; @@ -114,7 +111,7 @@ public class Main { throw new RuntimeException(e); } } - private static final CallBuilderFactory callBuilderFactory = new CallBuilderFactory(tuningAndConfig); + static final CallBuilderFactory callBuilderFactory = new CallBuilderFactory(tuningAndConfig); private static final Container container = new Container(); private static final ScheduledExecutorService wrappedExecutorService = Engine.wrappedExecutorService("operator", @@ -122,8 +119,11 @@ public class Main { static { container.getComponents().put(ProcessingConstants.MAIN_COMPONENT_NAME, - Component.createFor(ScheduledExecutorService.class, wrappedExecutorService, TuningParameters.class, - tuningAndConfig, callBuilderFactory, skoFactory)); + Component.createFor( + ScheduledExecutorService.class, wrappedExecutorService, + TuningParameters.class, tuningAndConfig, + ThreadFactory.class, factory, + callBuilderFactory, skoFactory)); } private static final Engine engine = new Engine(wrappedExecutorService); @@ -257,7 +257,8 @@ public NextAction onSuccess(Packet packet, DomainList result, int statusCode, }); Step initialize = ConfigMapHelper.createScriptConfigMapStep(namespace, ns, - new ConfigMapAfterStep(ns, callBuilderFactory.create().with($ -> { + new ConfigMapAfterStep(ns, configMapWatchers, stopping, Main::dispatchConfigMapWatch, + callBuilderFactory.create().with($ -> { $.labelSelector = LabelConstants.DOMAINUID_LABEL + "," + LabelConstants.CREATEDBYOPERATOR_LABEL; }).listPodAsync(ns, new ResponseStep(callBuilderFactory.create().with($ -> { $.fieldSelector = READINESS_PROBE_FAILURE_EVENT_FILTER; @@ -717,34 +718,6 @@ public void onThrowable(Packet packet, Throwable throwable) { LOGGER.exiting(); } - private static class DomainPrescenceStep extends Step { - private final Step managedServerStep; - - public DomainPrescenceStep(Step adminStep, Step managedServerStep) { - super(adminStep); - this.managedServerStep = managedServerStep; - } - - @Override - public NextAction apply(Packet packet) { - LOGGER.entering(); - DomainPresenceInfo info = packet.getSPI(DomainPresenceInfo.class); - - Domain dom = info.getDomain(); - DomainSpec spec = dom.getSpec(); - - String sc = spec.getStartupControl(); - if (sc == null || !StartupControlConstants.NONE_STARTUPCONTROL.equals(sc.toUpperCase())) { - LOGGER.exiting(); - return doNext(packet); - } - - LOGGER.exiting(); - // admin server will be stopped as part of scale down flow - return doNext(managedServerStep, packet); - } - } - // pre-conditions: DomainPresenceInfo SPI // "principal" private static Step bringAdminServerUp(Step next) { @@ -752,562 +725,15 @@ private static Step bringAdminServerUp(Step next) { PodHelper.createAdminPodStep(new BeforeAdminServiceStep(ServiceHelper.createForServerStep(next)))); } - private static class ListPersistentVolumeClaimStep extends Step { - public ListPersistentVolumeClaimStep(Step next) { - super(next); - } - - @Override - public NextAction apply(Packet packet) { - DomainPresenceInfo info = packet.getSPI(DomainPresenceInfo.class); - - Domain dom = info.getDomain(); - V1ObjectMeta meta = dom.getMetadata(); - DomainSpec spec = dom.getSpec(); - String namespace = meta.getNamespace(); - - String domainUID = spec.getDomainUID(); - - Step list = callBuilderFactory.create().with($ -> { - $.labelSelector = LabelConstants.DOMAINUID_LABEL + "=" + domainUID; - }).listPersistentVolumeClaimAsync(namespace, new ResponseStep(next) { - @Override - public NextAction onFailure(Packet packet, ApiException e, int statusCode, - Map> responseHeaders) { - if (statusCode == CallBuilder.NOT_FOUND) { - return onSuccess(packet, null, statusCode, responseHeaders); - } - return super.onFailure(packet, e, statusCode, responseHeaders); - } - - @Override - public NextAction onSuccess(Packet packet, V1PersistentVolumeClaimList result, int statusCode, - Map> responseHeaders) { - info.setClaims(result); - return doNext(packet); - } - }); - - return doNext(list, packet); - } - } - - private static class BeforeAdminServiceStep extends Step { - public BeforeAdminServiceStep(Step next) { - super(next); - } - - @Override - public NextAction apply(Packet packet) { - DomainPresenceInfo info = packet.getSPI(DomainPresenceInfo.class); - - Domain dom = info.getDomain(); - DomainSpec spec = dom.getSpec(); - - packet.put(ProcessingConstants.SERVER_NAME, spec.getAsName()); - packet.put(ProcessingConstants.PORT, spec.getAsPort()); - List ssl = spec.getServerStartup(); - if (ssl != null) { - for (ServerStartup ss : ssl) { - if (ss.getServerName().equals(spec.getAsName())) { - packet.put(ProcessingConstants.NODE_PORT, ss.getNodePort()); - break; - } - } - } - return doNext(packet); - } - } - private static Step connectToAdminAndInspectDomain(Step next) { - return new WatchPodReadyAdminStep(WlsRetriever.readConfigStep(new ExternalAdminChannelsStep(next))); - } - - private static class WatchPodReadyAdminStep extends Step { - public WatchPodReadyAdminStep(Step next) { - super(next); - } - - @Override - public NextAction apply(Packet packet) { - DomainPresenceInfo info = packet.getSPI(DomainPresenceInfo.class); - V1Pod adminPod = info.getAdmin().getPod().get(); - - PodWatcher pw = podWatchers.get(adminPod.getMetadata().getNamespace()); - packet.getComponents().put(ProcessingConstants.PODWATCHER_COMPONENT_NAME, Component.createFor(pw)); - - return doNext(pw.waitForReady(adminPod, next), packet); - } - } - - private static class ExternalAdminChannelsStep extends Step { - public ExternalAdminChannelsStep(Step next) { - super(next); - } - - @Override - public NextAction apply(Packet packet) { - DomainPresenceInfo info = packet.getSPI(DomainPresenceInfo.class); - - Collection validChannels = adminChannelsToCreate(info.getScan(), info.getDomain()); - if (validChannels != null && !validChannels.isEmpty()) { - return doNext(new ExternalAdminChannelIteratorStep(info, validChannels, next), packet); - } - - return doNext(packet); - } - } - - private static class ExternalAdminChannelIteratorStep extends Step { - private final DomainPresenceInfo info; - private final Iterator it; - - public ExternalAdminChannelIteratorStep(DomainPresenceInfo info, Collection naps, Step next) { - super(next); - this.info = info; - this.it = naps.iterator(); - } - - @Override - public NextAction apply(Packet packet) { - if (it.hasNext()) { - packet.put(ProcessingConstants.SERVER_NAME, info.getDomain().getSpec().getAsName()); - packet.put(ProcessingConstants.NETWORK_ACCESS_POINT, it.next()); - Step step = ServiceHelper.createForExternalChannelStep(this); - return doNext(step, packet); - } - return doNext(packet); - } + return new WatchPodReadyAdminStep(podWatchers, + WlsRetriever.readConfigStep(new ExternalAdminChannelsStep(next))); } private static Step bringManagedServersUp(Step next) { return new ManagedServersUpStep(next); } - private static class ManagedServersUpStep extends Step { - - public ManagedServersUpStep(Step next) { - super(next); - } - - @Override - public NextAction apply(Packet packet) { - LOGGER.entering(); - DomainPresenceInfo info = packet.getSPI(DomainPresenceInfo.class); - - Domain dom = info.getDomain(); - DomainSpec spec = dom.getSpec(); - - if (LOGGER.isFineEnabled()) { - Collection runningList = new ArrayList<>(); - for (Map.Entry entry : info.getServers().entrySet()) { - ServerKubernetesObjects sko = entry.getValue(); - if (sko != null && sko.getPod() != null) { - runningList.add(entry.getKey()); - } - } - LOGGER.fine("Running servers for domain with UID: " + spec.getDomainUID() + ", running list: " + runningList); - } - - String sc = spec.getStartupControl(); - if (sc == null) { - sc = StartupControlConstants.AUTO_STARTUPCONTROL; - } else { - sc = sc.toUpperCase(); - } - - WlsDomainConfig scan = info.getScan(); - Collection ssic = new ArrayList(); - - String asName = spec.getAsName(); - - boolean startAll = false; - Collection servers = new ArrayList(); - switch (sc) { - case StartupControlConstants.ALL_STARTUPCONTROL: - startAll = true; - case StartupControlConstants.AUTO_STARTUPCONTROL: - case StartupControlConstants.SPECIFIED_STARTUPCONTROL: - Collection clusters = new ArrayList(); - - // start specified servers with their custom options - List ssl = spec.getServerStartup(); - if (ssl != null) { - for (ServerStartup ss : ssl) { - String serverName = ss.getServerName(); - WlsServerConfig wlsServerConfig = scan.getServerConfig(serverName); - if (!serverName.equals(asName) && wlsServerConfig != null && !servers.contains(serverName)) { - // start server - servers.add(serverName); - // find cluster if this server is part of one - WlsClusterConfig cc = null; - find: for (WlsClusterConfig wlsClusterConfig : scan.getClusterConfigs().values()) { - for (WlsServerConfig clusterMemberServerConfig : wlsClusterConfig.getServerConfigs()) { - if (serverName.equals(clusterMemberServerConfig.getName())) { - cc = wlsClusterConfig; - break find; - } - } - } - List env = ss.getEnv(); - if (WebLogicConstants.ADMIN_STATE.equals(ss.getDesiredState())) { - env = startInAdminMode(env); - } - ssic.add(new ServerStartupInfo(wlsServerConfig, cc, env, ss)); - } - } - } - List lcs = spec.getClusterStartup(); - if (lcs != null) { - cluster: for (ClusterStartup cs : lcs) { - String clusterName = cs.getClusterName(); - clusters.add(clusterName); - int startedCount = 0; - // find cluster - WlsClusterConfig wlsClusterConfig = scan.getClusterConfig(clusterName); - if (wlsClusterConfig != null) { - for (WlsServerConfig wlsServerConfig : wlsClusterConfig.getServerConfigs()) { - // done with the current cluster - if (startedCount >= cs.getReplicas() && !startAll) - continue cluster; - - String serverName = wlsServerConfig.getName(); - if (!serverName.equals(asName) && !servers.contains(serverName)) { - List env = cs.getEnv(); - ServerStartup ssi = null; - ssl = spec.getServerStartup(); - if (ssl != null) { - for (ServerStartup ss : ssl) { - String s = ss.getServerName(); - if (serverName.equals(s)) { - env = ss.getEnv(); - ssi = ss; - break; - } - } - } - // start server - servers.add(serverName); - if (WebLogicConstants.ADMIN_STATE.equals(cs.getDesiredState())) { - env = startInAdminMode(env); - } - ssic.add(new ServerStartupInfo(wlsServerConfig, wlsClusterConfig, env, ssi)); - startedCount++; - } - } - } - } - } - if (startAll) { - // Look for any other servers - for (WlsClusterConfig wlsClusterConfig : scan.getClusterConfigs().values()) { - for (WlsServerConfig wlsServerConfig : wlsClusterConfig.getServerConfigs()) { - String serverName = wlsServerConfig.getListenAddress(); - // do not start admin server - if (!serverName.equals(asName) && !servers.contains(serverName)) { - // start server - servers.add(serverName); - ssic.add(new ServerStartupInfo(wlsServerConfig, wlsClusterConfig, null, null)); - } - } - } - for (Map.Entry wlsServerConfig : scan.getServerConfigs().entrySet()) { - String serverName = wlsServerConfig.getKey(); - // do not start admin server - if (!serverName.equals(asName) && !servers.contains(serverName)) { - // start server - servers.add(serverName); - ssic.add(new ServerStartupInfo(wlsServerConfig.getValue(), null, null, null)); - } - } - } else if (StartupControlConstants.AUTO_STARTUPCONTROL.equals(sc)) { - for (Map.Entry wlsClusterConfig : scan.getClusterConfigs().entrySet()) { - if (!clusters.contains(wlsClusterConfig.getKey())) { - int startedCount = 0; - WlsClusterConfig config = wlsClusterConfig.getValue(); - for (WlsServerConfig wlsServerConfig : config.getServerConfigs()) { - if (startedCount >= spec.getReplicas()) - break; - String serverName = wlsServerConfig.getName(); - if (!serverName.equals(asName) && !servers.contains(serverName)) { - // start server - servers.add(serverName); - ssic.add(new ServerStartupInfo(wlsServerConfig, config, null, null)); - startedCount++; - } - } - } - } - } - - info.setServerStartupInfo(ssic); - LOGGER.exiting(); - return doNext(scaleDownIfNecessary(info, servers, - new ClusterServicesStep(info, new ManagedServerUpIteratorStep(ssic, next))), packet); - case StartupControlConstants.ADMIN_STARTUPCONTROL: - case StartupControlConstants.NONE_STARTUPCONTROL: - default: - - info.setServerStartupInfo(null); - LOGGER.exiting(); - return doNext(scaleDownIfNecessary(info, servers, new ClusterServicesStep(info, next)), packet); - } - } - } - - private static List startInAdminMode(List env) { - if (env == null) { - env = new ArrayList<>(); - } - - // look for JAVA_OPTIONS - V1EnvVar jo = null; - for (V1EnvVar e : env) { - if ("JAVA_OPTIONS".equals(e.getName())) { - jo = e; - if (jo.getValueFrom() != null) { - throw new IllegalStateException(); - } - break; - } - } - if (jo == null) { - jo = new V1EnvVar(); - jo.setName("JAVA_OPTIONS"); - env.add(jo); - } - - // create or update value - String startInAdmin = "-Dweblogic.management.startupMode=ADMIN"; - String value = jo.getValue(); - value = (value != null) ? (startInAdmin + " " + value) : startInAdmin; - jo.setValue(value); - - return env; - } - - private static class ClusterServicesStep extends Step { - private final DomainPresenceInfo info; - - public ClusterServicesStep(DomainPresenceInfo info, Step next) { - super(next); - this.info = info; - } - - @Override - public NextAction apply(Packet packet) { - Collection startDetails = new ArrayList<>(); - - // Add cluster services - WlsDomainConfig scan = info.getScan(); - if (scan != null) { - for (Map.Entry entry : scan.getClusterConfigs().entrySet()) { - Packet p = packet.clone(); - WlsClusterConfig clusterConfig = entry.getValue(); - p.put(ProcessingConstants.CLUSTER_SCAN, clusterConfig); - p.put(ProcessingConstants.CLUSTER_NAME, clusterConfig.getClusterName()); - for (WlsServerConfig serverConfig : clusterConfig.getServerConfigs()) { - p.put(ProcessingConstants.PORT, serverConfig.getListenPort()); - break; - } - - startDetails - .add(new StepAndPacket(ServiceHelper.createForClusterStep(IngressHelper.createClusterStep(null)), p)); - } - } - - if (startDetails.isEmpty()) { - return doNext(packet); - } - return doForkJoin(next, packet, startDetails); - } - } - - private static Step scaleDownIfNecessary(DomainPresenceInfo info, Collection servers, Step next) { - Domain dom = info.getDomain(); - DomainSpec spec = dom.getSpec(); - - boolean shouldStopAdmin = false; - String sc = spec.getStartupControl(); - if (sc != null && StartupControlConstants.NONE_STARTUPCONTROL.equals(sc.toUpperCase())) { - shouldStopAdmin = true; - next = DomainStatusUpdater.createAvailableStep(DomainStatusUpdater.ALL_STOPPED_AVAILABLE_REASON, next); - } - - String adminName = spec.getAsName(); - Map currentServers = info.getServers(); - Collection> serversToStop = new ArrayList<>(); - for (Map.Entry entry : currentServers.entrySet()) { - if ((shouldStopAdmin || !entry.getKey().equals(adminName)) && !servers.contains(entry.getKey())) { - serversToStop.add(entry); - } - } - - if (!serversToStop.isEmpty()) { - return new ServerDownIteratorStep(serversToStop, next); - } - - return next; - } - - private static class ManagedServerUpIteratorStep extends Step { - private final Collection c; - - public ManagedServerUpIteratorStep(Collection c, Step next) { - super(next); - this.c = c; - } - - @Override - public NextAction apply(Packet packet) { - Collection startDetails = new ArrayList<>(); - Map rolling = new ConcurrentHashMap<>(); - packet.put(ProcessingConstants.SERVERS_TO_ROLL, rolling); - - for (ServerStartupInfo ssi : c) { - Packet p = packet.clone(); - p.put(ProcessingConstants.SERVER_SCAN, ssi.serverConfig); - p.put(ProcessingConstants.CLUSTER_SCAN, ssi.clusterConfig); - p.put(ProcessingConstants.ENVVARS, ssi.envVars); - - p.put(ProcessingConstants.SERVER_NAME, ssi.serverConfig.getName()); - p.put(ProcessingConstants.PORT, ssi.serverConfig.getListenPort()); - ServerStartup ss = ssi.serverStartup; - p.put(ProcessingConstants.NODE_PORT, ss != null ? ss.getNodePort() : null); - - startDetails.add(new StepAndPacket(bringManagedServerUp(ssi, null), p)); - } - - if (LOGGER.isFineEnabled()) { - DomainPresenceInfo info = packet.getSPI(DomainPresenceInfo.class); - - Domain dom = info.getDomain(); - DomainSpec spec = dom.getSpec(); - - Collection serverList = new ArrayList<>(); - for (ServerStartupInfo ssi : c) { - serverList.add(ssi.serverConfig.getName()); - } - LOGGER.fine("Starting or validating servers for domain with UID: " + spec.getDomainUID() + ", server list: " - + serverList); - } - - if (startDetails.isEmpty()) { - return doNext(packet); - } - return doForkJoin(new ManagedServerUpAfterStep(next), packet, startDetails); - } - } - - private static class ManagedServerUpAfterStep extends Step { - public ManagedServerUpAfterStep(Step next) { - super(next); - } - - @Override - public NextAction apply(Packet packet) { - @SuppressWarnings("unchecked") - Map rolling = (Map) packet.get(ProcessingConstants.SERVERS_TO_ROLL); - - if (LOGGER.isFineEnabled()) { - DomainPresenceInfo info = packet.getSPI(DomainPresenceInfo.class); - - Domain dom = info.getDomain(); - DomainSpec spec = dom.getSpec(); - - Collection rollingList = Collections.emptyList(); - if (rolling != null) { - rollingList = rolling.keySet(); - } - LOGGER.fine("Rolling servers for domain with UID: " + spec.getDomainUID() + ", rolling list: " + rollingList); - } - - if (rolling == null || rolling.isEmpty()) { - return doNext(packet); - } - - return doNext(RollingHelper.rollServers(rolling, next), packet); - } - } - - private static class ServerDownIteratorStep extends Step { - private final Collection> c; - - public ServerDownIteratorStep(Collection> serversToStop, Step next) { - super(next); - this.c = serversToStop; - } - - @Override - public NextAction apply(Packet packet) { - Collection startDetails = new ArrayList<>(); - - for (Map.Entry entry : c) { - startDetails.add(new StepAndPacket(new ServerDownStep(entry.getKey(), entry.getValue(), null), packet.clone())); - } - - if (LOGGER.isFineEnabled()) { - DomainPresenceInfo info = packet.getSPI(DomainPresenceInfo.class); - - Domain dom = info.getDomain(); - DomainSpec spec = dom.getSpec(); - - Collection stopList = new ArrayList<>(); - for (Map.Entry entry : c) { - stopList.add(entry.getKey()); - } - LOGGER.fine("Stopping servers for domain with UID: " + spec.getDomainUID() + ", stop list: " + stopList); - } - - if (startDetails.isEmpty()) { - return doNext(packet); - } - return doForkJoin(next, packet, startDetails); - } - } - - private static class ServerDownStep extends Step { - private final String serverName; - private final ServerKubernetesObjects sko; - - public ServerDownStep(String serverName, ServerKubernetesObjects sko, Step next) { - super(next); - this.serverName = serverName; - this.sko = sko; - } - - @Override - public NextAction apply(Packet packet) { - return doNext(PodHelper.deletePodStep(sko, - ServiceHelper.deleteServiceStep(sko, new ServerDownFinalizeStep(serverName, next))), packet); - } - } - - private static class ServerDownFinalizeStep extends Step { - private final String serverName; - - public ServerDownFinalizeStep(String serverName, Step next) { - super(next); - this.serverName = serverName; - } - - @Override - public NextAction apply(Packet packet) { - DomainPresenceInfo info = packet.getSPI(DomainPresenceInfo.class); - info.getServers().remove(serverName); - return doNext(next, packet); - } - } - - // pre-conditions: DomainPresenceInfo SPI - // "principal" - // "serverScan" - // "clusterScan" - // "envVars" - private static Step bringManagedServerUp(ServerStartupInfo ssi, Step next) { - return PodHelper.createManagedPodStep(ServiceHelper.createForServerStep(next)); - } - private static void deleteDomainPresence(Domain dom) { V1ObjectMeta meta = dom.getMetadata(); DomainSpec spec = dom.getSpec(); @@ -1341,166 +767,6 @@ public void onThrowable(Packet packet, Throwable throwable) { LOGGER.exiting(); } - private static class DeleteDomainStep extends Step { - private final String namespace; - private final String domainUID; - - public DeleteDomainStep(String namespace, String domainUID) { - super(null); - this.namespace = namespace; - this.domainUID = domainUID; - } - - @Override - public NextAction apply(Packet packet) { - Step deletePods = callBuilderFactory.create().with($ -> { - $.labelSelector = LabelConstants.DOMAINUID_LABEL + "=" + domainUID + "," - + LabelConstants.CREATEDBYOPERATOR_LABEL; - }).deleteCollectionPodAsync(namespace, new ResponseStep(next) { - @Override - public NextAction onFailure(Packet packet, ApiException e, int statusCode, - Map> responseHeaders) { - if (statusCode == CallBuilder.NOT_FOUND) { - return onSuccess(packet, null, statusCode, responseHeaders); - } - return super.onFailure(packet, e, statusCode, responseHeaders); - } - - @Override - public NextAction onSuccess(Packet packet, V1Status result, int statusCode, - Map> responseHeaders) { - return doNext(packet); - } - }); - - Step serviceList = callBuilderFactory.create().with($ -> { - $.labelSelector = LabelConstants.DOMAINUID_LABEL + "=" + domainUID + "," - + LabelConstants.CREATEDBYOPERATOR_LABEL; - }).listServiceAsync(namespace, new ResponseStep(deletePods) { - @Override - public NextAction onFailure(Packet packet, ApiException e, int statusCode, - Map> responseHeaders) { - if (statusCode == CallBuilder.NOT_FOUND) { - return onSuccess(packet, null, statusCode, responseHeaders); - } - return super.onFailure(packet, e, statusCode, responseHeaders); - } - - @Override - public NextAction onSuccess(Packet packet, V1ServiceList result, int statusCode, - Map> responseHeaders) { - if (result != null) { - return doNext(new DeleteServiceListStep(result.getItems(), deletePods), packet); - } - return doNext(packet); - } - }); - - LOGGER.finer(MessageKeys.LIST_INGRESS_FOR_DOMAIN, domainUID, namespace); - Step deleteIngress = callBuilderFactory.create().with($ -> { - $.labelSelector = LabelConstants.DOMAINUID_LABEL + "=" + domainUID + "," - + LabelConstants.CREATEDBYOPERATOR_LABEL; - }).listIngressAsync(namespace, new ResponseStep(serviceList) { - @Override - public NextAction onFailure(Packet packet, ApiException e, int statusCode, - Map> responseHeaders) { - if (statusCode == CallBuilder.NOT_FOUND) { - return onSuccess(packet, null, statusCode, responseHeaders); - } - return super.onFailure(packet, e, statusCode, responseHeaders); - } - - @Override - public NextAction onSuccess(Packet packet, V1beta1IngressList result, int statusCode, - Map> responseHeaders) { - if (result != null) { - - return doNext(new DeleteIngressListStep(result.getItems(), serviceList), packet); - } - return doNext(packet); - } - }); - - return doNext(deleteIngress, packet); - } - } - - private static class DeleteServiceListStep extends Step { - private final Iterator it; - - public DeleteServiceListStep(Collection c, Step next) { - super(next); - this.it = c.iterator(); - } - - @Override - public NextAction apply(Packet packet) { - if (it.hasNext()) { - V1Service service = it.next(); - V1ObjectMeta meta = service.getMetadata(); - Step delete = callBuilderFactory.create().deleteServiceAsync(meta.getName(), meta.getNamespace(), - new ResponseStep(this) { - @Override - public NextAction onFailure(Packet packet, ApiException e, int statusCode, - Map> responseHeaders) { - if (statusCode == CallBuilder.NOT_FOUND) { - return onSuccess(packet, null, statusCode, responseHeaders); - } - return super.onFailure(packet, e, statusCode, responseHeaders); - } - - @Override - public NextAction onSuccess(Packet packet, V1Status result, int statusCode, - Map> responseHeaders) { - return doNext(packet); - } - }); - return doNext(delete, packet); - } - - return doNext(packet); - } - } - - private static class DeleteIngressListStep extends Step { - private final Iterator it; - - public DeleteIngressListStep(Collection c, Step next) { - super(next); - this.it = c.iterator(); - } - - @Override - public NextAction apply(Packet packet) { - if (it.hasNext()) { - V1beta1Ingress v1beta1Ingress = it.next(); - V1ObjectMeta meta = v1beta1Ingress.getMetadata(); - String ingressName = meta.getName(); - String namespace = meta.getNamespace(); - LOGGER.finer(MessageKeys.REMOVING_INGRESS, ingressName, namespace); - Step delete = callBuilderFactory.create().deleteIngressAsync(ingressName, meta.getNamespace(), - new V1DeleteOptions(), new ResponseStep(this) { - @Override - public NextAction onFailure(Packet packet, ApiException e, int statusCode, - Map> responseHeaders) { - if (statusCode == CallBuilder.NOT_FOUND) { - return onSuccess(packet, null, statusCode, responseHeaders); - } - return super.onFailure(packet, e, statusCode, responseHeaders); - } - - @Override - public NextAction onSuccess(Packet packet, V1Status result, int statusCode, - Map> responseHeaders) { - return doNext(packet); - } - }); - return doNext(delete, packet); - } - return doNext(packet); - } - } - /** * Obtain the list of target namespaces * @@ -1775,27 +1041,6 @@ private static void dispatchIngressWatch(Watch.Response item) { } } - private static class ConfigMapAfterStep extends Step { - private final String ns; - - public ConfigMapAfterStep(String ns, Step next) { - super(next); - this.ns = ns; - } - - @Override - public NextAction apply(Packet packet) { - V1ConfigMap result = (V1ConfigMap) packet.get(ProcessingConstants.SCRIPT_CONFIG_MAP); - configMapWatchers.put(ns, - createConfigMapWatcher(ns, result != null ? result.getMetadata().getResourceVersion() : "")); - return doNext(packet); - } - } - - private static ConfigMapWatcher createConfigMapWatcher(String namespace, String initialResourceVersion) { - return ConfigMapWatcher.create(factory, namespace, initialResourceVersion, Main::dispatchConfigMapWatch, stopping); - } - private static void dispatchConfigMapWatch(Watch.Response item) { V1ConfigMap c = item.object; if (c != null) { @@ -1854,83 +1099,6 @@ private static void dispatchDomainWatch(Watch.Response item) { } } - /** - * This method checks the domain spec against any configured network access - * points defined for the domain. This implementation only handles T3 protocol - * for externalization. - * - * @param scan - * WlsDomainConfig from discovery containing configuration - * @param dom - * Domain containing Domain resource information - * @return Validated collection of network access points - */ - public static Collection adminChannelsToCreate(WlsDomainConfig scan, Domain dom) { - LOGGER.entering(); - - // The following hard-coded values for the nodePort min/max ranges are - // provided here until the appropriate API is discovered to obtain - // this information from Kubernetes. - Integer nodePortMin = 30000; - Integer nodePortMax = 32767; - - DomainSpec spec = dom.getSpec(); - if (spec.getExportT3Channels() == null) { - return null; - } - - WlsServerConfig adminServerConfig = scan.getServerConfig(spec.getAsName()); - - List naps = adminServerConfig.getNetworkAccessPoints(); - // This will become a list of valid channels to create services for. - Collection channels = new ArrayList<>(); - - // Pick out externalized channels from the server channels list - for (String incomingChannel : spec.getExportT3Channels()) { - boolean missingChannel = true; - for (NetworkAccessPoint nap : naps) { - if (nap.getName().equalsIgnoreCase(incomingChannel)) { - missingChannel = false; - channels.add(nap); - break; - } - } - if (missingChannel) { - LOGGER.warning(MessageKeys.EXCH_CHANNEL_NOT_DEFINED, incomingChannel, spec.getAsName()); - } - } - - // Iterate through the selected channels and validate - Collection validatedChannels = new ArrayList<>(); - for (NetworkAccessPoint nap : channels) { - - // Only supporting T3 for now. - if (!nap.getProtocol().equalsIgnoreCase("t3")) { - LOGGER.severe(MessageKeys.EXCH_WRONG_PROTOCOL, nap.getName(), nap.getProtocol()); - continue; - } - - // Until otherwise determined, ports must be the same. - if (!nap.getListenPort().equals(nap.getPublicPort())) { - // log a warning and ignore this item. - LOGGER.warning(MessageKeys.EXCH_UNEQUAL_LISTEN_PORTS, nap.getName()); - continue; - } - - // Make sure configured port is within NodePort range. - if (nap.getListenPort().compareTo(nodePortMin) < 0 || nap.getListenPort().compareTo(nodePortMax) > 0) { - // port setting is outside the NodePort range limits - LOGGER.warning(MessageKeys.EXCH_OUTSIDE_RANGE, nap.getName(), nap.getPublicPort(), nodePortMin, nodePortMax); - continue; - } - - validatedChannels.add(nap); - } - - LOGGER.exiting(); - return validatedChannels; - } - private static String getOperatorNamespace() { String namespace = System.getenv("OPERATOR_NAMESPACE"); if (namespace == null) { diff --git a/operator/src/main/java/oracle/kubernetes/operator/steps/BeforeAdminServiceStep.java b/operator/src/main/java/oracle/kubernetes/operator/steps/BeforeAdminServiceStep.java new file mode 100644 index 00000000000..09e7d9d7c05 --- /dev/null +++ b/operator/src/main/java/oracle/kubernetes/operator/steps/BeforeAdminServiceStep.java @@ -0,0 +1,42 @@ +// Copyright 2017, 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.steps; + +import java.util.List; + +import oracle.kubernetes.operator.ProcessingConstants; +import oracle.kubernetes.operator.helpers.DomainPresenceInfo; +import oracle.kubernetes.operator.work.NextAction; +import oracle.kubernetes.operator.work.Packet; +import oracle.kubernetes.operator.work.Step; +import oracle.kubernetes.weblogic.domain.v1.Domain; +import oracle.kubernetes.weblogic.domain.v1.DomainSpec; +import oracle.kubernetes.weblogic.domain.v1.ServerStartup; + +public class BeforeAdminServiceStep extends Step { + public BeforeAdminServiceStep(Step next) { + super(next); + } + + @Override + public NextAction apply(Packet packet) { + DomainPresenceInfo info = packet.getSPI(DomainPresenceInfo.class); + + Domain dom = info.getDomain(); + DomainSpec spec = dom.getSpec(); + + packet.put(ProcessingConstants.SERVER_NAME, spec.getAsName()); + packet.put(ProcessingConstants.PORT, spec.getAsPort()); + List ssl = spec.getServerStartup(); + if (ssl != null) { + for (ServerStartup ss : ssl) { + if (ss.getServerName().equals(spec.getAsName())) { + packet.put(ProcessingConstants.NODE_PORT, ss.getNodePort()); + break; + } + } + } + return doNext(packet); + } +} \ No newline at end of file diff --git a/operator/src/main/java/oracle/kubernetes/operator/steps/ClusterServicesStep.java b/operator/src/main/java/oracle/kubernetes/operator/steps/ClusterServicesStep.java new file mode 100644 index 00000000000..bf7a99a4217 --- /dev/null +++ b/operator/src/main/java/oracle/kubernetes/operator/steps/ClusterServicesStep.java @@ -0,0 +1,56 @@ +// Copyright 2017, 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.steps; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Map; + +import oracle.kubernetes.operator.ProcessingConstants; +import oracle.kubernetes.operator.helpers.DomainPresenceInfo; +import oracle.kubernetes.operator.helpers.IngressHelper; +import oracle.kubernetes.operator.helpers.ServiceHelper; +import oracle.kubernetes.operator.wlsconfig.WlsClusterConfig; +import oracle.kubernetes.operator.wlsconfig.WlsDomainConfig; +import oracle.kubernetes.operator.wlsconfig.WlsServerConfig; +import oracle.kubernetes.operator.work.NextAction; +import oracle.kubernetes.operator.work.Packet; +import oracle.kubernetes.operator.work.Step; + +public class ClusterServicesStep extends Step { + private final DomainPresenceInfo info; + + public ClusterServicesStep(DomainPresenceInfo info, Step next) { + super(next); + this.info = info; + } + + @Override + public NextAction apply(Packet packet) { + Collection startDetails = new ArrayList<>(); + + // Add cluster services + WlsDomainConfig scan = info.getScan(); + if (scan != null) { + for (Map.Entry entry : scan.getClusterConfigs().entrySet()) { + Packet p = packet.clone(); + WlsClusterConfig clusterConfig = entry.getValue(); + p.put(ProcessingConstants.CLUSTER_SCAN, clusterConfig); + p.put(ProcessingConstants.CLUSTER_NAME, clusterConfig.getClusterName()); + for (WlsServerConfig serverConfig : clusterConfig.getServerConfigs()) { + p.put(ProcessingConstants.PORT, serverConfig.getListenPort()); + break; + } + + startDetails + .add(new StepAndPacket(ServiceHelper.createForClusterStep(IngressHelper.createClusterStep(null)), p)); + } + } + + if (startDetails.isEmpty()) { + return doNext(packet); + } + return doForkJoin(next, packet, startDetails); + } +} \ No newline at end of file diff --git a/operator/src/main/java/oracle/kubernetes/operator/steps/ConfigMapAfterStep.java b/operator/src/main/java/oracle/kubernetes/operator/steps/ConfigMapAfterStep.java new file mode 100644 index 00000000000..d2c13e618bb --- /dev/null +++ b/operator/src/main/java/oracle/kubernetes/operator/steps/ConfigMapAfterStep.java @@ -0,0 +1,47 @@ +// Copyright 2017, 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.steps; + +import java.util.Map; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicBoolean; + +import io.kubernetes.client.models.V1ConfigMap; +import oracle.kubernetes.operator.ConfigMapWatcher; +import oracle.kubernetes.operator.ProcessingConstants; +import oracle.kubernetes.operator.watcher.WatchListener; +import oracle.kubernetes.operator.work.ContainerResolver; +import oracle.kubernetes.operator.work.NextAction; +import oracle.kubernetes.operator.work.Packet; +import oracle.kubernetes.operator.work.Step; + +public class ConfigMapAfterStep extends Step { + private final String ns; + private final Map configMapWatchers; + private final AtomicBoolean stopping; + private final WatchListener listener; + + public ConfigMapAfterStep(String ns, Map configMapWatchers, + AtomicBoolean stopping, WatchListener listener, Step next) { + super(next); + this.ns = ns; + this.configMapWatchers = configMapWatchers; + this.stopping = stopping; + this.listener = listener; + } + + @Override + public NextAction apply(Packet packet) { + V1ConfigMap result = (V1ConfigMap) packet.get(ProcessingConstants.SCRIPT_CONFIG_MAP); + configMapWatchers.put(ns, + createConfigMapWatcher(ns, result != null ? result.getMetadata().getResourceVersion() : "")); + return doNext(packet); + } + + private ConfigMapWatcher createConfigMapWatcher(String namespace, String initialResourceVersion) { + ThreadFactory factory = ContainerResolver.getInstance().getContainer().getSPI(ThreadFactory.class); + + return ConfigMapWatcher.create(factory, namespace, initialResourceVersion, listener, stopping); + } +} \ No newline at end of file diff --git a/operator/src/main/java/oracle/kubernetes/operator/steps/DeleteDomainStep.java b/operator/src/main/java/oracle/kubernetes/operator/steps/DeleteDomainStep.java new file mode 100644 index 00000000000..35e79ab1b51 --- /dev/null +++ b/operator/src/main/java/oracle/kubernetes/operator/steps/DeleteDomainStep.java @@ -0,0 +1,111 @@ +// Copyright 2017, 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.steps; + +import java.util.List; +import java.util.Map; + +import io.kubernetes.client.ApiException; +import io.kubernetes.client.models.V1ServiceList; +import io.kubernetes.client.models.V1Status; +import io.kubernetes.client.models.V1beta1IngressList; +import oracle.kubernetes.operator.LabelConstants; +import oracle.kubernetes.operator.helpers.CallBuilder; +import oracle.kubernetes.operator.helpers.CallBuilderFactory; +import oracle.kubernetes.operator.helpers.ResponseStep; +import oracle.kubernetes.operator.logging.LoggingFacade; +import oracle.kubernetes.operator.logging.LoggingFactory; +import oracle.kubernetes.operator.logging.MessageKeys; +import oracle.kubernetes.operator.work.ContainerResolver; +import oracle.kubernetes.operator.work.NextAction; +import oracle.kubernetes.operator.work.Packet; +import oracle.kubernetes.operator.work.Step; + +public class DeleteDomainStep extends Step { + private static final LoggingFacade LOGGER = LoggingFactory.getLogger("Operator", "Operator"); + + private final String namespace; + private final String domainUID; + + public DeleteDomainStep(String namespace, String domainUID) { + super(null); + this.namespace = namespace; + this.domainUID = domainUID; + } + + @Override + public NextAction apply(Packet packet) { + CallBuilderFactory factory = ContainerResolver.getInstance().getContainer().getSPI(CallBuilderFactory.class); + + Step deletePods = factory.create().with($ -> { + $.labelSelector = LabelConstants.DOMAINUID_LABEL + "=" + domainUID + "," + + LabelConstants.CREATEDBYOPERATOR_LABEL; + }).deleteCollectionPodAsync(namespace, new ResponseStep(next) { + @Override + public NextAction onFailure(Packet packet, ApiException e, int statusCode, + Map> responseHeaders) { + if (statusCode == CallBuilder.NOT_FOUND) { + return onSuccess(packet, null, statusCode, responseHeaders); + } + return super.onFailure(packet, e, statusCode, responseHeaders); + } + + @Override + public NextAction onSuccess(Packet packet, V1Status result, int statusCode, + Map> responseHeaders) { + return doNext(packet); + } + }); + + Step serviceList = factory.create().with($ -> { + $.labelSelector = LabelConstants.DOMAINUID_LABEL + "=" + domainUID + "," + + LabelConstants.CREATEDBYOPERATOR_LABEL; + }).listServiceAsync(namespace, new ResponseStep(deletePods) { + @Override + public NextAction onFailure(Packet packet, ApiException e, int statusCode, + Map> responseHeaders) { + if (statusCode == CallBuilder.NOT_FOUND) { + return onSuccess(packet, null, statusCode, responseHeaders); + } + return super.onFailure(packet, e, statusCode, responseHeaders); + } + + @Override + public NextAction onSuccess(Packet packet, V1ServiceList result, int statusCode, + Map> responseHeaders) { + if (result != null) { + return doNext(new DeleteServiceListStep(result.getItems(), deletePods), packet); + } + return doNext(packet); + } + }); + + LOGGER.finer(MessageKeys.LIST_INGRESS_FOR_DOMAIN, domainUID, namespace); + Step deleteIngress = factory.create().with($ -> { + $.labelSelector = LabelConstants.DOMAINUID_LABEL + "=" + domainUID + "," + + LabelConstants.CREATEDBYOPERATOR_LABEL; + }).listIngressAsync(namespace, new ResponseStep(serviceList) { + @Override + public NextAction onFailure(Packet packet, ApiException e, int statusCode, + Map> responseHeaders) { + if (statusCode == CallBuilder.NOT_FOUND) { + return onSuccess(packet, null, statusCode, responseHeaders); + } + return super.onFailure(packet, e, statusCode, responseHeaders); + } + + @Override + public NextAction onSuccess(Packet packet, V1beta1IngressList result, int statusCode, + Map> responseHeaders) { + if (result != null) { + + return doNext(new DeleteIngressListStep(result.getItems(), serviceList), packet); + } + return doNext(packet); + } + }); + + return doNext(deleteIngress, packet); + } +} \ No newline at end of file diff --git a/operator/src/main/java/oracle/kubernetes/operator/steps/DeleteIngressListStep.java b/operator/src/main/java/oracle/kubernetes/operator/steps/DeleteIngressListStep.java new file mode 100644 index 00000000000..9a5cc8e713d --- /dev/null +++ b/operator/src/main/java/oracle/kubernetes/operator/steps/DeleteIngressListStep.java @@ -0,0 +1,68 @@ +// Copyright 2017, 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.steps; + +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import io.kubernetes.client.ApiException; +import io.kubernetes.client.models.V1DeleteOptions; +import io.kubernetes.client.models.V1ObjectMeta; +import io.kubernetes.client.models.V1Status; +import io.kubernetes.client.models.V1beta1Ingress; +import oracle.kubernetes.operator.helpers.CallBuilder; +import oracle.kubernetes.operator.helpers.CallBuilderFactory; +import oracle.kubernetes.operator.helpers.ResponseStep; +import oracle.kubernetes.operator.logging.LoggingFacade; +import oracle.kubernetes.operator.logging.LoggingFactory; +import oracle.kubernetes.operator.logging.MessageKeys; +import oracle.kubernetes.operator.work.ContainerResolver; +import oracle.kubernetes.operator.work.NextAction; +import oracle.kubernetes.operator.work.Packet; +import oracle.kubernetes.operator.work.Step; + +public class DeleteIngressListStep extends Step { + private static final LoggingFacade LOGGER = LoggingFactory.getLogger("Operator", "Operator"); + + private final Iterator it; + + public DeleteIngressListStep(Collection c, Step next) { + super(next); + this.it = c.iterator(); + } + + @Override + public NextAction apply(Packet packet) { + CallBuilderFactory factory = ContainerResolver.getInstance().getContainer().getSPI(CallBuilderFactory.class); + + if (it.hasNext()) { + V1beta1Ingress v1beta1Ingress = it.next(); + V1ObjectMeta meta = v1beta1Ingress.getMetadata(); + String ingressName = meta.getName(); + String namespace = meta.getNamespace(); + LOGGER.finer(MessageKeys.REMOVING_INGRESS, ingressName, namespace); + Step delete = factory.create().deleteIngressAsync(ingressName, meta.getNamespace(), + new V1DeleteOptions(), new ResponseStep(this) { + @Override + public NextAction onFailure(Packet packet, ApiException e, int statusCode, + Map> responseHeaders) { + if (statusCode == CallBuilder.NOT_FOUND) { + return onSuccess(packet, null, statusCode, responseHeaders); + } + return super.onFailure(packet, e, statusCode, responseHeaders); + } + + @Override + public NextAction onSuccess(Packet packet, V1Status result, int statusCode, + Map> responseHeaders) { + return doNext(packet); + } + }); + return doNext(delete, packet); + } + return doNext(packet); + } +} \ No newline at end of file diff --git a/operator/src/main/java/oracle/kubernetes/operator/steps/DeleteServiceListStep.java b/operator/src/main/java/oracle/kubernetes/operator/steps/DeleteServiceListStep.java new file mode 100644 index 00000000000..7471b5348a1 --- /dev/null +++ b/operator/src/main/java/oracle/kubernetes/operator/steps/DeleteServiceListStep.java @@ -0,0 +1,60 @@ +// Copyright 2017, 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.steps; + +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import io.kubernetes.client.ApiException; +import io.kubernetes.client.models.V1ObjectMeta; +import io.kubernetes.client.models.V1Service; +import io.kubernetes.client.models.V1Status; +import oracle.kubernetes.operator.helpers.CallBuilder; +import oracle.kubernetes.operator.helpers.CallBuilderFactory; +import oracle.kubernetes.operator.helpers.ResponseStep; +import oracle.kubernetes.operator.work.ContainerResolver; +import oracle.kubernetes.operator.work.NextAction; +import oracle.kubernetes.operator.work.Packet; +import oracle.kubernetes.operator.work.Step; + +public class DeleteServiceListStep extends Step { + private final Iterator it; + + public DeleteServiceListStep(Collection c, Step next) { + super(next); + this.it = c.iterator(); + } + + @Override + public NextAction apply(Packet packet) { + CallBuilderFactory factory = ContainerResolver.getInstance().getContainer().getSPI(CallBuilderFactory.class); + + if (it.hasNext()) { + V1Service service = it.next(); + V1ObjectMeta meta = service.getMetadata(); + Step delete = factory.create().deleteServiceAsync(meta.getName(), meta.getNamespace(), + new ResponseStep(this) { + @Override + public NextAction onFailure(Packet packet, ApiException e, int statusCode, + Map> responseHeaders) { + if (statusCode == CallBuilder.NOT_FOUND) { + return onSuccess(packet, null, statusCode, responseHeaders); + } + return super.onFailure(packet, e, statusCode, responseHeaders); + } + + @Override + public NextAction onSuccess(Packet packet, V1Status result, int statusCode, + Map> responseHeaders) { + return doNext(packet); + } + }); + return doNext(delete, packet); + } + + return doNext(packet); + } +} \ No newline at end of file diff --git a/operator/src/main/java/oracle/kubernetes/operator/steps/DomainPrescenceStep.java b/operator/src/main/java/oracle/kubernetes/operator/steps/DomainPrescenceStep.java new file mode 100644 index 00000000000..8a08103047d --- /dev/null +++ b/operator/src/main/java/oracle/kubernetes/operator/steps/DomainPrescenceStep.java @@ -0,0 +1,44 @@ +// Copyright 2017, 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.steps; + +import oracle.kubernetes.operator.StartupControlConstants; +import oracle.kubernetes.operator.helpers.DomainPresenceInfo; +import oracle.kubernetes.operator.logging.LoggingFacade; +import oracle.kubernetes.operator.logging.LoggingFactory; +import oracle.kubernetes.operator.work.NextAction; +import oracle.kubernetes.operator.work.Packet; +import oracle.kubernetes.operator.work.Step; +import oracle.kubernetes.weblogic.domain.v1.Domain; +import oracle.kubernetes.weblogic.domain.v1.DomainSpec; + +public class DomainPrescenceStep extends Step { + private static final LoggingFacade LOGGER = LoggingFactory.getLogger("Operator", "Operator"); + + private final Step managedServerStep; + + public DomainPrescenceStep(Step adminStep, Step managedServerStep) { + super(adminStep); + this.managedServerStep = managedServerStep; + } + + @Override + public NextAction apply(Packet packet) { + LOGGER.entering(); + DomainPresenceInfo info = packet.getSPI(DomainPresenceInfo.class); + + Domain dom = info.getDomain(); + DomainSpec spec = dom.getSpec(); + + String sc = spec.getStartupControl(); + if (sc == null || !StartupControlConstants.NONE_STARTUPCONTROL.equals(sc.toUpperCase())) { + LOGGER.exiting(); + return doNext(packet); + } + + LOGGER.exiting(); + // admin server will be stopped as part of scale down flow + return doNext(managedServerStep, packet); + } +} \ No newline at end of file diff --git a/operator/src/main/java/oracle/kubernetes/operator/steps/ExternalAdminChannelIteratorStep.java b/operator/src/main/java/oracle/kubernetes/operator/steps/ExternalAdminChannelIteratorStep.java new file mode 100644 index 00000000000..78200f4a26c --- /dev/null +++ b/operator/src/main/java/oracle/kubernetes/operator/steps/ExternalAdminChannelIteratorStep.java @@ -0,0 +1,37 @@ +// Copyright 2017, 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.steps; + +import java.util.Collection; +import java.util.Iterator; + +import oracle.kubernetes.operator.ProcessingConstants; +import oracle.kubernetes.operator.helpers.DomainPresenceInfo; +import oracle.kubernetes.operator.helpers.ServiceHelper; +import oracle.kubernetes.operator.wlsconfig.NetworkAccessPoint; +import oracle.kubernetes.operator.work.NextAction; +import oracle.kubernetes.operator.work.Packet; +import oracle.kubernetes.operator.work.Step; + +public class ExternalAdminChannelIteratorStep extends Step { + private final DomainPresenceInfo info; + private final Iterator it; + + public ExternalAdminChannelIteratorStep(DomainPresenceInfo info, Collection naps, Step next) { + super(next); + this.info = info; + this.it = naps.iterator(); + } + + @Override + public NextAction apply(Packet packet) { + if (it.hasNext()) { + packet.put(ProcessingConstants.SERVER_NAME, info.getDomain().getSpec().getAsName()); + packet.put(ProcessingConstants.NETWORK_ACCESS_POINT, it.next()); + Step step = ServiceHelper.createForExternalChannelStep(this); + return doNext(step, packet); + } + return doNext(packet); + } +} \ No newline at end of file diff --git a/operator/src/main/java/oracle/kubernetes/operator/steps/ExternalAdminChannelsStep.java b/operator/src/main/java/oracle/kubernetes/operator/steps/ExternalAdminChannelsStep.java new file mode 100644 index 00000000000..3e5c3fcf92e --- /dev/null +++ b/operator/src/main/java/oracle/kubernetes/operator/steps/ExternalAdminChannelsStep.java @@ -0,0 +1,118 @@ +// Copyright 2017, 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.steps; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import oracle.kubernetes.operator.helpers.DomainPresenceInfo; +import oracle.kubernetes.operator.logging.LoggingFacade; +import oracle.kubernetes.operator.logging.LoggingFactory; +import oracle.kubernetes.operator.logging.MessageKeys; +import oracle.kubernetes.operator.wlsconfig.NetworkAccessPoint; +import oracle.kubernetes.operator.wlsconfig.WlsDomainConfig; +import oracle.kubernetes.operator.wlsconfig.WlsServerConfig; +import oracle.kubernetes.operator.work.NextAction; +import oracle.kubernetes.operator.work.Packet; +import oracle.kubernetes.operator.work.Step; +import oracle.kubernetes.weblogic.domain.v1.Domain; +import oracle.kubernetes.weblogic.domain.v1.DomainSpec; + +public class ExternalAdminChannelsStep extends Step { + private static final LoggingFacade LOGGER = LoggingFactory.getLogger("Operator", "Operator"); + + public ExternalAdminChannelsStep(Step next) { + super(next); + } + + @Override + public NextAction apply(Packet packet) { + DomainPresenceInfo info = packet.getSPI(DomainPresenceInfo.class); + + Collection validChannels = adminChannelsToCreate(info.getScan(), info.getDomain()); + if (validChannels != null && !validChannels.isEmpty()) { + return doNext(new ExternalAdminChannelIteratorStep(info, validChannels, next), packet); + } + + return doNext(packet); + } + + /** + * This method checks the domain spec against any configured network access + * points defined for the domain. This implementation only handles T3 protocol + * for externalization. + * + * @param scan + * WlsDomainConfig from discovery containing configuration + * @param dom + * Domain containing Domain resource information + * @return Validated collection of network access points + */ + public static Collection adminChannelsToCreate(WlsDomainConfig scan, Domain dom) { + LOGGER.entering(); + + // The following hard-coded values for the nodePort min/max ranges are + // provided here until the appropriate API is discovered to obtain + // this information from Kubernetes. + Integer nodePortMin = 30000; + Integer nodePortMax = 32767; + + DomainSpec spec = dom.getSpec(); + if (spec.getExportT3Channels() == null) { + return null; + } + + WlsServerConfig adminServerConfig = scan.getServerConfig(spec.getAsName()); + + List naps = adminServerConfig.getNetworkAccessPoints(); + // This will become a list of valid channels to create services for. + Collection channels = new ArrayList<>(); + + // Pick out externalized channels from the server channels list + for (String incomingChannel : spec.getExportT3Channels()) { + boolean missingChannel = true; + for (NetworkAccessPoint nap : naps) { + if (nap.getName().equalsIgnoreCase(incomingChannel)) { + missingChannel = false; + channels.add(nap); + break; + } + } + if (missingChannel) { + LOGGER.warning(MessageKeys.EXCH_CHANNEL_NOT_DEFINED, incomingChannel, spec.getAsName()); + } + } + + // Iterate through the selected channels and validate + Collection validatedChannels = new ArrayList<>(); + for (NetworkAccessPoint nap : channels) { + + // Only supporting T3 for now. + if (!nap.getProtocol().equalsIgnoreCase("t3")) { + LOGGER.severe(MessageKeys.EXCH_WRONG_PROTOCOL, nap.getName(), nap.getProtocol()); + continue; + } + + // Until otherwise determined, ports must be the same. + if (!nap.getListenPort().equals(nap.getPublicPort())) { + // log a warning and ignore this item. + LOGGER.warning(MessageKeys.EXCH_UNEQUAL_LISTEN_PORTS, nap.getName()); + continue; + } + + // Make sure configured port is within NodePort range. + if (nap.getListenPort().compareTo(nodePortMin) < 0 || nap.getListenPort().compareTo(nodePortMax) > 0) { + // port setting is outside the NodePort range limits + LOGGER.warning(MessageKeys.EXCH_OUTSIDE_RANGE, nap.getName(), nap.getPublicPort(), nodePortMin, nodePortMax); + continue; + } + + validatedChannels.add(nap); + } + + LOGGER.exiting(); + return validatedChannels; + } +} \ No newline at end of file diff --git a/operator/src/main/java/oracle/kubernetes/operator/steps/ListPersistentVolumeClaimStep.java b/operator/src/main/java/oracle/kubernetes/operator/steps/ListPersistentVolumeClaimStep.java new file mode 100644 index 00000000000..96c1c4c6852 --- /dev/null +++ b/operator/src/main/java/oracle/kubernetes/operator/steps/ListPersistentVolumeClaimStep.java @@ -0,0 +1,64 @@ +// Copyright 2017, 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.steps; + +import java.util.List; +import java.util.Map; + +import io.kubernetes.client.ApiException; +import io.kubernetes.client.models.V1ObjectMeta; +import io.kubernetes.client.models.V1PersistentVolumeClaimList; +import oracle.kubernetes.operator.LabelConstants; +import oracle.kubernetes.operator.helpers.CallBuilder; +import oracle.kubernetes.operator.helpers.CallBuilderFactory; +import oracle.kubernetes.operator.helpers.DomainPresenceInfo; +import oracle.kubernetes.operator.helpers.ResponseStep; +import oracle.kubernetes.operator.work.ContainerResolver; +import oracle.kubernetes.operator.work.NextAction; +import oracle.kubernetes.operator.work.Packet; +import oracle.kubernetes.operator.work.Step; +import oracle.kubernetes.weblogic.domain.v1.Domain; +import oracle.kubernetes.weblogic.domain.v1.DomainSpec; + +public class ListPersistentVolumeClaimStep extends Step { + public ListPersistentVolumeClaimStep(Step next) { + super(next); + } + + @Override + public NextAction apply(Packet packet) { + CallBuilderFactory factory = ContainerResolver.getInstance().getContainer().getSPI(CallBuilderFactory.class); + + DomainPresenceInfo info = packet.getSPI(DomainPresenceInfo.class); + + Domain dom = info.getDomain(); + V1ObjectMeta meta = dom.getMetadata(); + DomainSpec spec = dom.getSpec(); + String namespace = meta.getNamespace(); + + String domainUID = spec.getDomainUID(); + + Step list = factory.create().with($ -> { + $.labelSelector = LabelConstants.DOMAINUID_LABEL + "=" + domainUID; + }).listPersistentVolumeClaimAsync(namespace, new ResponseStep(next) { + @Override + public NextAction onFailure(Packet packet, ApiException e, int statusCode, + Map> responseHeaders) { + if (statusCode == CallBuilder.NOT_FOUND) { + return onSuccess(packet, null, statusCode, responseHeaders); + } + return super.onFailure(packet, e, statusCode, responseHeaders); + } + + @Override + public NextAction onSuccess(Packet packet, V1PersistentVolumeClaimList result, int statusCode, + Map> responseHeaders) { + info.setClaims(result); + return doNext(packet); + } + }); + + return doNext(list, packet); + } +} \ No newline at end of file diff --git a/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServerUpAfterStep.java b/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServerUpAfterStep.java new file mode 100644 index 00000000000..7f37c33c0af --- /dev/null +++ b/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServerUpAfterStep.java @@ -0,0 +1,52 @@ +// Copyright 2017, 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.steps; + +import java.util.Collection; +import java.util.Collections; +import java.util.Map; + +import oracle.kubernetes.operator.ProcessingConstants; +import oracle.kubernetes.operator.helpers.DomainPresenceInfo; +import oracle.kubernetes.operator.helpers.RollingHelper; +import oracle.kubernetes.operator.logging.LoggingFacade; +import oracle.kubernetes.operator.logging.LoggingFactory; +import oracle.kubernetes.operator.work.NextAction; +import oracle.kubernetes.operator.work.Packet; +import oracle.kubernetes.operator.work.Step; +import oracle.kubernetes.weblogic.domain.v1.Domain; +import oracle.kubernetes.weblogic.domain.v1.DomainSpec; + +public class ManagedServerUpAfterStep extends Step { + private static final LoggingFacade LOGGER = LoggingFactory.getLogger("Operator", "Operator"); + + public ManagedServerUpAfterStep(Step next) { + super(next); + } + + @Override + public NextAction apply(Packet packet) { + @SuppressWarnings("unchecked") + Map rolling = (Map) packet.get(ProcessingConstants.SERVERS_TO_ROLL); + + if (LOGGER.isFineEnabled()) { + DomainPresenceInfo info = packet.getSPI(DomainPresenceInfo.class); + + Domain dom = info.getDomain(); + DomainSpec spec = dom.getSpec(); + + Collection rollingList = Collections.emptyList(); + if (rolling != null) { + rollingList = rolling.keySet(); + } + LOGGER.fine("Rolling servers for domain with UID: " + spec.getDomainUID() + ", rolling list: " + rollingList); + } + + if (rolling == null || rolling.isEmpty()) { + return doNext(packet); + } + + return doNext(RollingHelper.rollServers(rolling, next), packet); + } +} \ No newline at end of file diff --git a/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServerUpIteratorStep.java b/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServerUpIteratorStep.java new file mode 100644 index 00000000000..16108c2a45e --- /dev/null +++ b/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServerUpIteratorStep.java @@ -0,0 +1,83 @@ +// Copyright 2017, 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.steps; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import oracle.kubernetes.operator.ProcessingConstants; +import oracle.kubernetes.operator.helpers.DomainPresenceInfo; +import oracle.kubernetes.operator.helpers.PodHelper; +import oracle.kubernetes.operator.helpers.ServiceHelper; +import oracle.kubernetes.operator.helpers.DomainPresenceInfo.ServerStartupInfo; +import oracle.kubernetes.operator.logging.LoggingFacade; +import oracle.kubernetes.operator.logging.LoggingFactory; +import oracle.kubernetes.operator.work.NextAction; +import oracle.kubernetes.operator.work.Packet; +import oracle.kubernetes.operator.work.Step; +import oracle.kubernetes.weblogic.domain.v1.Domain; +import oracle.kubernetes.weblogic.domain.v1.DomainSpec; +import oracle.kubernetes.weblogic.domain.v1.ServerStartup; + +public class ManagedServerUpIteratorStep extends Step { + private static final LoggingFacade LOGGER = LoggingFactory.getLogger("Operator", "Operator"); + + private final Collection c; + + public ManagedServerUpIteratorStep(Collection c, Step next) { + super(next); + this.c = c; + } + + @Override + public NextAction apply(Packet packet) { + Collection startDetails = new ArrayList<>(); + Map rolling = new ConcurrentHashMap<>(); + packet.put(ProcessingConstants.SERVERS_TO_ROLL, rolling); + + for (ServerStartupInfo ssi : c) { + Packet p = packet.clone(); + p.put(ProcessingConstants.SERVER_SCAN, ssi.serverConfig); + p.put(ProcessingConstants.CLUSTER_SCAN, ssi.clusterConfig); + p.put(ProcessingConstants.ENVVARS, ssi.envVars); + + p.put(ProcessingConstants.SERVER_NAME, ssi.serverConfig.getName()); + p.put(ProcessingConstants.PORT, ssi.serverConfig.getListenPort()); + ServerStartup ss = ssi.serverStartup; + p.put(ProcessingConstants.NODE_PORT, ss != null ? ss.getNodePort() : null); + + startDetails.add(new StepAndPacket(bringManagedServerUp(ssi, null), p)); + } + + if (LOGGER.isFineEnabled()) { + DomainPresenceInfo info = packet.getSPI(DomainPresenceInfo.class); + + Domain dom = info.getDomain(); + DomainSpec spec = dom.getSpec(); + + Collection serverList = new ArrayList<>(); + for (ServerStartupInfo ssi : c) { + serverList.add(ssi.serverConfig.getName()); + } + LOGGER.fine("Starting or validating servers for domain with UID: " + spec.getDomainUID() + ", server list: " + + serverList); + } + + if (startDetails.isEmpty()) { + return doNext(packet); + } + return doForkJoin(new ManagedServerUpAfterStep(next), packet, startDetails); + } + + // pre-conditions: DomainPresenceInfo SPI + // "principal" + // "serverScan" + // "clusterScan" + // "envVars" + private static Step bringManagedServerUp(ServerStartupInfo ssi, Step next) { + return PodHelper.createManagedPodStep(ServiceHelper.createForServerStep(next)); + } +} \ No newline at end of file diff --git a/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServersUpStep.java b/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServersUpStep.java new file mode 100644 index 00000000000..4e374556e06 --- /dev/null +++ b/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServersUpStep.java @@ -0,0 +1,259 @@ +// Copyright 2017, 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.steps; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +import io.kubernetes.client.models.V1EnvVar; +import oracle.kubernetes.operator.DomainStatusUpdater; +import oracle.kubernetes.operator.StartupControlConstants; +import oracle.kubernetes.operator.WebLogicConstants; +import oracle.kubernetes.operator.helpers.DomainPresenceInfo; +import oracle.kubernetes.operator.helpers.ServerKubernetesObjects; +import oracle.kubernetes.operator.helpers.DomainPresenceInfo.ServerStartupInfo; +import oracle.kubernetes.operator.logging.LoggingFacade; +import oracle.kubernetes.operator.logging.LoggingFactory; +import oracle.kubernetes.operator.wlsconfig.WlsClusterConfig; +import oracle.kubernetes.operator.wlsconfig.WlsDomainConfig; +import oracle.kubernetes.operator.wlsconfig.WlsServerConfig; +import oracle.kubernetes.operator.work.NextAction; +import oracle.kubernetes.operator.work.Packet; +import oracle.kubernetes.operator.work.Step; +import oracle.kubernetes.weblogic.domain.v1.ClusterStartup; +import oracle.kubernetes.weblogic.domain.v1.Domain; +import oracle.kubernetes.weblogic.domain.v1.DomainSpec; +import oracle.kubernetes.weblogic.domain.v1.ServerStartup; + +public class ManagedServersUpStep extends Step { + private static final LoggingFacade LOGGER = LoggingFactory.getLogger("Operator", "Operator"); + + public ManagedServersUpStep(Step next) { + super(next); + } + + @Override + public NextAction apply(Packet packet) { + LOGGER.entering(); + DomainPresenceInfo info = packet.getSPI(DomainPresenceInfo.class); + + Domain dom = info.getDomain(); + DomainSpec spec = dom.getSpec(); + + if (LOGGER.isFineEnabled()) { + Collection runningList = new ArrayList<>(); + for (Map.Entry entry : info.getServers().entrySet()) { + ServerKubernetesObjects sko = entry.getValue(); + if (sko != null && sko.getPod() != null) { + runningList.add(entry.getKey()); + } + } + LOGGER.fine("Running servers for domain with UID: " + spec.getDomainUID() + ", running list: " + runningList); + } + + String sc = spec.getStartupControl(); + if (sc == null) { + sc = StartupControlConstants.AUTO_STARTUPCONTROL; + } else { + sc = sc.toUpperCase(); + } + + WlsDomainConfig scan = info.getScan(); + Collection ssic = new ArrayList(); + + String asName = spec.getAsName(); + + boolean startAll = false; + Collection servers = new ArrayList(); + switch (sc) { + case StartupControlConstants.ALL_STARTUPCONTROL: + startAll = true; + case StartupControlConstants.AUTO_STARTUPCONTROL: + case StartupControlConstants.SPECIFIED_STARTUPCONTROL: + Collection clusters = new ArrayList(); + + // start specified servers with their custom options + List ssl = spec.getServerStartup(); + if (ssl != null) { + for (ServerStartup ss : ssl) { + String serverName = ss.getServerName(); + WlsServerConfig wlsServerConfig = scan.getServerConfig(serverName); + if (!serverName.equals(asName) && wlsServerConfig != null && !servers.contains(serverName)) { + // start server + servers.add(serverName); + // find cluster if this server is part of one + WlsClusterConfig cc = null; + find: for (WlsClusterConfig wlsClusterConfig : scan.getClusterConfigs().values()) { + for (WlsServerConfig clusterMemberServerConfig : wlsClusterConfig.getServerConfigs()) { + if (serverName.equals(clusterMemberServerConfig.getName())) { + cc = wlsClusterConfig; + break find; + } + } + } + List env = ss.getEnv(); + if (WebLogicConstants.ADMIN_STATE.equals(ss.getDesiredState())) { + env = startInAdminMode(env); + } + ssic.add(new ServerStartupInfo(wlsServerConfig, cc, env, ss)); + } + } + } + List lcs = spec.getClusterStartup(); + if (lcs != null) { + cluster: for (ClusterStartup cs : lcs) { + String clusterName = cs.getClusterName(); + clusters.add(clusterName); + int startedCount = 0; + // find cluster + WlsClusterConfig wlsClusterConfig = scan.getClusterConfig(clusterName); + if (wlsClusterConfig != null) { + for (WlsServerConfig wlsServerConfig : wlsClusterConfig.getServerConfigs()) { + // done with the current cluster + if (startedCount >= cs.getReplicas() && !startAll) + continue cluster; + + String serverName = wlsServerConfig.getName(); + if (!serverName.equals(asName) && !servers.contains(serverName)) { + List env = cs.getEnv(); + ServerStartup ssi = null; + ssl = spec.getServerStartup(); + if (ssl != null) { + for (ServerStartup ss : ssl) { + String s = ss.getServerName(); + if (serverName.equals(s)) { + env = ss.getEnv(); + ssi = ss; + break; + } + } + } + // start server + servers.add(serverName); + if (WebLogicConstants.ADMIN_STATE.equals(cs.getDesiredState())) { + env = startInAdminMode(env); + } + ssic.add(new ServerStartupInfo(wlsServerConfig, wlsClusterConfig, env, ssi)); + startedCount++; + } + } + } + } + } + if (startAll) { + // Look for any other servers + for (WlsClusterConfig wlsClusterConfig : scan.getClusterConfigs().values()) { + for (WlsServerConfig wlsServerConfig : wlsClusterConfig.getServerConfigs()) { + String serverName = wlsServerConfig.getListenAddress(); + // do not start admin server + if (!serverName.equals(asName) && !servers.contains(serverName)) { + // start server + servers.add(serverName); + ssic.add(new ServerStartupInfo(wlsServerConfig, wlsClusterConfig, null, null)); + } + } + } + for (Map.Entry wlsServerConfig : scan.getServerConfigs().entrySet()) { + String serverName = wlsServerConfig.getKey(); + // do not start admin server + if (!serverName.equals(asName) && !servers.contains(serverName)) { + // start server + servers.add(serverName); + ssic.add(new ServerStartupInfo(wlsServerConfig.getValue(), null, null, null)); + } + } + } else if (StartupControlConstants.AUTO_STARTUPCONTROL.equals(sc)) { + for (Map.Entry wlsClusterConfig : scan.getClusterConfigs().entrySet()) { + if (!clusters.contains(wlsClusterConfig.getKey())) { + int startedCount = 0; + WlsClusterConfig config = wlsClusterConfig.getValue(); + for (WlsServerConfig wlsServerConfig : config.getServerConfigs()) { + if (startedCount >= spec.getReplicas()) + break; + String serverName = wlsServerConfig.getName(); + if (!serverName.equals(asName) && !servers.contains(serverName)) { + // start server + servers.add(serverName); + ssic.add(new ServerStartupInfo(wlsServerConfig, config, null, null)); + startedCount++; + } + } + } + } + } + + info.setServerStartupInfo(ssic); + LOGGER.exiting(); + return doNext(scaleDownIfNecessary(info, servers, + new ClusterServicesStep(info, new ManagedServerUpIteratorStep(ssic, next))), packet); + case StartupControlConstants.ADMIN_STARTUPCONTROL: + case StartupControlConstants.NONE_STARTUPCONTROL: + default: + + info.setServerStartupInfo(null); + LOGGER.exiting(); + return doNext(scaleDownIfNecessary(info, servers, new ClusterServicesStep(info, next)), packet); + } + } + + private static List startInAdminMode(List env) { + if (env == null) { + env = new ArrayList<>(); + } + + // look for JAVA_OPTIONS + V1EnvVar jo = null; + for (V1EnvVar e : env) { + if ("JAVA_OPTIONS".equals(e.getName())) { + jo = e; + if (jo.getValueFrom() != null) { + throw new IllegalStateException(); + } + break; + } + } + if (jo == null) { + jo = new V1EnvVar(); + jo.setName("JAVA_OPTIONS"); + env.add(jo); + } + + // create or update value + String startInAdmin = "-Dweblogic.management.startupMode=ADMIN"; + String value = jo.getValue(); + value = (value != null) ? (startInAdmin + " " + value) : startInAdmin; + jo.setValue(value); + + return env; + } + + private static Step scaleDownIfNecessary(DomainPresenceInfo info, Collection servers, Step next) { + Domain dom = info.getDomain(); + DomainSpec spec = dom.getSpec(); + + boolean shouldStopAdmin = false; + String sc = spec.getStartupControl(); + if (sc != null && StartupControlConstants.NONE_STARTUPCONTROL.equals(sc.toUpperCase())) { + shouldStopAdmin = true; + next = DomainStatusUpdater.createAvailableStep(DomainStatusUpdater.ALL_STOPPED_AVAILABLE_REASON, next); + } + + String adminName = spec.getAsName(); + Map currentServers = info.getServers(); + Collection> serversToStop = new ArrayList<>(); + for (Map.Entry entry : currentServers.entrySet()) { + if ((shouldStopAdmin || !entry.getKey().equals(adminName)) && !servers.contains(entry.getKey())) { + serversToStop.add(entry); + } + } + + if (!serversToStop.isEmpty()) { + return new ServerDownIteratorStep(serversToStop, next); + } + + return next; + } +} \ No newline at end of file diff --git a/operator/src/main/java/oracle/kubernetes/operator/steps/ServerDownFinalizeStep.java b/operator/src/main/java/oracle/kubernetes/operator/steps/ServerDownFinalizeStep.java new file mode 100644 index 00000000000..2a8827ffd44 --- /dev/null +++ b/operator/src/main/java/oracle/kubernetes/operator/steps/ServerDownFinalizeStep.java @@ -0,0 +1,22 @@ +package oracle.kubernetes.operator.steps; + +import oracle.kubernetes.operator.helpers.DomainPresenceInfo; +import oracle.kubernetes.operator.work.NextAction; +import oracle.kubernetes.operator.work.Packet; +import oracle.kubernetes.operator.work.Step; + +public class ServerDownFinalizeStep extends Step { + private final String serverName; + + public ServerDownFinalizeStep(String serverName, Step next) { + super(next); + this.serverName = serverName; + } + + @Override + public NextAction apply(Packet packet) { + DomainPresenceInfo info = packet.getSPI(DomainPresenceInfo.class); + info.getServers().remove(serverName); + return doNext(next, packet); + } +} \ No newline at end of file diff --git a/operator/src/main/java/oracle/kubernetes/operator/steps/ServerDownIteratorStep.java b/operator/src/main/java/oracle/kubernetes/operator/steps/ServerDownIteratorStep.java new file mode 100644 index 00000000000..03796254798 --- /dev/null +++ b/operator/src/main/java/oracle/kubernetes/operator/steps/ServerDownIteratorStep.java @@ -0,0 +1,56 @@ +// Copyright 2017, 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.steps; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Map; + +import oracle.kubernetes.operator.helpers.DomainPresenceInfo; +import oracle.kubernetes.operator.helpers.ServerKubernetesObjects; +import oracle.kubernetes.operator.logging.LoggingFacade; +import oracle.kubernetes.operator.logging.LoggingFactory; +import oracle.kubernetes.operator.work.NextAction; +import oracle.kubernetes.operator.work.Packet; +import oracle.kubernetes.operator.work.Step; +import oracle.kubernetes.weblogic.domain.v1.Domain; +import oracle.kubernetes.weblogic.domain.v1.DomainSpec; + +public class ServerDownIteratorStep extends Step { + private static final LoggingFacade LOGGER = LoggingFactory.getLogger("Operator", "Operator"); + + private final Collection> c; + + public ServerDownIteratorStep(Collection> serversToStop, Step next) { + super(next); + this.c = serversToStop; + } + + @Override + public NextAction apply(Packet packet) { + Collection startDetails = new ArrayList<>(); + + for (Map.Entry entry : c) { + startDetails.add(new StepAndPacket(new ServerDownStep(entry.getKey(), entry.getValue(), null), packet.clone())); + } + + if (LOGGER.isFineEnabled()) { + DomainPresenceInfo info = packet.getSPI(DomainPresenceInfo.class); + + Domain dom = info.getDomain(); + DomainSpec spec = dom.getSpec(); + + Collection stopList = new ArrayList<>(); + for (Map.Entry entry : c) { + stopList.add(entry.getKey()); + } + LOGGER.fine("Stopping servers for domain with UID: " + spec.getDomainUID() + ", stop list: " + stopList); + } + + if (startDetails.isEmpty()) { + return doNext(packet); + } + return doForkJoin(next, packet, startDetails); + } +} \ No newline at end of file diff --git a/operator/src/main/java/oracle/kubernetes/operator/steps/ServerDownStep.java b/operator/src/main/java/oracle/kubernetes/operator/steps/ServerDownStep.java new file mode 100644 index 00000000000..83853c2c9a1 --- /dev/null +++ b/operator/src/main/java/oracle/kubernetes/operator/steps/ServerDownStep.java @@ -0,0 +1,28 @@ +// Copyright 2017, 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.steps; + +import oracle.kubernetes.operator.helpers.PodHelper; +import oracle.kubernetes.operator.helpers.ServerKubernetesObjects; +import oracle.kubernetes.operator.helpers.ServiceHelper; +import oracle.kubernetes.operator.work.NextAction; +import oracle.kubernetes.operator.work.Packet; +import oracle.kubernetes.operator.work.Step; + +public class ServerDownStep extends Step { + private final String serverName; + private final ServerKubernetesObjects sko; + + public ServerDownStep(String serverName, ServerKubernetesObjects sko, Step next) { + super(next); + this.serverName = serverName; + this.sko = sko; + } + + @Override + public NextAction apply(Packet packet) { + return doNext(PodHelper.deletePodStep(sko, + ServiceHelper.deleteServiceStep(sko, new ServerDownFinalizeStep(serverName, next))), packet); + } +} \ No newline at end of file diff --git a/operator/src/main/java/oracle/kubernetes/operator/steps/WatchPodReadyAdminStep.java b/operator/src/main/java/oracle/kubernetes/operator/steps/WatchPodReadyAdminStep.java new file mode 100644 index 00000000000..6ffcc243211 --- /dev/null +++ b/operator/src/main/java/oracle/kubernetes/operator/steps/WatchPodReadyAdminStep.java @@ -0,0 +1,35 @@ +// Copyright 2017, 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.steps; + +import java.util.Map; + +import io.kubernetes.client.models.V1Pod; +import oracle.kubernetes.operator.PodWatcher; +import oracle.kubernetes.operator.ProcessingConstants; +import oracle.kubernetes.operator.helpers.DomainPresenceInfo; +import oracle.kubernetes.operator.work.Component; +import oracle.kubernetes.operator.work.NextAction; +import oracle.kubernetes.operator.work.Packet; +import oracle.kubernetes.operator.work.Step; + +public class WatchPodReadyAdminStep extends Step { + private final Map podWatchers; + + public WatchPodReadyAdminStep(Map podWatchers, Step next) { + super(next); + this.podWatchers = podWatchers; + } + + @Override + public NextAction apply(Packet packet) { + DomainPresenceInfo info = packet.getSPI(DomainPresenceInfo.class); + V1Pod adminPod = info.getAdmin().getPod().get(); + + PodWatcher pw = podWatchers.get(adminPod.getMetadata().getNamespace()); + packet.getComponents().put(ProcessingConstants.PODWATCHER_COMPONENT_NAME, Component.createFor(pw)); + + return doNext(pw.waitForReady(adminPod, next), packet); + } +} \ No newline at end of file diff --git a/operator/src/test/java/oracle/kubernetes/operator/ExternalChannelTest.java b/operator/src/test/java/oracle/kubernetes/operator/ExternalChannelTest.java index e9f6942ca43..be985cb8ef6 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/ExternalChannelTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/ExternalChannelTest.java @@ -8,6 +8,7 @@ import oracle.kubernetes.weblogic.domain.v1.Domain; import oracle.kubernetes.weblogic.domain.v1.DomainSpec; import oracle.kubernetes.operator.logging.LoggingFactory; +import oracle.kubernetes.operator.steps.ExternalAdminChannelsStep; import oracle.kubernetes.operator.wlsconfig.NetworkAccessPoint; import oracle.kubernetes.operator.wlsconfig.WlsDomainConfig; import oracle.kubernetes.operator.wlsconfig.WlsServerConfig; @@ -110,7 +111,7 @@ public void externalChannelNotDefinedTest() { chlist.clear(); chlist.add("channel-99"); - Collection validNaps = Main.adminChannelsToCreate(wlsDomainConfig, domain); + Collection validNaps = ExternalAdminChannelsStep.adminChannelsToCreate(wlsDomainConfig, domain); assertNotNull(validNaps); assertEquals(0, validNaps.size()); } @@ -135,7 +136,7 @@ public void externalChannelOutsideRangeTest() throws Exception { setListenPort(7001); setPublicPort(7001); - Collection validNaps = Main.adminChannelsToCreate(wlsDomainConfig, domain); + Collection validNaps = ExternalAdminChannelsStep.adminChannelsToCreate(wlsDomainConfig, domain); assertNotNull(validNaps); assertEquals(0, validNaps.size()); } @@ -148,7 +149,7 @@ public void externalChannelUnequalListenPortsTest() throws IllegalAccessExceptio setListenPort(39001); setPublicPort(39002); - Collection validNaps = Main.adminChannelsToCreate(wlsDomainConfig, domain); + Collection validNaps = ExternalAdminChannelsStep.adminChannelsToCreate(wlsDomainConfig, domain); assertNotNull(validNaps); assertEquals(0, validNaps.size()); } @@ -162,7 +163,7 @@ public void externalChannelWrongProtocolTest() throws IllegalAccessException { setPublicPort(39001); setProtocol("http"); - Collection validNaps = Main.adminChannelsToCreate(wlsDomainConfig, domain); + Collection validNaps = ExternalAdminChannelsStep.adminChannelsToCreate(wlsDomainConfig, domain); assertNotNull(validNaps); assertEquals(0, validNaps.size()); } @@ -176,7 +177,7 @@ public void externalChannelTest() throws IllegalAccessException { setPublicPort(30999); setProtocol("t3"); - Collection validNaps = Main.adminChannelsToCreate(wlsDomainConfig, domain); + Collection validNaps = ExternalAdminChannelsStep.adminChannelsToCreate(wlsDomainConfig, domain); assertNotNull(validNaps); assertEquals(1, validNaps.size()); assertEquals("Channel-3", validNaps.iterator().next().getName()); From 07cd04d44b9a2d13bf3856d8141672c35643af90 Mon Sep 17 00:00:00 2001 From: Anthony Lai Date: Mon, 16 Apr 2018 09:12:11 -0700 Subject: [PATCH 039/186] address pull request comments by removing unused codes --- .../operator/helpers/ServiceHelper.java | 7 --- .../operator/wlsconfig/WlsDomainConfig.java | 2 +- .../operator/wlsconfig/WlsRetriever.java | 44 ------------------- 3 files changed, 1 insertion(+), 52 deletions(-) diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/ServiceHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/ServiceHelper.java index c912952f984..2e82ad02591 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/ServiceHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/ServiceHelper.java @@ -99,17 +99,10 @@ public NextAction apply(Packet packet) { List ports = new ArrayList<>(); V1ServicePort servicePort = new V1ServicePort(); servicePort.setPort(port); - servicePort.setName("weblogic"); if (nodePort != null) { servicePort.setNodePort(nodePort); } ports.add(servicePort); - - V1ServicePort nmPort = new V1ServicePort(); - nmPort.setPort(5556); - nmPort.setName("node-manager"); - ports.add(nmPort); - serviceSpec.setPorts(ports); service.setSpec(serviceSpec); diff --git a/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfig.java b/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfig.java index b1ec6e7ec02..cef45432d18 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfig.java +++ b/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfig.java @@ -16,7 +16,7 @@ import java.util.Map; /** - * Contains a snapsot of configuration for a WebLogic Domain + * Contains a snapshot of configuration for a WebLogic Domain */ public class WlsDomainConfig { private static final LoggingFacade LOGGER = LoggingFactory.getLogger("Operator", "Operator"); diff --git a/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsRetriever.java b/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsRetriever.java index b94b56f3603..e232c083893 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsRetriever.java +++ b/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsRetriever.java @@ -471,50 +471,6 @@ private String executePostUrlWithRetry(final String url, final String payload, f return jsonResult; } - /** - * Update the dynamic cluster size of the WLS cluster configuration. - * - * @param wlsClusterConfig The WlsClusterConfig object of the WLS cluster whose cluster size needs to be updated - * @param machineNamePrefix Prefix of names of new machines to be created - * @param targetClusterSize The target dynamic cluster size - * @return true if the request to update the cluster size is successful, false if it was not successful within the - * time period, or the cluster is not a dynamic cluster - */ - public boolean updateDynamicClusterSize(final WlsClusterConfig wlsClusterConfig, - final String machineNamePrefix, - final int targetClusterSize) { - - LOGGER.entering(); - - final long timeout = UPDATE_CONFIG_TIMEOUT_MILLIS; - - String clusterName = wlsClusterConfig == null? "null": wlsClusterConfig.getClusterName(); - - if (wlsClusterConfig == null || !wlsClusterConfig.hasDynamicServers()) { - LOGGER.warning(MessageKeys.WLS_UPDATE_CLUSTER_SIZE_INVALID_CLUSTER, clusterName); - return false; - } - - ScheduledExecutorService executorService = ContainerResolver.getInstance().getContainer().getSPI(ScheduledExecutorService.class); - long startTime = System.currentTimeMillis(); - Future future = executorService.submit(() -> doUpdateDynamicClusterSize(wlsClusterConfig, machineNamePrefix, targetClusterSize)); - boolean result = false; - try { - result = future.get(timeout, TimeUnit.MILLISECONDS); - if (result) { - LOGGER.info(MessageKeys.WLS_CLUSTER_SIZE_UPDATED, clusterName, targetClusterSize, (System.currentTimeMillis() - startTime)); - } else { - LOGGER.warning(MessageKeys.WLS_UPDATE_CLUSTER_SIZE_FAILED, clusterName, null); - } - } catch (InterruptedException | ExecutionException e) { - LOGGER.warning(MessageKeys.WLS_UPDATE_CLUSTER_SIZE_FAILED, clusterName, e); - } catch (TimeoutException e) { - LOGGER.warning(MessageKeys.WLS_UPDATE_CLUSTER_SIZE_TIMED_OUT, clusterName, timeout); - } - LOGGER.exiting(result); - return result; - } - /** * Method called by the Callable that is submitted from the updateDynamicClusterSize method for updating the * WLS dynamic cluster size configuration. From 5d4aefc152a7582b5ee943b3ce0658fb807f0f8d Mon Sep 17 00:00:00 2001 From: anthony_lai Date: Mon, 16 Apr 2018 14:52:10 -0700 Subject: [PATCH 040/186] no longer set maxDynamicClusterSize during scaling and no longer initialize it to 800. Remove machine creation related code --- .../create-weblogic-domain-job-template.yaml | 8 +- kubernetes/internal/create-weblogic-domain.sh | 1 + .../operator/helpers/PodHelper.java | 8 -- .../kubernetes/operator/wlsconfig/Util.java | 28 ----- .../operator/wlsconfig/WlsClusterConfig.java | 39 +++---- .../operator/wlsconfig/WlsDomainConfig.java | 3 +- .../operator/wlsconfig/WlsRetriever.java | 42 +------ .../operator/wlsconfig/UtilTest.java | 31 ----- .../wlsconfig/WlsClusterConfigTest.java | 110 +++++++++++++----- 9 files changed, 103 insertions(+), 167 deletions(-) delete mode 100644 operator/src/main/java/oracle/kubernetes/operator/wlsconfig/Util.java delete mode 100644 operator/src/test/java/oracle/kubernetes/operator/wlsconfig/UtilTest.java diff --git a/kubernetes/internal/create-weblogic-domain-job-template.yaml b/kubernetes/internal/create-weblogic-domain-job-template.yaml index 24ecb3d755a..848f5f9b0b4 100644 --- a/kubernetes/internal/create-weblogic-domain-job-template.yaml +++ b/kubernetes/internal/create-weblogic-domain-job-template.yaml @@ -119,6 +119,7 @@ data: domain_path = os.environ.get("DOMAIN_HOME") cluster_name = "%CLUSTER_NAME%" number_of_ms = %CONFIGURED_MANAGED_SERVER_COUNT% + initial_replicas = %INITIAL_MANAGED_SERVER_REPLICAS% cluster_type = "%CLUSTER_TYPE%" print('domain_path : [%s]' % domain_path); @@ -230,12 +231,9 @@ data: cd('DynamicServers/%s' % cluster_name) set('ServerTemplate', st1) set('ServerNamePrefix', "%MANAGED_SERVER_NAME_BASE%") - set('DynamicClusterSize', number_of_ms) - set('MaxDynamicClusterSize', 800) - #set('CalculatedMachineNames', true) + set('DynamicClusterSize', initial_replicas) + set('MaxDynamicClusterSize', number_of_ms) set('CalculatedListenPorts', false) - #machineNameExpression = '%DOMAIN_UID%-%s-machine*' % cluster_name - #set('MachineNameMatchExpression', machineNameExpression) set('Id', 1) print('Done setting attributes for Dynamic Cluster: %s' % cluster_name); diff --git a/kubernetes/internal/create-weblogic-domain.sh b/kubernetes/internal/create-weblogic-domain.sh index 5dbd68d3f46..cef6115af19 100755 --- a/kubernetes/internal/create-weblogic-domain.sh +++ b/kubernetes/internal/create-weblogic-domain.sh @@ -456,6 +456,7 @@ function createYamlFiles { sed -i -e "s:%ADMIN_SERVER_NAME%:${adminServerName}:g" ${jobOutput} sed -i -e "s:%ADMIN_PORT%:${adminPort}:g" ${jobOutput} sed -i -e "s:%CONFIGURED_MANAGED_SERVER_COUNT%:${configuredManagedServerCount}:g" ${jobOutput} + sed -i -e "s:%INITIAL_MANAGED_SERVER_REPLICAS%:${initialManagedServerReplicas}:g" ${jobOutput} sed -i -e "s:%MANAGED_SERVER_NAME_BASE%:${managedServerNameBase}:g" ${jobOutput} sed -i -e "s:%MANAGED_SERVER_PORT%:${managedServerPort}:g" ${jobOutput} sed -i -e "s:%T3_CHANNEL_PORT%:${t3ChannelPort}:g" ${jobOutput} diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java index 2540f5ad5e3..d37faae90d1 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java @@ -639,16 +639,8 @@ protected V1Pod computeManagedPodConfig(TuningParameters configMapHelper, Packet V1ContainerPort containerPort = new V1ContainerPort(); containerPort.setContainerPort(scan.getListenPort()); containerPort.setProtocol("TCP"); -// containerPort.setName("weblogic"); // commented out as we are not exposing node manager port container.addPortsItem(containerPort); - // Commented out as we are not exposing node manager port -// V1ContainerPort nmPort = new V1ContainerPort(); -// nmPort.setContainerPort(5556); -// nmPort.setProtocol("TCP"); -// nmPort.setName("node-manager"); -// container.addPortsItem(nmPort); - V1Lifecycle lifecycle = new V1Lifecycle(); V1Handler preStop = new V1Handler(); V1ExecAction exec = new V1ExecAction(); diff --git a/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/Util.java b/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/Util.java deleted file mode 100644 index 54787b446ff..00000000000 --- a/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/Util.java +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2018, Oracle Corporation and/or its affiliates. All rights reserved. -// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. -package oracle.kubernetes.operator.wlsconfig; - - -import oracle.kubernetes.weblogic.domain.v1.DomainSpec; - -/** - * Utility class for WebLogic configuration related classes - */ -public class Util { - - /** - * Return prefix for names of machines that will be used servers in a dynamic cluster. - * The prefix would be "[domainUID]-[clusterName]-machine" - * - * @param domainSpec DomainSpec for the weblogic operator domain - * @param wlsClusterConfig WlsClusterConfig object containing the dynamic cluster - * @return Prefix for names of machines that will be used servers in a dynamic cluster - */ - static String getMachineNamePrefix(DomainSpec domainSpec, WlsClusterConfig wlsClusterConfig) { - if (domainSpec != null && domainSpec.getDomainUID() != null - && wlsClusterConfig != null && wlsClusterConfig.getClusterName() != null) { - return domainSpec.getDomainUID() + "-" + wlsClusterConfig.getClusterName() + "-machine"; - } - return null; - } -} diff --git a/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfig.java b/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfig.java index af6dc14960d..05be34cddd7 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfig.java +++ b/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfig.java @@ -171,14 +171,11 @@ public int getMaxDynamicClusterSize() { * It is the responsibility of the caller to persist the changes to ClusterStartup to kubernetes. * * @param clusterStartup The ClusterStartup to be validated against the WLS configuration - * @param machineNamePrefix Optional, if this is not null, also validate whether the WebLogic domain already contains - * all the machines that will be used by the dynamic cluster * @param suggestedConfigUpdates A List containing suggested WebLogic configuration update to be filled in by this * method. Optional. * @return true if the DomainSpec has been updated, false otherwise */ public boolean validateClusterStartup(ClusterStartup clusterStartup, - String machineNamePrefix, List suggestedConfigUpdates) { LOGGER.entering(); @@ -190,7 +187,7 @@ public boolean validateClusterStartup(ClusterStartup clusterStartup, } // Warns if replicas is larger than the number of servers configured in the cluster - validateReplicas(clusterStartup.getReplicas(), machineNamePrefix,"clusterStartup", suggestedConfigUpdates); + validateReplicas(clusterStartup.getReplicas(),"clusterStartup", suggestedConfigUpdates); LOGGER.exiting(modified); @@ -204,34 +201,33 @@ public boolean validateClusterStartup(ClusterStartup clusterStartup, * * @param replicas The configured replicas value for this cluster in the kubernetes weblogic domain spec * for this cluster - * @param machineNamePrefix Optional, if this is not null, also validate whether the WebLogic domain already contains - * all the machines that will be used by the dynamic cluster * @param source The name of the section in the domain spec where the replicas is specified, * for logging purposes * @param suggestedConfigUpdates A List containing suggested WebLogic configuration update to be filled in by this * method. Optional. */ - public void validateReplicas(Integer replicas, String machineNamePrefix, + public void validateReplicas(Integer replicas, String source, List suggestedConfigUpdates) { if (replicas == null) { return; } - // log warning if replicas is too large and cluster only contains statically configured servers - if (!hasDynamicServers() && replicas > getClusterSize()) { - LOGGER.warning(MessageKeys.REPLICA_MORE_THAN_WLS_SERVERS, source, clusterName, replicas, getClusterSize()); + // log warning if replicas is too large and cluster + int maxClusterSize = getClusterSize(); + if (hasDynamicServers()) { + maxClusterSize += getMaxDynamicClusterSize(); + } + if (replicas > maxClusterSize) { + LOGGER.warning(MessageKeys.REPLICA_MORE_THAN_WLS_SERVERS, source, clusterName, replicas, maxClusterSize); } // recommend updating WLS dynamic cluster size and machines if requested to recommend // updates, ie, suggestedConfigUpdates is not null, and if replicas value is larger than - // the current dynamic cluster size, or if some of the machines to be used for the dynamic - // servers are not yet configured. + // the current dynamic cluster size. // // Note: Never reduce the value of dynamicClusterSize even during scale down - if (suggestedConfigUpdates != null) { - if (hasDynamicServers()) { -// if (replicas > getDynamicClusterSize() || !verifyMachinesConfigured(machineNamePrefix, replicas)) { - if (replicas > getDynamicClusterSize() ) { - suggestedConfigUpdates.add(new DynamicClusterSizeConfigUpdate(this, Math.max(replicas, getDynamicClusterSize()))); - } + if (suggestedConfigUpdates != null && hasDynamicServers()) { + if (replicas > getDynamicClusterSize() && getDynamicClusterSize() < getMaxDynamicClusterSize()) { + // increase dynamic cluster size to satisfy replicas, but only up to the configured max dynamic cluster size + suggestedConfigUpdates.add(new DynamicClusterSizeConfigUpdate(this, Math.min(replicas, getMaxDynamicClusterSize()))); } } } @@ -329,17 +325,14 @@ public String getUpdateDynamicClusterSizeUrl() { /** * Return the payload used in the REST request for updating the dynamic cluster size. It will - * be used to update the cluster size and if necessary, the max cluster size of the dynamic servers - * of this cluster. + * be used to update the cluster size of the dynamic servers of this cluster. * * @param clusterSize Desired dynamic cluster size * @return A string containing the payload to be used in the REST request for updating the dynamic * cluster size to the specified value. */ public String getUpdateDynamicClusterSizePayload(final int clusterSize) { - return "{ dynamicClusterSize: " + clusterSize + ", " + - " maxDynamicClusterSize: " + (clusterSize > getMaxDynamicClusterSize()? clusterSize: getMaxDynamicClusterSize()) + - " }"; + return "{ dynamicClusterSize: " + clusterSize + " }"; } @Override diff --git a/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfig.java b/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfig.java index cef45432d18..7be91ec3ab7 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfig.java +++ b/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsDomainConfig.java @@ -306,7 +306,7 @@ public boolean validate(DomainSpec domainSpec, List suggestedConfi if (clusterName != null) { WlsClusterConfig wlsClusterConfig = getClusterConfig(clusterName); updated |= wlsClusterConfig.validateClusterStartup(clusterStartup, - Util.getMachineNamePrefix(domainSpec, wlsClusterConfig), suggestedConfigUpdates); + suggestedConfigUpdates); } } } @@ -318,7 +318,6 @@ public boolean validate(DomainSpec domainSpec, List suggestedConfi if (clusterConfigs != null && clusterConfigs.size() == 1) { for (WlsClusterConfig wlsClusterConfig : clusterConfigs) { wlsClusterConfig.validateReplicas(domainSpec.getReplicas(), - Util.getMachineNamePrefix(domainSpec, wlsClusterConfig), "domainSpec", suggestedConfigUpdates); } diff --git a/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsRetriever.java b/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsRetriever.java index e232c083893..57565bc8a13 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsRetriever.java +++ b/operator/src/main/java/oracle/kubernetes/operator/wlsconfig/WlsRetriever.java @@ -210,12 +210,8 @@ public NextAction apply(Packet packet) { String serviceURL = HttpClient.getServiceURL(info.getAdmin().getService().get()); - Domain dom = info.getDomain(); - DomainSpec domainSpec = dom.getSpec(); - String machineNamePrefix = Util.getMachineNamePrefix(domainSpec, wlsClusterConfig); - boolean successful = updateDynamicClusterSizeWithServiceURL(wlsClusterConfig, - machineNamePrefix, targetClusterSize, httpClient, serviceURL); + targetClusterSize, httpClient, serviceURL); if (successful) { LOGGER.info(MessageKeys.WLS_CLUSTER_SIZE_UPDATED, clusterName, targetClusterSize, (System.currentTimeMillis() - startTime)); @@ -471,37 +467,11 @@ private String executePostUrlWithRetry(final String url, final String payload, f return jsonResult; } - /** - * Method called by the Callable that is submitted from the updateDynamicClusterSize method for updating the - * WLS dynamic cluster size configuration. - * - * @param wlsClusterConfig The WlsClusterConfig object of the WLS cluster whose cluster size needs to be updated. The - * caller should make sure that the cluster is a dynamic cluster. - * @param machineNamePrefix Prefix of names of new machines to be created - * @param targetClusterSize The target dynamic cluster size - * @return true if the request to update the cluster size is successful, false if it was not successful - */ - - private boolean doUpdateDynamicClusterSize(final WlsClusterConfig wlsClusterConfig, - final String machineNamePrefix, - final int targetClusterSize) throws Exception { - LOGGER.entering(); - - String serviceURL = connectAndGetServiceURL(); - - boolean result = updateDynamicClusterSizeWithServiceURL(wlsClusterConfig, machineNamePrefix, - targetClusterSize, httpClient, serviceURL); - - LOGGER.exiting(result); - return result; - } - /** * Static method to update the WebLogic dynamic cluster size configuration. * * @param wlsClusterConfig The WlsClusterConfig object of the WLS cluster whose cluster size needs to be updated. The * caller should make sure that the cluster is a dynamic cluster. - * @param machineNamePrefix Prefix of names of new machines to be created * @param targetClusterSize The target dynamic cluster size * @param httpClient HttpClient object for issuing the REST request * @param serviceURL service URL of the WebLogic admin server @@ -510,22 +480,12 @@ private boolean doUpdateDynamicClusterSize(final WlsClusterConfig wlsClusterConf */ private static boolean updateDynamicClusterSizeWithServiceURL(final WlsClusterConfig wlsClusterConfig, - final String machineNamePrefix, final int targetClusterSize, final HttpClient httpClient, final String serviceURL) { LOGGER.entering(); boolean result = false; - // Create machine(s) - // Commented out as we are not configuring machines for servers -// String newMachineNames[] = wlsClusterConfig.getMachineNamesForDynamicServers(machineNamePrefix, targetClusterSize); -// for (String machineName: newMachineNames) { -// LOGGER.info(MessageKeys.WLS_CREATING_MACHINE, machineName); -// httpClient.executePostUrlOnServiceClusterIP(WlsMachineConfig.getCreateUrl(), -// serviceURL, WlsMachineConfig.getCreatePayload(machineName)); -// } - // Update the dynamic cluster size of the WebLogic cluster String jsonResult = httpClient.executePostUrlOnServiceClusterIP( wlsClusterConfig.getUpdateDynamicClusterSizeUrl(), diff --git a/operator/src/test/java/oracle/kubernetes/operator/wlsconfig/UtilTest.java b/operator/src/test/java/oracle/kubernetes/operator/wlsconfig/UtilTest.java deleted file mode 100644 index 8d358e5651a..00000000000 --- a/operator/src/test/java/oracle/kubernetes/operator/wlsconfig/UtilTest.java +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2018, Oracle Corporation and/or its affiliates. All rights reserved. -// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. - -package oracle.kubernetes.operator.wlsconfig; - -import oracle.kubernetes.weblogic.domain.v1.DomainSpec; -import org.junit.Test; - -import static org.junit.Assert.*; - -public class UtilTest { - - @Test - public void verifyMachineNamePrefixFromGetMachineNamePrefix() throws Exception { - DomainSpec domainSpec = new DomainSpec().withDomainUID("domain1"); - WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1"); - assertEquals("domain1-cluster1-machine", Util.getMachineNamePrefix(domainSpec, wlsClusterConfig)); - } - - @Test - public void verifyNoNPEwithNullArgumentsToGetMachineNamePrefix() throws Exception { - DomainSpec domainSpec = new DomainSpec().withDomainUID("domain1"); - WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1"); - - assertNull(Util.getMachineNamePrefix(null, wlsClusterConfig)); - assertNull(Util.getMachineNamePrefix(domainSpec, null)); - assertNull(Util.getMachineNamePrefix(null, null)); - } - - -} \ No newline at end of file diff --git a/operator/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfigTest.java b/operator/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfigTest.java index 810f3367dea..1f2d84fc59f 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfigTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/wlsconfig/WlsClusterConfigTest.java @@ -164,7 +164,7 @@ public void verifyValidateClusterStartupWarnsIfNoServersInCluster() throws Excep TestUtil.LogHandlerImpl handler = null; try { handler = TestUtil.setupLogHandler(wlsClusterConfig); - wlsClusterConfig.validateClusterStartup(cs, null,null); + wlsClusterConfig.validateClusterStartup(cs, null); assertTrue("Message logged: " + handler.getAllFormattedMessage(), handler.hasWarningMessageWithSubString("No servers configured in WebLogic cluster with name cluster1")); } finally { TestUtil.removeLogHandler(wlsClusterConfig, handler); @@ -179,7 +179,7 @@ public void verifyValidateClusterStartupWarnsIfReplicasTooHigh() throws Exceptio TestUtil.LogHandlerImpl handler = null; try { handler = TestUtil.setupLogHandler(wlsClusterConfig); - wlsClusterConfig.validateClusterStartup(cs, null,null); + wlsClusterConfig.validateClusterStartup(cs, null); assertTrue("Message logged: " + handler.getAllFormattedMessage(), handler.hasWarningMessageWithSubString("Replicas in clusterStartup for cluster cluster1 is specified with a value of 2 which is larger than the number of configured WLS servers in the cluster: 1")); } finally { TestUtil.removeLogHandler(wlsClusterConfig, handler); @@ -192,32 +192,66 @@ public void verifyValidateClusterStartupDoNotSuggestsUpdateToConfiguredClusterIf wlsClusterConfig.addServerConfig(createWlsServerConfig("ms-0", 8011, null)); ClusterStartup cs = new ClusterStartup().withClusterName("cluster1").withReplicas(2); ArrayList suggestedConfigUpdates = new ArrayList<>(); - wlsClusterConfig.validateClusterStartup(cs, null, suggestedConfigUpdates); + wlsClusterConfig.validateClusterStartup(cs, suggestedConfigUpdates); assertEquals(0, suggestedConfigUpdates.size()); } @Test - public void verifyValidateClusterStartupDoesNotWarnIfDynamicCluster() throws Exception { + public void verifyValidateClusterStartupWarnsIfReplicasTooHigh_DynamicCluster() throws Exception { WlsDynamicServersConfig wlsDynamicServersConfig = createDynamicServersConfig(1, 1, "ms-", "cluster1"); WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1", wlsDynamicServersConfig); ClusterStartup cs = new ClusterStartup().withClusterName("cluster1").withReplicas(2); TestUtil.LogHandlerImpl handler = null; try { handler = TestUtil.setupLogHandler(wlsClusterConfig); - wlsClusterConfig.validateClusterStartup(cs, null, null); + wlsClusterConfig.validateClusterStartup(cs, null); + assertTrue("Message logged: " + handler.getAllFormattedMessage(), handler.hasWarningMessageWithSubString("Replicas in clusterStartup for cluster cluster1 is specified with a value of 2 which is larger than the number of configured WLS servers in the cluster: 1")); + } finally { + TestUtil.removeLogHandler(wlsClusterConfig, handler); + } + } + + @Test + public void verifyValidateClusterStartupWarnsIfReplicasTooHigh_mixedCluster() throws Exception { + WlsDynamicServersConfig wlsDynamicServersConfig = createDynamicServersConfig(1, 1, "ms-", "cluster1"); + WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1", wlsDynamicServersConfig); + wlsClusterConfig.addServerConfig(createWlsServerConfig("ms-0", 8011, null)); + ClusterStartup cs = new ClusterStartup().withClusterName("cluster1").withReplicas(3); + TestUtil.LogHandlerImpl handler = null; + try { + handler = TestUtil.setupLogHandler(wlsClusterConfig); + wlsClusterConfig.validateClusterStartup(cs, null); + assertTrue("Message logged: " + handler.getAllFormattedMessage(), handler.hasWarningMessageWithSubString("Replicas in clusterStartup for cluster cluster1 is specified with a value of 3 which is larger than the number of configured WLS servers in the cluster: 2")); + } finally { + TestUtil.removeLogHandler(wlsClusterConfig, handler); + } + } + + @Test + public void verifyValidateClusterStartupDoNotWarnIfReplicasNotHigh_mixedCluster() throws Exception { + WlsDynamicServersConfig wlsDynamicServersConfig = createDynamicServersConfig(1, 1, "ms-", "cluster1"); + WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1", wlsDynamicServersConfig); + wlsClusterConfig.addServerConfig(createWlsServerConfig("ms-0", 8011, null)); + ClusterStartup cs = new ClusterStartup().withClusterName("cluster1").withReplicas(2); + TestUtil.LogHandlerImpl handler = null; + try { + handler = TestUtil.setupLogHandler(wlsClusterConfig); + wlsClusterConfig.validateClusterStartup(cs, null); assertFalse("No message should be logged, but found: " + handler.getAllFormattedMessage(), handler.hasWarningMessageLogged()); } finally { TestUtil.removeLogHandler(wlsClusterConfig, handler); } } + + @Test public void verifyValidateClusterStartupSuggestsUpdateToDynamicClusterIfReplicasTooHigh() throws Exception { - WlsDynamicServersConfig wlsDynamicServersConfig = createDynamicServersConfig(1, 1, "ms-", "cluster1"); + WlsDynamicServersConfig wlsDynamicServersConfig = createDynamicServersConfig(1, 2, "ms-", "cluster1"); WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1", wlsDynamicServersConfig); ClusterStartup cs = new ClusterStartup().withClusterName("cluster1").withReplicas(2); ArrayList suggestedConfigUpdates = new ArrayList<>(); - wlsClusterConfig.validateClusterStartup(cs, null, suggestedConfigUpdates); + wlsClusterConfig.validateClusterStartup(cs, suggestedConfigUpdates); assertEquals(1, suggestedConfigUpdates.size()); WlsClusterConfig.DynamicClusterSizeConfigUpdate configUpdate = (WlsClusterConfig.DynamicClusterSizeConfigUpdate) suggestedConfigUpdates.get(0); @@ -225,16 +259,48 @@ public void verifyValidateClusterStartupSuggestsUpdateToDynamicClusterIfReplicas } @Test - public void verifyValidateClusterStartupDoNotSuggestsUpdateToDynamicClusterIfReplicasNotHigh() throws Exception { - WlsDynamicServersConfig wlsDynamicServersConfig = createDynamicServersConfig(1, 1, "ms-", "cluster1"); + public void verifyValidateClusterStartupDoNotSuggestsUpdateToDynamicClusterIfReplicasSameAsClusterSize() throws Exception { + WlsDynamicServersConfig wlsDynamicServersConfig = createDynamicServersConfig(1, 2, "ms-", "cluster1"); WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1", wlsDynamicServersConfig); ClusterStartup cs = new ClusterStartup().withClusterName("cluster1").withReplicas(1); TestUtil.LogHandlerImpl handler = null; ArrayList suggestedConfigUpdates = new ArrayList<>(); - wlsClusterConfig.validateClusterStartup(cs, null, suggestedConfigUpdates); + wlsClusterConfig.validateClusterStartup(cs, suggestedConfigUpdates); assertEquals(0, suggestedConfigUpdates.size()); } + @Test + public void verifyValidateClusterStartupDoNotSuggestsUpdateToDynamicClusterIfReplicasLowerThanClusterSize() throws Exception { + WlsDynamicServersConfig wlsDynamicServersConfig = createDynamicServersConfig(2, 2, "ms-", "cluster1"); + WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1", wlsDynamicServersConfig); + ClusterStartup cs = new ClusterStartup().withClusterName("cluster1").withReplicas(1); + ArrayList suggestedConfigUpdates = new ArrayList<>(); + wlsClusterConfig.validateClusterStartup(cs, suggestedConfigUpdates); + assertEquals(0, suggestedConfigUpdates.size()); + } + + @Test + public void verifyValidateClusterStartupDoNotSuggestsUpdateToDynamicClusterIfCurrentSizeAlreadyMax() throws Exception { + WlsDynamicServersConfig wlsDynamicServersConfig = createDynamicServersConfig(2, 2, "ms-", "cluster1"); + WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1", wlsDynamicServersConfig); + ClusterStartup cs = new ClusterStartup().withClusterName("cluster1").withReplicas(3); + ArrayList suggestedConfigUpdates = new ArrayList<>(); + wlsClusterConfig.validateClusterStartup(cs, suggestedConfigUpdates); + assertEquals(0, suggestedConfigUpdates.size()); + } + + @Test + public void verifyValidateClusterStartupSuggestsUpdateToDynamicClusterEvenIfReplicasExceedsMaxClusterSize() throws Exception { + WlsDynamicServersConfig wlsDynamicServersConfig = createDynamicServersConfig(1, 2, "ms-", "cluster1"); + WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1", wlsDynamicServersConfig); + ClusterStartup cs = new ClusterStartup().withClusterName("cluster1").withReplicas(10); + ArrayList suggestedConfigUpdates = new ArrayList<>(); + wlsClusterConfig.validateClusterStartup(cs, suggestedConfigUpdates); + WlsClusterConfig.DynamicClusterSizeConfigUpdate configUpdate = + (WlsClusterConfig.DynamicClusterSizeConfigUpdate) suggestedConfigUpdates.get(0); + assertEquals(2, configUpdate.targetClusterSize); + } + @Ignore // we are currently not suggesting updates based on number of machines @Test public void verifyValidateClusterStartupSuggestsUpdateToDynamicClusterIfNotEnoughMachines() throws Exception { @@ -254,7 +320,7 @@ public void verifyValidateClusterStartupSuggestsUpdateToDynamicClusterIfNotEnoug ClusterStartup cs = new ClusterStartup().withClusterName("cluster1").withReplicas(2); ArrayList suggestedConfigUpdates = new ArrayList<>(); - wlsClusterConfig.validateClusterStartup(cs, "domain1-cluster1-machine", suggestedConfigUpdates); + wlsClusterConfig.validateClusterStartup(cs, suggestedConfigUpdates); assertEquals(1, suggestedConfigUpdates.size()); WlsClusterConfig.DynamicClusterSizeConfigUpdate configUpdate = (WlsClusterConfig.DynamicClusterSizeConfigUpdate) suggestedConfigUpdates.get(0); @@ -281,7 +347,7 @@ public void verifyValidateClusterStartupDoNotLowerClusterSizeIfNotEnoughMachines // replica = 1, dynamicClusterSize = 2, num of machines = 0 ==> need to create one machine // but need to ensure that the update will not reduce dynamicClusterSize from 2 to 1 - wlsClusterConfig.validateClusterStartup(cs, "domain1-cluster1-machine", suggestedConfigUpdates); + wlsClusterConfig.validateClusterStartup(cs, suggestedConfigUpdates); assertEquals(1, suggestedConfigUpdates.size()); WlsClusterConfig.DynamicClusterSizeConfigUpdate configUpdate = (WlsClusterConfig.DynamicClusterSizeConfigUpdate) suggestedConfigUpdates.get(0); @@ -307,7 +373,7 @@ public void verifyValidateClusterStartupDoNotSuggestUpdateToDynamicClusterIfEnou ClusterStartup cs = new ClusterStartup().withClusterName("cluster1").withReplicas(1); ArrayList suggestedConfigUpdates = new ArrayList<>(); - wlsClusterConfig.validateClusterStartup(cs, "domain1-cluster1-machine", suggestedConfigUpdates); + wlsClusterConfig.validateClusterStartup(cs, suggestedConfigUpdates); assertEquals(0, suggestedConfigUpdates.size()); } @@ -319,24 +385,10 @@ public void verifyGetUpdateDynamicClusterSizeUrlIncludesClusterName() { } @Test - public void verifyGetUpdateDynamicClusterSizePayload1() { + public void verifyGetUpdateDynamicClusterSizePayload() { WlsDynamicServersConfig wlsDynamicServersConfig = createDynamicServersConfig(1, 5, "ms-", "cluster1"); WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1", wlsDynamicServersConfig); - assertEquals(wlsClusterConfig.getUpdateDynamicClusterSizePayload(2), "{ dynamicClusterSize: 2, maxDynamicClusterSize: 5 }"); - } - - @Test - public void verifyGetUpdateDynamicClusterSizePayload2() { - WlsDynamicServersConfig wlsDynamicServersConfig = createDynamicServersConfig(2, 5, "ms-", "cluster1"); - WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1", wlsDynamicServersConfig); - assertEquals(wlsClusterConfig.getUpdateDynamicClusterSizePayload(1), "{ dynamicClusterSize: 1, maxDynamicClusterSize: 5 }"); - } - - @Test - public void verifyGetUpdateDynamicClusterSizePayload3() { - WlsDynamicServersConfig wlsDynamicServersConfig = createDynamicServersConfig(2, 5, "ms-", "cluster1"); - WlsClusterConfig wlsClusterConfig = new WlsClusterConfig("cluster1", wlsDynamicServersConfig); - assertEquals(wlsClusterConfig.getUpdateDynamicClusterSizePayload(8), "{ dynamicClusterSize: 8, maxDynamicClusterSize: 8 }"); + assertEquals(wlsClusterConfig.getUpdateDynamicClusterSizePayload(2), "{ dynamicClusterSize: 2 }"); } @Test From 381cbd0ed8fb604bdbeb516998335074924da1f7 Mon Sep 17 00:00:00 2001 From: anthony_lai Date: Mon, 16 Apr 2018 17:35:32 -0700 Subject: [PATCH 041/186] use configuredManagedServerCount for dynamic cluster size. Remove unused log message --- kubernetes/internal/create-weblogic-domain-job-template.yaml | 3 +-- kubernetes/internal/create-weblogic-domain.sh | 1 - .../java/oracle/kubernetes/operator/logging/MessageKeys.java | 1 - operator/src/main/resources/Operator.properties | 1 - 4 files changed, 1 insertion(+), 5 deletions(-) diff --git a/kubernetes/internal/create-weblogic-domain-job-template.yaml b/kubernetes/internal/create-weblogic-domain-job-template.yaml index 848f5f9b0b4..86efcdf8827 100644 --- a/kubernetes/internal/create-weblogic-domain-job-template.yaml +++ b/kubernetes/internal/create-weblogic-domain-job-template.yaml @@ -119,7 +119,6 @@ data: domain_path = os.environ.get("DOMAIN_HOME") cluster_name = "%CLUSTER_NAME%" number_of_ms = %CONFIGURED_MANAGED_SERVER_COUNT% - initial_replicas = %INITIAL_MANAGED_SERVER_REPLICAS% cluster_type = "%CLUSTER_TYPE%" print('domain_path : [%s]' % domain_path); @@ -231,7 +230,7 @@ data: cd('DynamicServers/%s' % cluster_name) set('ServerTemplate', st1) set('ServerNamePrefix', "%MANAGED_SERVER_NAME_BASE%") - set('DynamicClusterSize', initial_replicas) + set('DynamicClusterSize', number_of_ms) set('MaxDynamicClusterSize', number_of_ms) set('CalculatedListenPorts', false) set('Id', 1) diff --git a/kubernetes/internal/create-weblogic-domain.sh b/kubernetes/internal/create-weblogic-domain.sh index cef6115af19..5dbd68d3f46 100755 --- a/kubernetes/internal/create-weblogic-domain.sh +++ b/kubernetes/internal/create-weblogic-domain.sh @@ -456,7 +456,6 @@ function createYamlFiles { sed -i -e "s:%ADMIN_SERVER_NAME%:${adminServerName}:g" ${jobOutput} sed -i -e "s:%ADMIN_PORT%:${adminPort}:g" ${jobOutput} sed -i -e "s:%CONFIGURED_MANAGED_SERVER_COUNT%:${configuredManagedServerCount}:g" ${jobOutput} - sed -i -e "s:%INITIAL_MANAGED_SERVER_REPLICAS%:${initialManagedServerReplicas}:g" ${jobOutput} sed -i -e "s:%MANAGED_SERVER_NAME_BASE%:${managedServerNameBase}:g" ${jobOutput} sed -i -e "s:%MANAGED_SERVER_PORT%:${managedServerPort}:g" ${jobOutput} sed -i -e "s:%T3_CHANNEL_PORT%:${t3ChannelPort}:g" ${jobOutput} diff --git a/operator/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java b/operator/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java index b3914fb118d..93e1577cc88 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java +++ b/operator/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java @@ -141,5 +141,4 @@ private MessageKeys() {} public static final String WLS_UPDATE_CLUSTER_SIZE_INVALID_CLUSTER = "WLSKO-0131"; public static final String WLS_CLUSTER_SIZE_UPDATED = "WLSKO-0132"; public static final String WLS_SERVER_TEMPLATE_NOT_FOUND = "WLSKO-0133"; - public static final String WLS_CREATING_MACHINE = "WLSKO-0134"; } diff --git a/operator/src/main/resources/Operator.properties b/operator/src/main/resources/Operator.properties index b532ca97bbe..0881992486f 100644 --- a/operator/src/main/resources/Operator.properties +++ b/operator/src/main/resources/Operator.properties @@ -132,4 +132,3 @@ WLSKO-0130=Failed to update WebLogic dynamic cluster size for cluster {0} within WLSKO-0131=Failed to update WebLogic dynamic cluster size for cluster {0}. Cluster is not a dynamic cluster WLSKO-0132=Updated cluster size for WebLogic dynamic cluster {0} to {1}. Time taken {2} ms WLSKO-0133=Cannot find WebLogic server template with name {0} which is referenced by WebLogic cluster {1} -WLSKO-0134=Creating WebLogic machine named {0} From 4a16532941b39fd575b7ccee5dfb9dd0a682d322 Mon Sep 17 00:00:00 2001 From: Ryan Eberhard Date: Tue, 17 Apr 2018 08:13:18 -0400 Subject: [PATCH 042/186] Update run.sh --- src/integration-tests/bash/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/integration-tests/bash/run.sh b/src/integration-tests/bash/run.sh index 81b1718103f..04a8e1e0d0d 100755 --- a/src/integration-tests/bash/run.sh +++ b/src/integration-tests/bash/run.sh @@ -709,7 +709,7 @@ function test_second_operator { # function dom_define { if [ "$#" != 14 ] ; then - fail "requires 13 parameters: DOM_KEY OP_KEY NAMESPACE DOMAIN_UID STARTUP_CONTROL WL_CLUSTER_NAME WL_CLUSTER_TYPE MS_BASE_NAME ADMIN_PORT ADMIN_WLST_PORT ADMIN_NODE_PORT MS_PORT LOAD_BALANCER_WEB_PORT LOAD_BALANCER_DASHBOARD_PORT" + fail "requires 14 parameters: DOM_KEY OP_KEY NAMESPACE DOMAIN_UID STARTUP_CONTROL WL_CLUSTER_NAME WL_CLUSTER_TYPE MS_BASE_NAME ADMIN_PORT ADMIN_WLST_PORT ADMIN_NODE_PORT MS_PORT LOAD_BALANCER_WEB_PORT LOAD_BALANCER_DASHBOARD_PORT" fi local DOM_KEY="`echo \"${1}\" | sed 's/-/_/g'`" eval export DOM_${DOM_KEY}_OP_KEY="$2" From 7962d80a55ac7c7fae0094a82c0d3e37fee87205 Mon Sep 17 00:00:00 2001 From: Mark Nelson Date: Tue, 17 Apr 2018 08:32:38 -0400 Subject: [PATCH 043/186] temporary fix for yum issue --- wercker.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wercker.yml b/wercker.yml index 6495f12fd9d..2804c00d859 100644 --- a/wercker.yml +++ b/wercker.yml @@ -52,7 +52,8 @@ build: - script: name: Remove things we do not want in the Docker image in order to reduce size of image code: | - yum -y remove tar gzip + rpm -e --nodeps tar + rpm -e --nodeps gzip yum clean all # push the image to quay.io using the GIT branch as the tag # this image needs to be available to the integration-test pipeline for testing From 8421a3a729737bb2f52c193f8278cf308f290dc3 Mon Sep 17 00:00:00 2001 From: Mark Nelson Date: Tue, 17 Apr 2018 08:35:20 -0400 Subject: [PATCH 044/186] temporary fix for yum issue --- wercker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/wercker.yml b/wercker.yml index 2804c00d859..e3c5c9c3ccc 100644 --- a/wercker.yml +++ b/wercker.yml @@ -55,6 +55,7 @@ build: rpm -e --nodeps tar rpm -e --nodeps gzip yum clean all + rm -rf /var/cache/yum # push the image to quay.io using the GIT branch as the tag # this image needs to be available to the integration-test pipeline for testing - internal/docker-push: From d4b9b014d93c1d7fea6aea46ce12ab293aaef9f8 Mon Sep 17 00:00:00 2001 From: Rosemary Marano Date: Tue, 17 Apr 2018 09:00:12 -0400 Subject: [PATCH 045/186] fix typos --- site/name-changes.md | 6 +++--- site/recent-changes.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/site/name-changes.md b/site/name-changes.md index aa3b10bd718..16fc30bce54 100644 --- a/site/name-changes.md +++ b/site/name-changes.md @@ -10,7 +10,7 @@ We addressed this by: This changes how operators and domains are created, and also changes the generated artifacts for operators and domains. -We're not providing an upgrade tool or backwards compatibility with the previous names. Instead, customers need to use the new script, inputs file, and parameter names to recreate their operators and domains. +We're not providing an upgrade tool or backward compatibility with the previous names. Instead, customers need to use the new script, inputs file, and parameter names to recreate their operators and domains. This document lists the customer visible naming changes. Also, the WebLogic Server Kubernetes Operator documentation has been updated. @@ -35,7 +35,7 @@ Typically, customers do not use these YAML files. However, customers can look a #### Directory for the Generated YAML Files -Previously, these files were placed in the `kubernetes` directory (for example, `kubernetes/weblogic-operator.yaml`). Now, they are placed in per-operator and per-domain directories (since a Kubernetes cluster can have more than one operator and an operator can manage more than one domain). +Previously, these files were placed in the `kubernetes` directory (for example, `kubernetes/weblogic-operator.yaml`). Now, they are placed in per-operator and per-domain directories (because a Kubernetes cluster can have more than one operator and an operator can manage more than one domain). The customer must create a directory that will parent the per-operator and per-domain directories, and use the `-o` option to pass the name of that directory to the create script, for example: `mkdir /scratch/my-user-projects @@ -54,7 +54,7 @@ Similarly, the per-domain directory name is: * `kubernetes/delete-weblogic-domain-resources.sh yourDomainUID` * Remove the resources that were created for the operator: * `kubectl delete -f weblogic-operator.yaml` - * kubectl delete -f weblogic-operator-security.yaml + * `kubectl delete -f weblogic-operator-security.yaml` * Either remove the directory that was generated for that operator or domain, or remove the generated YAML files and the copy of the input file from it. * Make whatever changes you need in your inputs file. * Re-run the create script. diff --git a/site/recent-changes.md b/site/recent-changes.md index 942ae6e0574..ba64301c8ba 100644 --- a/site/recent-changes.md +++ b/site/recent-changes.md @@ -1,6 +1,6 @@ # Recent Changes to the Oracle WebLogic Server Kubernetes Operator -This page track recent changes to the operator, especially ones that introduce backward incompatibilities. +This page tracks recent changes to the operator, especially ones that introduce backward incompatibilities. | Date | Introduces Backward Incompatibilities | Change | | --- | --- | --- | From ca729879536d1cce927c07e78cab035947e59e98 Mon Sep 17 00:00:00 2001 From: Lily He Date: Wed, 18 Apr 2018 01:19:53 -0700 Subject: [PATCH 046/186] add option in itest to set load balancer type --- kubernetes/internal/create-weblogic-domain.sh | 24 ++++++++++--------- .../internal/voyager-ingress-template.yaml | 16 +++++++++---- src/integration-tests/bash/run.sh | 11 +++++++-- 3 files changed, 33 insertions(+), 18 deletions(-) diff --git a/kubernetes/internal/create-weblogic-domain.sh b/kubernetes/internal/create-weblogic-domain.sh index f6c04c49143..c6a5d8032c1 100755 --- a/kubernetes/internal/create-weblogic-domain.sh +++ b/kubernetes/internal/create-weblogic-domain.sh @@ -325,7 +325,7 @@ function initialize { voyagerInput="${scriptDir}/voyager-ingress-template.yaml" if [ ! -f ${voyagerInput} ]; then - validationError "The template file ${voyagerInput} for generating the voyager ingress was not found" + validationError "The template file ${voyagerInput} for generating the Voyager Ingress was not found" fi failIfValidationErrors @@ -504,10 +504,12 @@ function createYamlFiles { fi if [ "${loadBalancer}" = "VOYAGER" ]; then - # Voyager ingress file + # Voyager Ingress file cp ${voyagerInput} ${voyagerOutput} echo Generating ${voyagerOutput} + sed -i -e "s:%NAMESPACE%:$namespace:g" ${voyagerOutput} sed -i -e "s:%DOMAIN_UID%:${domainUID}:g" ${voyagerOutput} + sed -i -e "s:%DOMAIN_NAME%:${domainName}:g" ${voyagerOutput} sed -i -e "s:%CLUSTER_NAME%:${clusterName}:g" ${voyagerOutput} sed -i -e "s:%MANAGED_SERVER_PORT%:${managedServerPort}:g" ${voyagerOutput} sed -i -e "s:%LOAD_BALANCER_WEB_PORT%:$loadBalancerWebPort:g" ${voyagerOutput} @@ -606,25 +608,25 @@ function createDomain { } # -# Deploy voyager/HAProxy load balancer +# Deploy Voyager/HAProxy load balancer # function setupVoyagerLoadBalancer { - # deploy voyager ingress controller + # deploy Voyager Ingress controller kubectl create namespace voyager curl -fsSL https://raw.githubusercontent.com/appscode/voyager/6.0.0/hack/deploy/voyager.sh \ | bash -s -- --provider=baremetal --namespace=voyager - # deploy voyager ingress resource - kubectl create -f ${voyagerOutput} + # deploy Voyager Ingress resource + kubectl apply -f ${voyagerOutput} - echo Checking voyager deploy - vdep=`kubectl get deploy | grep voyager | wc | awk ' { print $1; } '` + echo Checking Voyager deploy + vdep=`kubectl get deploy -n ${namespace} | grep voyager | wc | awk ' { print $1; } '` if [ "$vdep" != "1" ]; then - fail "The deployment of voyager ingress was not created" + fail "The deployment of Voyager Ingress was not created" fi -echo Checking voyager service - vscv=`kubectl get service voyager-stats | grep voyager-stats | wc | awk ' { print $1; } '` +echo Checking Voyager service + vscv=`kubectl get service voyager-stats -n ${namespace} | grep voyager-stats | wc | awk ' { print $1; } '` if [ "$vscv" != "1" ]; then fail "The service voyager-stats was not created" fi diff --git a/kubernetes/internal/voyager-ingress-template.yaml b/kubernetes/internal/voyager-ingress-template.yaml index 88fb5c63558..4f9bed4e5e6 100644 --- a/kubernetes/internal/voyager-ingress-template.yaml +++ b/kubernetes/internal/voyager-ingress-template.yaml @@ -1,8 +1,11 @@ apiVersion: voyager.appscode.com/v1beta1 kind: Ingress metadata: - name: voyager-ingress - namespace: default + name: %DOMAIN_UID%-voyager + namespace: %NAMESPACE% + labels: + weblogic.domainUID: %DOMAIN_UID% + weblogic.domainName: %DOMAIN_NAME% annotations: ingress.appscode.com/type: 'NodePort' ingress.appscode.com/stats: 'true' @@ -20,9 +23,12 @@ spec: apiVersion: v1 kind: Service metadata: - name: voyager-stats + name: %DOMAIN_UID%-voyager-stats + namespace: %NAMESPACE% labels: - app: voyager-stats + app: %DOMAIN_UID%-voyager-stats + weblogic.domainUID: %DOMAIN_UID% + weblogic.domainName: %DOMAIN_NAME% spec: type: NodePort ports: @@ -33,4 +39,4 @@ spec: nodePort: %LOAD_BALANCER_DASHBOARD_PORT% selector: origin: voyager - origin-name: voyager-ingress + origin-name: %DOMAIN_UID%-voyager diff --git a/src/integration-tests/bash/run.sh b/src/integration-tests/bash/run.sh index 26542bc04f9..ce3170f1756 100755 --- a/src/integration-tests/bash/run.sh +++ b/src/integration-tests/bash/run.sh @@ -78,6 +78,9 @@ # See "Directory Configuration and Structure" below for # defaults and a detailed description of test directories. # +# LB_TYPE Load balancer type. Can be 'TRAEFIK', 'VOYAGER'. +# Default is 'TRAEFIK'. +# # VERBOSE Set to 'true' to echo verbose output to stdout. # Default is 'false'. # @@ -768,8 +771,8 @@ function run_create_domain_job { local DOMAIN_STORAGE_DIR="domain-${DOMAIN_UID}-storage" - trace "Create $DOMAIN_UID in $NAMESPACE namespace " - + trace "Create $DOMAIN_UID in $NAMESPACE namespace with load balancer $LB_TYPE" + local tmp_dir="$TMP_DIR" mkdir -p $tmp_dir @@ -823,6 +826,7 @@ function run_create_domain_job { if [ -n "${WEBLOGIC_IMAGE_PULL_SECRET_NAME}" ]; then sed -i -e "s|#weblogicImagePullSecretName:.*|weblogicImagePullSecretName: ${WEBLOGIC_IMAGE_PULL_SECRET_NAME}|g" $inputs fi + sed -i -e "s/^loadBalancer:.*/loadBalancer: $LB_TYPE/" $inputs sed -i -e "s/^loadBalancerWebPort:.*/loadBalancerWebPort: $LOAD_BALANCER_WEB_PORT/" $inputs sed -i -e "s/^loadBalancerDashboardPort:.*/loadBalancerDashboardPort: $LOAD_BALANCER_DASHBOARD_PORT/" $inputs sed -i -e "s/^javaOptions:.*/javaOptions: $WLS_JAVA_OPTIONS/" $inputs @@ -2443,6 +2447,7 @@ function test_suite_init { local varname for varname in RESULT_ROOT \ PV_ROOT \ + LB_TYPE \ VERBOSE \ QUICKTEST \ NODEPORT_HOST \ @@ -2462,6 +2467,7 @@ function test_suite_init { export RESULT_ROOT=${RESULT_ROOT:-/scratch/$USER/wl_k8s_test_results} export PV_ROOT=${PV_ROOT:-$RESULT_ROOT} + export LB_TYPE=${LB_TPYE:-TRAEFIK} export NODEPORT_HOST=${K8S_NODEPORT_HOST:-`hostname | awk -F. '{print $1}'`} export JVM_ARGS="${JVM_ARGS:-'-Dweblogic.StdoutDebugEnabled=false'}" export BRANCH_NAME="${BRANCH_NAME:-$WERCKER_GIT_BRANCH}" @@ -2485,6 +2491,7 @@ function test_suite_init { local varname for varname in RESULT_ROOT \ PV_ROOT \ + LB_TYPE \ VERBOSE \ QUICKTEST \ NODEPORT_HOST \ From 0ef0127fd684de6b30e5461a158a362ccb320565 Mon Sep 17 00:00:00 2001 From: Lily He Date: Wed, 18 Apr 2018 02:08:23 -0700 Subject: [PATCH 047/186] fix parsing LB_TYPE in run.sh --- src/integration-tests/bash/run.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/integration-tests/bash/run.sh b/src/integration-tests/bash/run.sh index ce3170f1756..e676ccd57a0 100755 --- a/src/integration-tests/bash/run.sh +++ b/src/integration-tests/bash/run.sh @@ -2467,7 +2467,6 @@ function test_suite_init { export RESULT_ROOT=${RESULT_ROOT:-/scratch/$USER/wl_k8s_test_results} export PV_ROOT=${PV_ROOT:-$RESULT_ROOT} - export LB_TYPE=${LB_TPYE:-TRAEFIK} export NODEPORT_HOST=${K8S_NODEPORT_HOST:-`hostname | awk -F. '{print $1}'`} export JVM_ARGS="${JVM_ARGS:-'-Dweblogic.StdoutDebugEnabled=false'}" export BRANCH_NAME="${BRANCH_NAME:-$WERCKER_GIT_BRANCH}" @@ -2477,6 +2476,10 @@ function test_suite_init { [ ! "$?" = "0" ] && fail "Error: Could not determine branch. Run script from within a git repo". fi + if [ -z "$LB_TYPE" ]; then + export LB_TYPE=TRAEFIK + fi + export LEASE_ID="${LEASE_ID}" # The following customizable exports are currently only customized by WERCKER From 3e4e43e7ff3b1424ed30a8822f1fbceee8f26f44 Mon Sep 17 00:00:00 2001 From: Lily He Date: Wed, 18 Apr 2018 02:33:21 -0700 Subject: [PATCH 048/186] fix checking voyager service failure --- kubernetes/internal/create-weblogic-domain.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/internal/create-weblogic-domain.sh b/kubernetes/internal/create-weblogic-domain.sh index c6a5d8032c1..13e32cd8a80 100755 --- a/kubernetes/internal/create-weblogic-domain.sh +++ b/kubernetes/internal/create-weblogic-domain.sh @@ -626,7 +626,7 @@ function setupVoyagerLoadBalancer { fi echo Checking Voyager service - vscv=`kubectl get service voyager-stats -n ${namespace} | grep voyager-stats | wc | awk ' { print $1; } '` + vscv=`kubectl get service -n ${namespace} | grep voyager-stats | wc | awk ' { print $1; } '` if [ "$vscv" != "1" ]; then fail "The service voyager-stats was not created" fi From 062e1b83bdd2d6813b37bc3f25ae5b6f2e13c87e Mon Sep 17 00:00:00 2001 From: Lily He Date: Wed, 18 Apr 2018 05:15:51 -0700 Subject: [PATCH 049/186] fix checking voyager service --- kubernetes/internal/create-weblogic-domain.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/internal/create-weblogic-domain.sh b/kubernetes/internal/create-weblogic-domain.sh index 13e32cd8a80..8bbcfd00b10 100755 --- a/kubernetes/internal/create-weblogic-domain.sh +++ b/kubernetes/internal/create-weblogic-domain.sh @@ -626,7 +626,7 @@ function setupVoyagerLoadBalancer { fi echo Checking Voyager service - vscv=`kubectl get service -n ${namespace} | grep voyager-stats | wc | awk ' { print $1; } '` + vscv=`kubectl get service ${domainUID}-voyager-stats -n ${namespace} | grep voyager-stats | wc | awk ' { print $1; } '` if [ "$vscv" != "1" ]; then fail "The service voyager-stats was not created" fi From cb3edfde6d08d3fc576b0e2fbc12793bac2082d7 Mon Sep 17 00:00:00 2001 From: Ryan Eberhard Date: Wed, 18 Apr 2018 10:31:06 -0400 Subject: [PATCH 050/186] Security health check helper improvements and minor updates to roles --- .../internal/generate-security-policy.sh | 20 +- .../java/oracle/kubernetes/operator/Main.java | 2 +- .../operator/helpers/AuthenticationProxy.java | 2 +- .../operator/helpers/AuthorizationProxy.java | 121 ++++++--- .../operator/helpers/CallBuilder.java | 36 +++ .../operator/helpers/ClientFactory.java | 12 + .../operator/helpers/ClientPool.java | 37 ++- .../operator/helpers/HealthCheckHelper.java | 149 +++-------- .../kubernetes/operator/helpers/Pool.java | 6 + .../operator/rest/RestBackendImpl.java | 4 +- .../src/main/resources/Operator.properties | 4 +- .../operator/HealthCheckHelperTest.java | 248 +++++++++++++++--- .../CreateOperatorGeneratedFilesBaseTest.java | 26 +- 13 files changed, 464 insertions(+), 203 deletions(-) create mode 100644 operator/src/main/java/oracle/kubernetes/operator/helpers/ClientFactory.java diff --git a/kubernetes/internal/generate-security-policy.sh b/kubernetes/internal/generate-security-policy.sh index a92185d0dcb..beb90170e82 100755 --- a/kubernetes/internal/generate-security-policy.sh +++ b/kubernetes/internal/generate-security-policy.sh @@ -94,8 +94,11 @@ metadata: weblogic.operatorName: ${NAMESPACE} rules: - apiGroups: [""] - resources: ["namespaces", "persistentvolumes"] + resources: ["namespaces"] verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete", "deletecollection"] - apiGroups: ["apiextensions.k8s.io"] resources: ["customresourcedefinitions"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete", "deletecollection"] @@ -108,6 +111,12 @@ rules: - apiGroups: ["extensions"] resources: ["ingresses"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete", "deletecollection"] +- apiGroups: ["authentication.k8s.io"] + resources: ["tokenreviews"] + verbs: ["create"] +- apiGroups: ["authorization.k8s.io"] + resources: ["selfsubjectaccessreviews", "localsubjectaccessreviews", "subjectaccessreviews", "selfsubjectrulesreviews"] + verbs: ["create"] --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1beta1 @@ -197,13 +206,13 @@ metadata: weblogic.operatorName: ${NAMESPACE} rules: - apiGroups: [""] - resources: ["secrets", "persistentvolumeclaims"] + resources: ["secrets"] verbs: ["get", "list", "watch"] - apiGroups: ["storage.k8s.io"] resources: ["storageclasses"] verbs: ["get", "list", "watch"] - apiGroups: [""] - resources: ["services", "configmaps", "pods", "jobs", "events"] + resources: ["services", "configmaps", "pods", "podtemplates", "events", "persistentvolumeclaims"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete", "deletecollection"] - apiGroups: [""] resources: ["pods/logs"] @@ -211,6 +220,9 @@ rules: - apiGroups: [""] resources: ["pods/exec"] verbs: ["create"] +- apiGroups: ["batch"] + resources: ["jobs", "cronjobs"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete", "deletecollection"] - apiGroups: ["settings.k8s.io"] resources: ["podpresets"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete", "deletecollection"] @@ -253,4 +265,4 @@ EOF # echo "Create the WebLogic Operator Security configuration using kubectl as follows: kubectl create -f ${SCRIPT}" # -echo "Ensure you start the API server with the --authorization-mode=RBAC option." +echo "Ensure you start the API server with the --authorization-mode=RBAC option." \ No newline at end of file diff --git a/operator/src/main/java/oracle/kubernetes/operator/Main.java b/operator/src/main/java/oracle/kubernetes/operator/Main.java index 9d1e1b32823..4636a3855bb 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/Main.java +++ b/operator/src/main/java/oracle/kubernetes/operator/Main.java @@ -220,7 +220,7 @@ private static void begin() { HealthCheckHelper healthCheck = new HealthCheckHelper(namespace, targetNamespaces); version = healthCheck.performK8sVersionCheck(); healthCheck.performNonSecurityChecks(); - healthCheck.performSecurityChecks(serviceAccountName); + healthCheck.performSecurityChecks(); } catch (ApiException e) { LOGGER.warning(MessageKeys.EXCEPTION, e); } diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/AuthenticationProxy.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/AuthenticationProxy.java index a7b3b694c02..38ea68ac2c7 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/AuthenticationProxy.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/AuthenticationProxy.java @@ -36,7 +36,7 @@ public V1TokenReviewStatus check(String principal, String token) { try { boolean allowed = authorizationProxy.check(principal, AuthorizationProxy.Operation.create, - AuthorizationProxy.Resource.tokenreviews, + AuthorizationProxy.Resource.TOKENREVIEWS, null, AuthorizationProxy.Scope.cluster, null); diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/AuthorizationProxy.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/AuthorizationProxy.java index c82d202a46b..d58606cb16f 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/AuthorizationProxy.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/AuthorizationProxy.java @@ -8,6 +8,8 @@ import io.kubernetes.client.ApiException; import io.kubernetes.client.models.V1ObjectMeta; import io.kubernetes.client.models.V1ResourceAttributes; +import io.kubernetes.client.models.V1SelfSubjectAccessReview; +import io.kubernetes.client.models.V1SelfSubjectAccessReviewSpec; import io.kubernetes.client.models.V1SubjectAccessReview; import io.kubernetes.client.models.V1SubjectAccessReviewSpec; import io.kubernetes.client.models.V1SubjectAccessReviewStatus; @@ -37,18 +39,50 @@ public enum Operation { } public enum Resource { - pods, - services, - namespaces, - customresources, - customresourcedefinitions, - domains, - tokenreviews, - networkpolicies, - secrets, - persistentvolumes, - persistentvolumeclaims, - ingresses + CONFIGMAPS ("configmaps", ""), + PODS ("pods", ""), + LOGS ("pods", "logs", ""), + EXEC ("pods", "exec", ""), + PODTEMPLATES ("podtemplates", ""), + EVENTS ("events", ""), + SERVICES ("services", ""), + NAMESPACES ("namespaces", ""), + JOBS ("jobs", "batch"), + CRONJOBS ("cronjobs", "batch"), + CRDS ("customresourcedefinitions", "apiextensions.k8s.io"), + DOMAINS ("domains", "weblogic.oracle"), + DOMAINSTATUSS ("domains", "status", "weblogic.oracle"), + SUBJECTACCESSREVIEWS ("subjectaccessreviews", "authorization.k8s.io"), + SELFSUBJECTACCESSREVIEWS ("selfsubjectaccessreviews", "authorization.k8s.io"), + LOCALSUBJECTACCESSREVIEWS ("localsubjectaccessreviews", "authorization.k8s.io"), + SELFSUBJECTRULESREVIEWS ("selfsubjectrulesreviews", "authorization.k8s.io"), + TOKENREVIEWS ("tokenreviews", "authentication.k8s.io"), + SECRETS ("secrets", ""), + PERSISTENTVOLUMES ("persistentvolumes", ""), + PERSISTENTVOLUMECLAIMS ("persistentvolumeclaims", ""), + STORAGECLASSES ("storageclasses", "storage.k8s.io"), + PODPRESETS ("podpresets" , "settings.k8s.io"), + INGRESSES ("ingresses", "extensions"), + NETWORKPOLICIES ("networkpolicies", "extensions"), + PODSECURITYPOLICIES ("podsecuritypolicies", "extensions"); + + private final String resource; + private final String subResource; + private final String apiGroup; + + Resource(String resource, String apiGroup) { + this(resource, "", apiGroup); + } + + Resource(String resource, String subResource, String apiGroup) { + this.resource = resource; + this.subResource = subResource; + this.apiGroup = apiGroup; + } + + public String getResource() { return resource; } + public String getSubResource() { return subResource; } + public String getAPIGroup() { return apiGroup; } } public enum Scope { @@ -104,6 +138,24 @@ public boolean check(String principal, final List groups, Operation oper return result; } + public boolean check(Operation operation, Resource resource, String resourceName, Scope scope, String namespaceName) { + LOGGER.entering(); + V1SelfSubjectAccessReview subjectAccessReview = prepareSelfSubjectAccessReview(operation, resource, resourceName, scope, namespaceName); + try { + CallBuilderFactory factory = ContainerResolver.getInstance().getContainer().getSPI(CallBuilderFactory.class); + subjectAccessReview = factory.create().createSelfSubjectAccessReview(subjectAccessReview); + } catch (ApiException e) { + LOGGER.severe(MessageKeys.APIEXCEPTION_FROM_SUBJECT_ACCESS_REVIEW, e); + LOGGER.exiting(Boolean.FALSE); + return Boolean.FALSE; + + } + V1SubjectAccessReviewStatus subjectAccessReviewStatus = subjectAccessReview.getStatus(); + Boolean result = subjectAccessReviewStatus.isAllowed(); + LOGGER.exiting(result); + return result; + } + /** * Prepares an instance of SubjectAccessReview and returns same. * @@ -133,6 +185,21 @@ private V1SubjectAccessReview prepareSubjectAccessReview(String principal, final return subjectAccessReview; } + private V1SelfSubjectAccessReview prepareSelfSubjectAccessReview(Operation operation, Resource resource, String resourceName, Scope scope, String namespaceName) { + LOGGER.entering(); + V1SelfSubjectAccessReviewSpec subjectAccessReviewSpec = new V1SelfSubjectAccessReviewSpec(); + + subjectAccessReviewSpec.setResourceAttributes(prepareResourceAttributes(operation, resource, resourceName, scope, namespaceName)); + + V1SelfSubjectAccessReview subjectAccessReview = new V1SelfSubjectAccessReview(); + subjectAccessReview.setApiVersion("authorization.k8s.io/v1"); + subjectAccessReview.setKind("SelfSubjectAccessReview"); + subjectAccessReview.setMetadata(new V1ObjectMeta()); + subjectAccessReview.setSpec(subjectAccessReviewSpec); + LOGGER.exiting(subjectAccessReview); + return subjectAccessReview; + } + /** * Prepares an instance of ResourceAttributes and returns same. * @@ -150,12 +217,9 @@ private V1ResourceAttributes prepareResourceAttributes(Operation operation, Reso resourceAttributes.setVerb(operation.toString()); } if (null != resource) { - resourceAttributes.setResource(resource.toString()); - } - - String apiGroup = getApiGroup(resource); - if (apiGroup != null) { - resourceAttributes.setGroup(apiGroup); + resourceAttributes.setResource(resource.resource); + resourceAttributes.setSubresource(resource.subResource); + resourceAttributes.setGroup(resource.apiGroup); } if (null != resourceName) { @@ -168,25 +232,4 @@ private V1ResourceAttributes prepareResourceAttributes(Operation operation, Reso LOGGER.exiting(resourceAttributes); return resourceAttributes; } - - private String getApiGroup(Resource resource) { - if (resource == Resource.domains) { - return "weblogic.oracle"; - } - - if (resource == Resource.customresourcedefinitions) { - return "apiextensions.k8s.io"; - } - - if (resource == Resource.tokenreviews) { - return "authentication.k8s.io"; - } - - if (resource == Resource.ingresses) { - return "extensions"; - } - - // TODO - do we need to specify the api group for any of the other Resource values? - return null; - } } diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java index 760121c94a1..346638c1b84 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java @@ -36,6 +36,7 @@ import io.kubernetes.client.models.V1Pod; import io.kubernetes.client.models.V1PodList; import io.kubernetes.client.models.V1Secret; +import io.kubernetes.client.models.V1SelfSubjectAccessReview; import io.kubernetes.client.models.V1Service; import io.kubernetes.client.models.V1ServiceList; import io.kubernetes.client.models.V1Status; @@ -1384,6 +1385,41 @@ public Step createSubjectAccessReviewAsync(V1SubjectAccessReview body, ResponseS return createRequestAsync(responseStep, new RequestParams("createSubjectAccessReview", null, null, body), CREATE_SUBJECTACCESSREVIEW); } + /* Self Subject Access Review */ + + /** + * Create self subject access review + * @param body Body + * @return Created self subject access review + * @throws ApiException API Exception + */ + public V1SelfSubjectAccessReview createSelfSubjectAccessReview(V1SelfSubjectAccessReview body) throws ApiException { + ApiClient client = helper.take(); + try { + return new AuthorizationV1Api(client).createSelfSubjectAccessReview(body, pretty); + } finally { + helper.recycle(client); + } + } + + private com.squareup.okhttp.Call createSelfSubjectAccessReviewAsync(ApiClient client, V1SelfSubjectAccessReview body, ApiCallback callback) throws ApiException { + return new AuthorizationV1Api(client).createSelfSubjectAccessReviewAsync(body, pretty, callback); + } + + private final CallFactory CREATE_SELFSUBJECTACCESSREVIEW = (requestParams, usage, cont, callback) -> { + return createSelfSubjectAccessReviewAsync(usage, (V1SelfSubjectAccessReview) requestParams.body, callback); + }; + + /** + * Asynchronous step for creating self subject access review + * @param body Body + * @param responseStep Response step for when call completes + * @return Asynchronous step + */ + public Step createSelfSubjectAccessReviewAsync(V1SelfSubjectAccessReview body, ResponseStep responseStep) { + return createRequestAsync(responseStep, new RequestParams("createSelfSubjectAccessReview", null, null, body), CREATE_SELFSUBJECTACCESSREVIEW); + } + /* Token Review */ /** diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/ClientFactory.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/ClientFactory.java new file mode 100644 index 00000000000..a0133eb167f --- /dev/null +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/ClientFactory.java @@ -0,0 +1,12 @@ +// Copyright 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.helpers; + +import java.util.function.Supplier; + +import io.kubernetes.client.ApiClient; + +public interface ClientFactory extends Supplier { + +} diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/ClientPool.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/ClientPool.java index 8ecc7cbc7d2..b37ee60ce89 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/ClientPool.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/ClientPool.java @@ -9,15 +9,18 @@ import oracle.kubernetes.operator.logging.LoggingFacade; import oracle.kubernetes.operator.logging.LoggingFactory; import oracle.kubernetes.operator.logging.MessageKeys; +import oracle.kubernetes.operator.work.Container; +import oracle.kubernetes.operator.work.ContainerResolver; +import java.io.IOException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; public class ClientPool extends Pool { private static final LoggingFacade LOGGER = LoggingFactory.getLogger("Operator", "Operator"); private static final ClientPool SINGLETON = new ClientPool(); - - private final AtomicBoolean first = new AtomicBoolean(true); + + private static final DefaultClientFactory FACTORY = new DefaultClientFactory(); public static ClientPool getInstance() { return SINGLETON; @@ -37,10 +40,16 @@ private ApiClient getApiClient() { ApiClient client = null; LOGGER.fine(MessageKeys.CREATING_API_CLIENT); try { - client = Config.defaultClient(); - if (first.getAndSet(false)) { - Configuration.setDefaultApiClient(client); + ClientFactory factory = null; + Container c = ContainerResolver.getInstance().getContainer(); + if (c != null) { + factory = c.getSPI(ClientFactory.class); + } + if (factory == null) { + factory = FACTORY; } + + client = factory.get(); } catch (Throwable e) { LOGGER.warning(MessageKeys.EXCEPTION, e); } @@ -53,4 +62,22 @@ private ApiClient getApiClient() { return client; } + private static class DefaultClientFactory implements ClientFactory { + private final AtomicBoolean first = new AtomicBoolean(true); + + @Override + public ApiClient get() { + ApiClient client; + try { + client = Config.defaultClient(); + if (first.getAndSet(false)) { + Configuration.setDefaultApiClient(client); + } + return client; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + } } \ No newline at end of file diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/HealthCheckHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/HealthCheckHelper.java index 4aa636610e4..1d60b1a2a4c 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/HealthCheckHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/HealthCheckHelper.java @@ -44,84 +44,29 @@ public class HealthCheckHelper { AuthorizationProxy.Operation.delete, AuthorizationProxy.Operation.deletecollection}; - private static final AuthorizationProxy.Operation[] domainOperations = { + private static final AuthorizationProxy.Operation[] cOperations = { + AuthorizationProxy.Operation.create}; + + private static final AuthorizationProxy.Operation[] glOperations = { AuthorizationProxy.Operation.get, - AuthorizationProxy.Operation.list, - AuthorizationProxy.Operation.watch, - AuthorizationProxy.Operation.update, - AuthorizationProxy.Operation.patch}; + AuthorizationProxy.Operation.list}; - private static final AuthorizationProxy.Operation[] tokenReviewOperations = { + private static final AuthorizationProxy.Operation[] glwOperations = { AuthorizationProxy.Operation.get, AuthorizationProxy.Operation.list, AuthorizationProxy.Operation.watch}; - private static final AuthorizationProxy.Operation[] ingressOperations = { + private static final AuthorizationProxy.Operation[] glwupOperations = { AuthorizationProxy.Operation.get, AuthorizationProxy.Operation.list, AuthorizationProxy.Operation.watch, - AuthorizationProxy.Operation.create, AuthorizationProxy.Operation.update, - AuthorizationProxy.Operation.patch, - AuthorizationProxy.Operation.delete, - AuthorizationProxy.Operation.deletecollection}; - - private static final AuthorizationProxy.Operation[] namespaceOperations = { - AuthorizationProxy.Operation.get, - AuthorizationProxy.Operation.list, - AuthorizationProxy.Operation.watch}; + AuthorizationProxy.Operation.patch}; - private static final AuthorizationProxy.Operation[] persistentVolumeOperations = { - AuthorizationProxy.Operation.get, - AuthorizationProxy.Operation.list, - AuthorizationProxy.Operation.watch}; - - private static final AuthorizationProxy.Operation[] persistentVolumeClaimOperations = { - AuthorizationProxy.Operation.get, - AuthorizationProxy.Operation.list, - AuthorizationProxy.Operation.watch}; - - private static final AuthorizationProxy.Operation[] secretsOperations = { - AuthorizationProxy.Operation.get, - AuthorizationProxy.Operation.list, - AuthorizationProxy.Operation.watch}; - - private static final AuthorizationProxy.Operation[] serviceOperations = { - AuthorizationProxy.Operation.get, - AuthorizationProxy.Operation.list, - AuthorizationProxy.Operation.watch, - AuthorizationProxy.Operation.create, - AuthorizationProxy.Operation.update, - AuthorizationProxy.Operation.patch, - AuthorizationProxy.Operation.delete, - AuthorizationProxy.Operation.deletecollection}; - - private static final AuthorizationProxy.Operation[] podOperations = { - AuthorizationProxy.Operation.get, - AuthorizationProxy.Operation.list, - AuthorizationProxy.Operation.watch, - AuthorizationProxy.Operation.create, - AuthorizationProxy.Operation.update, - AuthorizationProxy.Operation.patch, - AuthorizationProxy.Operation.delete, - AuthorizationProxy.Operation.deletecollection}; - - private static final AuthorizationProxy.Operation[] networkPoliciesOperations = { - AuthorizationProxy.Operation.get, - AuthorizationProxy.Operation.list, - AuthorizationProxy.Operation.watch, - AuthorizationProxy.Operation.create, - AuthorizationProxy.Operation.update, - AuthorizationProxy.Operation.patch, - AuthorizationProxy.Operation.delete, - AuthorizationProxy.Operation.deletecollection}; // default namespace or svc account name private static final String DEFAULT_NAMESPACE = "default"; - // API Server command constants - private static final String SVC_ACCOUNT_PREFIX = "system:serviceaccount:"; - private static final String DOMAIN_UID_LABEL = "weblogic.domainUID"; private static final String MINIMUM_K8S_VERSION = "v1.7.5"; private static final String DOMAIN_IMAGE = "store/oracle/weblogic:12.2.1.3"; @@ -140,22 +85,34 @@ public HealthCheckHelper(String operatorNamespace, Collection targetName // Initialize access checks to be performed // CRUD resources - namespaceAccessChecks.put(AuthorizationProxy.Resource.pods, podOperations); - namespaceAccessChecks.put(AuthorizationProxy.Resource.services, serviceOperations); - namespaceAccessChecks.put(AuthorizationProxy.Resource.ingresses, ingressOperations); - namespaceAccessChecks.put(AuthorizationProxy.Resource.networkpolicies, networkPoliciesOperations); - clusterAccessChecks.put(AuthorizationProxy.Resource.customresourcedefinitions, crudOperations); + namespaceAccessChecks.put(AuthorizationProxy.Resource.PODS, crudOperations); + namespaceAccessChecks.put(AuthorizationProxy.Resource.PODPRESETS, crudOperations); + namespaceAccessChecks.put(AuthorizationProxy.Resource.PODTEMPLATES, crudOperations); + namespaceAccessChecks.put(AuthorizationProxy.Resource.SERVICES, crudOperations); + namespaceAccessChecks.put(AuthorizationProxy.Resource.CONFIGMAPS, crudOperations); + namespaceAccessChecks.put(AuthorizationProxy.Resource.EVENTS, crudOperations); + namespaceAccessChecks.put(AuthorizationProxy.Resource.JOBS, crudOperations); + namespaceAccessChecks.put(AuthorizationProxy.Resource.CRONJOBS, crudOperations); + namespaceAccessChecks.put(AuthorizationProxy.Resource.PERSISTENTVOLUMECLAIMS, crudOperations); + namespaceAccessChecks.put(AuthorizationProxy.Resource.NETWORKPOLICIES, crudOperations); + namespaceAccessChecks.put(AuthorizationProxy.Resource.PODSECURITYPOLICIES, crudOperations); + + namespaceAccessChecks.put(AuthorizationProxy.Resource.LOGS, glOperations); + namespaceAccessChecks.put(AuthorizationProxy.Resource.EXEC, cOperations); - clusterAccessChecks.put(AuthorizationProxy.Resource.domains, domainOperations); + clusterAccessChecks.put(AuthorizationProxy.Resource.DOMAINS, glwupOperations); + + clusterAccessChecks.put(AuthorizationProxy.Resource.CRDS, crudOperations); + clusterAccessChecks.put(AuthorizationProxy.Resource.INGRESSES, crudOperations); + clusterAccessChecks.put(AuthorizationProxy.Resource.PERSISTENTVOLUMES, crudOperations); // Readonly resources - clusterAccessChecks.put(AuthorizationProxy.Resource.namespaces, namespaceOperations); - clusterAccessChecks.put(AuthorizationProxy.Resource.persistentvolumes, persistentVolumeOperations); - namespaceAccessChecks.put(AuthorizationProxy.Resource.secrets, secretsOperations); - namespaceAccessChecks.put(AuthorizationProxy.Resource.persistentvolumeclaims, persistentVolumeClaimOperations); + clusterAccessChecks.put(AuthorizationProxy.Resource.NAMESPACES, glwOperations); + namespaceAccessChecks.put(AuthorizationProxy.Resource.SECRETS, glwOperations); + namespaceAccessChecks.put(AuthorizationProxy.Resource.STORAGECLASSES, glwOperations); // tokenreview - namespaceAccessChecks.put(AuthorizationProxy.Resource.tokenreviews, tokenReviewOperations); + namespaceAccessChecks.put(AuthorizationProxy.Resource.TOKENREVIEWS, cOperations); } /** @@ -174,35 +131,25 @@ public void performNonSecurityChecks() throws ApiException { /** * Verify Access. * - * @param svcAccountName service account for checking access * @throws ApiException exception for k8s API **/ - public void performSecurityChecks(String svcAccountName) throws ApiException { + public void performSecurityChecks() throws ApiException { // Validate namespace if (DEFAULT_NAMESPACE.equals(operatorNamespace)) { LOGGER.info(MessageKeys.NAMESPACE_IS_DEFAULT); } - // Validate svc account - String principalName = svcAccountName; - if (svcAccountName == null || DEFAULT_NAMESPACE.equals(svcAccountName)) { - LOGGER.info(MessageKeys.SVC_ACCOUNT_IS_DEFAULT); - principalName = DEFAULT_NAMESPACE; - } - - String fullName = SVC_ACCOUNT_PREFIX + operatorNamespace + ":" + principalName; - - // Validate RBAC or ABAC policies allow service account to perform required opertions + // Validate RBAC or ABAC policies allow service account to perform required operations AuthorizationProxy ap = new AuthorizationProxy(); - LOGGER.info(MessageKeys.VERIFY_ACCESS_START, svcAccountName); + LOGGER.info(MessageKeys.VERIFY_ACCESS_START); for (AuthorizationProxy.Resource r : namespaceAccessChecks.keySet()) { for (AuthorizationProxy.Operation op : namespaceAccessChecks.get(r)) { for (String ns : targetNamespaces) { - if (!ap.check(fullName, op, r, null, AuthorizationProxy.Scope.namespace, ns)) { - logHealthCheckEvent(MessageKeys.VERIFY_ACCESS_DENIED, fullName, op, r); + if (!ap.check(op, r, null, AuthorizationProxy.Scope.namespace, ns)) { + LOGGER.warning(MessageKeys.VERIFY_ACCESS_DENIED, op, r.getResource()); } } } @@ -211,8 +158,8 @@ public void performSecurityChecks(String svcAccountName) throws ApiException { for (AuthorizationProxy.Resource r : clusterAccessChecks.keySet()) { for (AuthorizationProxy.Operation op : clusterAccessChecks.get(r)) { - if (!ap.check(fullName, op, r, null, AuthorizationProxy.Scope.cluster, null)) { - logHealthCheckEvent(MessageKeys.VERIFY_ACCESS_DENIED, fullName, op, r); + if (!ap.check(op, r, null, AuthorizationProxy.Scope.cluster, null)) { + LOGGER.warning(MessageKeys.VERIFY_ACCESS_DENIED, op, r.getResource()); } } } @@ -280,7 +227,7 @@ public KubernetesVersion performK8sVersionCheck() throws ApiException { // Minimum k8s version not satisfied. if (!k8sMinVersion) { - logHealthCheckEvent(MessageKeys.K8S_MIN_VERSION_CHECK_FAILED, MINIMUM_K8S_VERSION, gitVersion); + LOGGER.warning(MessageKeys.K8S_MIN_VERSION_CHECK_FAILED, MINIMUM_K8S_VERSION, gitVersion); } else { LOGGER.info(MessageKeys.K8S_VERSION_CHECK, gitVersion); } @@ -310,7 +257,7 @@ private HashMap verifyDomainUidUniqueness() throws ApiException Domain domain2 = domainUIDMap.put(domain.getSpec().getDomainUID(), domain); // Domain UID already exist if not null if (domain2 != null) { - logHealthCheckEvent(MessageKeys.DOMAIN_UID_UNIQUENESS_FAILED, domain.getSpec().getDomainUID(), + LOGGER.warning(MessageKeys.DOMAIN_UID_UNIQUENESS_FAILED, domain.getSpec().getDomainUID(), domain.getMetadata().getName(), domain2.getMetadata().getName()); } } @@ -348,7 +295,7 @@ private void verifyPersistentVolume(HashMap domainUIDMap) throws // Persistent volume does not have ReadWriteMany access mode, if (!foundAccessMode) { - logHealthCheckEvent(MessageKeys.PV_ACCESS_MODE_FAILED, pv.getMetadata().getName(), + LOGGER.warning(MessageKeys.PV_ACCESS_MODE_FAILED, pv.getMetadata().getName(), domain.getMetadata().getName(), domainUID, READ_WRITE_MANY_ACCESS); } //TODO: Should we verify the claim, also? @@ -357,7 +304,7 @@ private void verifyPersistentVolume(HashMap domainUIDMap) throws // Persistent volume for domain UID not found if (!foundLabel) { - logHealthCheckEvent(MessageKeys.PV_NOT_FOUND_FOR_DOMAIN_UID, domain.getMetadata().getName(), domainUID); + LOGGER.warning(MessageKeys.PV_NOT_FOUND_FOR_DOMAIN_UID, domain.getMetadata().getName(), domainUID); } } @@ -374,7 +321,7 @@ private void verifyDomainImage(HashMap domainUIDMap) throws ApiE for (Domain domain : domainUIDMap.values()) { LOGGER.finest(MessageKeys.WEBLOGIC_DOMAIN, domain.toString()); if (!domain.getSpec().getImage().equals(DOMAIN_IMAGE)) { - logHealthCheckEvent(MessageKeys.DOMAIN_IMAGE_FAILED, DOMAIN_IMAGE, domain.getSpec().getImage()); + LOGGER.warning(MessageKeys.DOMAIN_IMAGE_FAILED, DOMAIN_IMAGE, domain.getSpec().getImage()); } } } @@ -411,14 +358,4 @@ private boolean isAdminServerRunning(Domain domain) { // TODO: Need to call API to find if an Admin server is running for a given domain return true; } - - /** - * Log health check failures. - * - * @param msg the message to log - * @param params varargs list of objects to include in the log message - */ - private void logHealthCheckEvent(String msg, Object... params) { - LOGGER.warning(msg, params); - } } diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/Pool.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/Pool.java index d50924c4494..d77d20875e5 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/Pool.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/Pool.java @@ -74,4 +74,10 @@ public final void recycle(T instance) { */ protected abstract T create(); + /** + * Drains pool of all entries; useful for unit-testing + */ + public void drain() { + getQueue().clear(); + } } diff --git a/operator/src/main/java/oracle/kubernetes/operator/rest/RestBackendImpl.java b/operator/src/main/java/oracle/kubernetes/operator/rest/RestBackendImpl.java index d70dd8a2693..0d40a7e602b 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/rest/RestBackendImpl.java +++ b/operator/src/main/java/oracle/kubernetes/operator/rest/RestBackendImpl.java @@ -89,7 +89,7 @@ private void authorize(String domainUID, Operation operation) { userInfo.getUsername(), userInfo.getGroups(), operation, - Resource.domains, + Resource.DOMAINS, null, Scope.cluster, null @@ -100,7 +100,7 @@ private void authorize(String domainUID, Operation operation) { userInfo.getUsername(), userInfo.getGroups(), operation, - Resource.domains, + Resource.DOMAINS, domainUID, Scope.namespace, getNamespace(domainUID) diff --git a/operator/src/main/resources/Operator.properties b/operator/src/main/resources/Operator.properties index b5267a29f22..93541f6bcf5 100644 --- a/operator/src/main/resources/Operator.properties +++ b/operator/src/main/resources/Operator.properties @@ -27,8 +27,8 @@ WLSKO-0025=Got exception {0} while trying to retrieve WLS configuration from adm WLSKO-0026=Fail to parse REST response from WLS. Json response is {0}. Exception is {1} WLSKO-0027=Service URL is {0} WLSKO-0028=No servers configured in WebLogic cluster with name {0} -WLSKO-0029=Verifying that service account {0} can access required operations on required resources -WLSKO-0030=Access denied for service account {0} for operation {1} on resource {2} +WLSKO-0029=Verifying that operator service account can access required operations on required resources +WLSKO-0030=Access denied for operator service account for operation {0} on resource {1} WLSKO-0031=A namespace has not been created for the Oracle WebLogic Server Operator for Kubernetes WLSKO-0032=A service account has not been created for the Oracle WebLogic Server Operator for Kubernetes WLSKO-0033=RBAC authorization mode is not enabled for the Kubernetes API server: {0}. To enable RBAC, start the apiserver with --authorization-mode=RBAC diff --git a/operator/src/test/java/oracle/kubernetes/operator/HealthCheckHelperTest.java b/operator/src/test/java/oracle/kubernetes/operator/HealthCheckHelperTest.java index a400087bb72..d81a2f88c58 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/HealthCheckHelperTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/HealthCheckHelperTest.java @@ -3,14 +3,31 @@ package oracle.kubernetes.operator; +import io.kubernetes.client.ApiClient; import io.kubernetes.client.ApiException; +import io.kubernetes.client.apis.CoreV1Api; +import io.kubernetes.client.apis.RbacAuthorizationV1beta1Api; import io.kubernetes.client.models.V1Namespace; import io.kubernetes.client.models.V1ObjectMeta; +import io.kubernetes.client.models.V1ServiceAccount; +import io.kubernetes.client.models.V1beta1ClusterRole; +import io.kubernetes.client.models.V1beta1ClusterRoleBinding; +import io.kubernetes.client.models.V1beta1RoleBinding; +import io.kubernetes.client.util.ClientBuilder; +import io.kubernetes.client.util.Config; import oracle.kubernetes.TestUtils; +import oracle.kubernetes.operator.create.CreateOperatorInputs; +import oracle.kubernetes.operator.create.OperatorFiles; +import oracle.kubernetes.operator.create.ParsedWeblogicOperatorSecurityYaml; +import oracle.kubernetes.operator.create.UserProjects; +import oracle.kubernetes.operator.helpers.CallBuilder; import oracle.kubernetes.operator.helpers.CallBuilderFactory; +import oracle.kubernetes.operator.helpers.ClientFactory; +import oracle.kubernetes.operator.helpers.ClientPool; import oracle.kubernetes.operator.helpers.HealthCheckHelper; import oracle.kubernetes.operator.logging.LoggingFacade; import oracle.kubernetes.operator.logging.LoggingFactory; +import oracle.kubernetes.operator.logging.LoggingFormatter; import oracle.kubernetes.operator.work.Component; import oracle.kubernetes.operator.work.ContainerResolver; @@ -18,10 +35,15 @@ import org.junit.Assert; import org.junit.Assume; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; +import static oracle.kubernetes.operator.create.CreateOperatorInputs.readDefaultInputsFile; +import static oracle.kubernetes.operator.create.ExecCreateOperator.execCreateOperator; +import static oracle.kubernetes.operator.create.ExecResultMatcher.succeedsAndPrints; +import static org.hamcrest.MatcherAssert.assertThat; + import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -29,15 +51,16 @@ public class HealthCheckHelperTest { - private HealthCheckHelper defaultHealthCheckHelper; private HealthCheckHelper unitHealthCheckHelper; private final static String UNIT_NAMESPACE = "unit-test-namespace"; private static final LoggingFacade LOGGER = LoggingFactory.getLogger("Operator", "Operator"); private static ByteArrayOutputStream bos = new ByteArrayOutputStream(); - private static Handler hdlr = new StreamHandler(bos, new SimpleFormatter()); + private static Handler hdlr = new StreamHandler(bos, new LoggingFormatter()); private List savedHandlers = new ArrayList<>(); + + private UserProjects userProjects; @Before public void setUp() throws Exception { @@ -46,10 +69,15 @@ public void setUp() throws Exception { if (!TestUtils.isKubernetesAvailable()) return; + ContainerResolver.getInstance().getContainer().getComponents().remove( + ProcessingConstants.MAIN_COMPONENT_NAME); + ClientPool.getInstance().drain(); + createNamespace(UNIT_NAMESPACE); - defaultHealthCheckHelper = new HealthCheckHelper("default", Collections.singleton("default")); unitHealthCheckHelper = new HealthCheckHelper(UNIT_NAMESPACE, Collections.singleton(UNIT_NAMESPACE)); + + userProjects = UserProjects.createUserProjectsDirectory(); } @After @@ -58,64 +86,208 @@ public void tearDown() throws Exception { LOGGER.getUnderlyingLogger().removeHandler(hdlr); // Delete anything we created + if (userProjects != null) { + userProjects.remove(); + } } @Test - @Ignore - public void testDefaultNamespace() throws Exception { - ContainerResolver.getInstance().getContainer().getComponents().put( - ProcessingConstants.MAIN_COMPONENT_NAME, - Component.createFor(new CallBuilderFactory(null))); + public void testAccountNoPrivs() throws Exception { + Assume.assumeTrue(TestUtils.isKubernetesAvailable()); + + // Create service account + ApiClient apiClient= Config.defaultClient(); + CoreV1Api core = new CoreV1Api(apiClient); + V1ServiceAccount alice = new V1ServiceAccount(); + alice.setMetadata(new V1ObjectMeta().name("alice")); + try { + alice = core.createNamespacedServiceAccount(UNIT_NAMESPACE, alice, "false"); + } catch (ApiException api) { + if (api.getCode() != CallBuilder.CONFLICT) { + throw api; + } + } + alice = core.readNamespacedServiceAccount("alice", UNIT_NAMESPACE, "false", false, false); + + /* If we need to authenticate as sa rather than just impersonate + String secretName = alice.getSecrets().get(0).getName(); + V1Secret secret = core.readNamespacedSecret(secretName, UNIT_NAMESPACE, "false", false, false); + String token = new String(secret.getData().get("token"), StandardCharsets.UTF_8); + */ - defaultHealthCheckHelper.performSecurityChecks("default"); - hdlr.flush(); - String logOutput = bos.toString(); - - Assert.assertTrue("Log output did not contain namespace warning", - logOutput.contains("A namespace has not been created for the")); - Assert.assertTrue("Log output did not contain account warning", - logOutput.contains("A service account has not been created")); - Assert.assertFalse("Log output contained Access Denied error", - logOutput.contains("Access denied for service account")); - bos.reset(); - } - - @Ignore - // TODO work out why this test is failing - it is a rbac issue, need to trace where it is coming from - public void testUnitTestNamespace() throws Exception { ContainerResolver.getInstance().getContainer().getComponents().put( ProcessingConstants.MAIN_COMPONENT_NAME, - Component.createFor(new CallBuilderFactory(null))); + Component.createFor( + ClientFactory.class, new ClientFactory() { + @Override + public ApiClient get() { + try { + //return ClientBuilder.standard().setAuthentication( + // new AccessTokenAuthentication(token)).build(); + ApiClient client = ClientBuilder.standard().build(); + client.addDefaultHeader("Impersonate-User", "system:serviceaccount:unit-test-namespace:alice"); + return client; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + }, + new CallBuilderFactory(null))); - unitHealthCheckHelper.performSecurityChecks("weblogic-operator-account"); + ClientPool.getInstance().drain(); + + unitHealthCheckHelper.performSecurityChecks(); hdlr.flush(); String logOutput = bos.toString(); - Assert.assertFalse("Log output did contain namespace warning", - logOutput.contains("A namespace has not been created for the")); - Assert.assertFalse("Log output did contain account warning", - logOutput.contains("A service account has not been created")); - Assert.assertFalse("Log output contained Access Denied error", - logOutput.contains("Access denied for service account")); + Assert.assertTrue("Log output did not contain Access Denied error: " + logOutput, + logOutput.contains("Access denied")); bos.reset(); } @Test - public void testAccountNoPrivs() throws Exception { + public void testAccountPrivs() throws Exception { Assume.assumeTrue(TestUtils.isKubernetesAvailable()); + // Create service account + ApiClient apiClient= Config.defaultClient(); + CoreV1Api core = new CoreV1Api(apiClient); + V1ServiceAccount theo = new V1ServiceAccount(); + theo.setMetadata(new V1ObjectMeta().name("theo")); + try { + theo = core.createNamespacedServiceAccount(UNIT_NAMESPACE, theo, "false"); + } catch (ApiException api) { + if (api.getCode() != CallBuilder.CONFLICT) { + throw api; + } + } + theo = core.readNamespacedServiceAccount("theo", UNIT_NAMESPACE, "false", false, false); + + applySecurity(apiClient, UNIT_NAMESPACE, "theo"); + ContainerResolver.getInstance().getContainer().getComponents().put( ProcessingConstants.MAIN_COMPONENT_NAME, - Component.createFor(new CallBuilderFactory(null))); + Component.createFor( + ClientFactory.class, new ClientFactory() { + @Override + public ApiClient get() { + try { + //return ClientBuilder.standard().setAuthentication( + // new AccessTokenAuthentication(token)).build(); + ApiClient client = ClientBuilder.standard().build(); + client.addDefaultHeader("Impersonate-User", "system:serviceaccount:unit-test-namespace:theo"); + return client; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + }, + new CallBuilderFactory(null))); - unitHealthCheckHelper.performSecurityChecks("unit-test-svc-account-no-privs"); + ClientPool.getInstance().drain(); + + unitHealthCheckHelper.performSecurityChecks(); hdlr.flush(); String logOutput = bos.toString(); - Assert.assertTrue("Log output did not contain Access Denied error", - logOutput.contains("Access denied for service account")); + Assert.assertFalse("Log output must not contain Access Denied error: " + logOutput, + logOutput.contains("Access denied")); bos.reset(); } + + private void applySecurity(ApiClient apiClient, String namespace, String sa) throws Exception { + CreateOperatorInputs inputs = readDefaultInputsFile() + .namespace(namespace) + .targetNamespaces(namespace) + .serviceAccount(sa); + + OperatorFiles operatorFiles = new OperatorFiles(userProjects.getPath(), inputs); + assertThat(execCreateOperator(userProjects.getPath(), inputs), succeedsAndPrints("Completed")); + ParsedWeblogicOperatorSecurityYaml weblogicOperatorSecurityYaml = + new ParsedWeblogicOperatorSecurityYaml(operatorFiles.getWeblogicOperatorSecurityYamlPath(), inputs); + + // apply roles and bindings + V1beta1ClusterRole clusterRole; + V1beta1ClusterRoleBinding clusterRoleBinding; + V1beta1RoleBinding roleBinding; + + RbacAuthorizationV1beta1Api rbac = new RbacAuthorizationV1beta1Api(apiClient); + + clusterRole = weblogicOperatorSecurityYaml.getWeblogicOperatorClusterRole(); + try { + rbac.createClusterRole(clusterRole, "false"); + } catch (ApiException api) { + if (api.getCode() != CallBuilder.CONFLICT) { + throw api; + } + rbac.replaceClusterRole(clusterRole.getMetadata().getName(), clusterRole, "false"); + } + clusterRole = weblogicOperatorSecurityYaml.getWeblogicOperatorClusterRoleNonResource(); + try { + rbac.createClusterRole(clusterRole, "false"); + } catch (ApiException api) { + if (api.getCode() != CallBuilder.CONFLICT) { + throw api; + } + rbac.replaceClusterRole(clusterRole.getMetadata().getName(), clusterRole, "false"); + } + clusterRole = weblogicOperatorSecurityYaml.getWeblogicOperatorNamespaceRole(); + try { + rbac.createClusterRole(clusterRole, "false"); + } catch (ApiException api) { + if (api.getCode() != CallBuilder.CONFLICT) { + throw api; + } + rbac.replaceClusterRole(clusterRole.getMetadata().getName(), clusterRole, "false"); + } + + clusterRoleBinding = weblogicOperatorSecurityYaml.getOperatorRoleBinding(); + try { + rbac.createClusterRoleBinding(clusterRoleBinding, "false"); + } catch (ApiException api) { + if (api.getCode() != CallBuilder.CONFLICT) { + throw api; + } + rbac.replaceClusterRoleBinding(clusterRoleBinding.getMetadata().getName(), clusterRoleBinding, "false"); + } + clusterRoleBinding = weblogicOperatorSecurityYaml.getOperatorRoleBindingNonResource(); + try { + rbac.createClusterRoleBinding(clusterRoleBinding, "false"); + } catch (ApiException api) { + if (api.getCode() != CallBuilder.CONFLICT) { + throw api; + } + rbac.replaceClusterRoleBinding(clusterRoleBinding.getMetadata().getName(), clusterRoleBinding, "false"); + } + clusterRoleBinding = weblogicOperatorSecurityYaml.getOperatorRoleBindingDiscovery(); + try { + rbac.createClusterRoleBinding(clusterRoleBinding, "false"); + } catch (ApiException api) { + if (api.getCode() != CallBuilder.CONFLICT) { + throw api; + } + rbac.replaceClusterRoleBinding(clusterRoleBinding.getMetadata().getName(), clusterRoleBinding, "false"); + } + clusterRoleBinding = weblogicOperatorSecurityYaml.getOperatorRoleBindingAuthDelegator(); + try { + rbac.createClusterRoleBinding(clusterRoleBinding, "false"); + } catch (ApiException api) { + if (api.getCode() != CallBuilder.CONFLICT) { + throw api; + } + rbac.replaceClusterRoleBinding(clusterRoleBinding.getMetadata().getName(), clusterRoleBinding, "false"); + } + + roleBinding = weblogicOperatorSecurityYaml.getWeblogicOperatorRoleBinding(namespace); + try { + rbac.createNamespacedRoleBinding(namespace, roleBinding, "false"); + } catch (ApiException api) { + if (api.getCode() != CallBuilder.CONFLICT) { + throw api; + } + rbac.replaceNamespacedRoleBinding(roleBinding.getMetadata().getName(), namespace, roleBinding, "false"); + } + } // Create a named namespace private V1Namespace createNamespace(String name) throws Exception { diff --git a/operator/src/test/java/oracle/kubernetes/operator/create/CreateOperatorGeneratedFilesBaseTest.java b/operator/src/test/java/oracle/kubernetes/operator/create/CreateOperatorGeneratedFilesBaseTest.java index bb4c7934de4..f7b367ca8f4 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/create/CreateOperatorGeneratedFilesBaseTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/create/CreateOperatorGeneratedFilesBaseTest.java @@ -321,8 +321,12 @@ protected V1beta1ClusterRole getExpectedWeblogicOperatorClusterRole() { .putLabelsItem(OPERATORNAME_LABEL, getInputs().getNamespace())) .addRulesItem(newPolicyRule() .addApiGroupsItem("") - .resources(asList("namespaces", "persistentvolumes")) + .resources(asList("namespaces")) .verbs(asList("get", "list", "watch"))) + .addRulesItem(newPolicyRule() + .addApiGroupsItem("") + .resources(asList("persistentvolumes")) + .verbs(asList("get", "list", "watch", "create", "update", "patch", "delete", "deletecollection"))) .addRulesItem(newPolicyRule() .addApiGroupsItem("apiextensions.k8s.io") .addResourcesItem("customresourcedefinitions") @@ -338,7 +342,15 @@ protected V1beta1ClusterRole getExpectedWeblogicOperatorClusterRole() { .addRulesItem(newPolicyRule() .addApiGroupsItem("extensions") .addResourcesItem("ingresses") - .verbs(asList("get", "list", "watch", "create", "update", "patch", "delete", "deletecollection"))); + .verbs(asList("get", "list", "watch", "create", "update", "patch", "delete", "deletecollection"))) + .addRulesItem(newPolicyRule() + .addApiGroupsItem("authentication.k8s.io") + .addResourcesItem("tokenreviews") + .verbs(asList("create"))) + .addRulesItem(newPolicyRule() + .addApiGroupsItem("authorization.k8s.io") + .resources(asList("selfsubjectaccessreviews", "localsubjectaccessreviews", "subjectaccessreviews", "selfsubjectrulesreviews")) + .verbs(asList("create"))); } @Test @@ -482,7 +494,7 @@ protected V1beta1ClusterRole getExpectedWeblogicOperatorNamespaceRole() { .putLabelsItem(OPERATORNAME_LABEL, getInputs().getNamespace())) .addRulesItem(newPolicyRule() .addApiGroupsItem("") - .resources(asList("secrets", "persistentvolumeclaims")) + .resources(asList("secrets")) .verbs(asList("get", "list", "watch"))) .addRulesItem(newPolicyRule() .addApiGroupsItem("storage.k8s.io") @@ -490,7 +502,7 @@ protected V1beta1ClusterRole getExpectedWeblogicOperatorNamespaceRole() { .verbs(asList("get", "list", "watch"))) .addRulesItem(newPolicyRule() .addApiGroupsItem("") - .resources(asList("services", "configmaps", "pods", "jobs", "events")) + .resources(asList("services", "configmaps", "pods", "podtemplates", "events", "persistentvolumeclaims")) .verbs(asList("get", "list", "watch", "create", "update", "patch", "delete", "deletecollection"))) .addRulesItem(newPolicyRule() .addApiGroupsItem("") @@ -500,6 +512,10 @@ protected V1beta1ClusterRole getExpectedWeblogicOperatorNamespaceRole() { .addApiGroupsItem("") .resources(asList("pods/exec")) .verbs(asList("create"))) + .addRulesItem(newPolicyRule() + .addApiGroupsItem("batch") + .resources(asList("jobs", "cronjobs")) + .verbs(asList("get", "list", "watch", "create", "update", "patch", "delete", "deletecollection"))) .addRulesItem(newPolicyRule() .addApiGroupsItem("settings.k8s.io") .addResourcesItem("podpresets") @@ -565,4 +581,4 @@ protected V1Service getExpectedExternalOperatorService(boolean debuggingEnabled, .putLabelsItem(OPERATORNAME_LABEL, getInputs().getNamespace())) .spec(spec); } -} +} \ No newline at end of file From 6845c1dc70dafe369360981dfd2540406b43bee9 Mon Sep 17 00:00:00 2001 From: Ryan Eberhard Date: Wed, 18 Apr 2018 12:36:50 -0400 Subject: [PATCH 051/186] Optimize health check using self subject rules review --- .../java/oracle/kubernetes/operator/Main.java | 2 +- .../operator/helpers/AuthorizationProxy.java | 16 ++++ .../operator/helpers/CallBuilder.java | 36 ++++++++ .../operator/helpers/HealthCheckHelper.java | 84 ++++++++++++++++++- .../operator/HealthCheckHelperTest.java | 55 +++++++++++- 5 files changed, 186 insertions(+), 7 deletions(-) diff --git a/operator/src/main/java/oracle/kubernetes/operator/Main.java b/operator/src/main/java/oracle/kubernetes/operator/Main.java index 4636a3855bb..6403e1b4074 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/Main.java +++ b/operator/src/main/java/oracle/kubernetes/operator/Main.java @@ -220,7 +220,7 @@ private static void begin() { HealthCheckHelper healthCheck = new HealthCheckHelper(namespace, targetNamespaces); version = healthCheck.performK8sVersionCheck(); healthCheck.performNonSecurityChecks(); - healthCheck.performSecurityChecks(); + healthCheck.performSecurityChecks(version); } catch (ApiException e) { LOGGER.warning(MessageKeys.EXCEPTION, e); } diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/AuthorizationProxy.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/AuthorizationProxy.java index d58606cb16f..756747c88f8 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/AuthorizationProxy.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/AuthorizationProxy.java @@ -10,6 +10,8 @@ import io.kubernetes.client.models.V1ResourceAttributes; import io.kubernetes.client.models.V1SelfSubjectAccessReview; import io.kubernetes.client.models.V1SelfSubjectAccessReviewSpec; +import io.kubernetes.client.models.V1SelfSubjectRulesReview; +import io.kubernetes.client.models.V1SelfSubjectRulesReviewSpec; import io.kubernetes.client.models.V1SubjectAccessReview; import io.kubernetes.client.models.V1SubjectAccessReviewSpec; import io.kubernetes.client.models.V1SubjectAccessReviewStatus; @@ -232,4 +234,18 @@ private V1ResourceAttributes prepareResourceAttributes(Operation operation, Reso LOGGER.exiting(resourceAttributes); return resourceAttributes; } + + public V1SelfSubjectRulesReview review(String namespace) { + V1SelfSubjectRulesReview subjectRulesReview = new V1SelfSubjectRulesReview(); + V1SelfSubjectRulesReviewSpec spec = new V1SelfSubjectRulesReviewSpec(); + spec.setNamespace(namespace); + subjectRulesReview.setSpec(spec); + CallBuilderFactory factory = ContainerResolver.getInstance().getContainer().getSPI(CallBuilderFactory.class); + try { + return factory.create().createSelfSubjectRulesReview(subjectRulesReview); + } catch (ApiException e) { + LOGGER.warning(MessageKeys.EXCEPTION, e); + return null; + } + } } diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java index 346638c1b84..8f42a21ed1c 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java @@ -37,6 +37,7 @@ import io.kubernetes.client.models.V1PodList; import io.kubernetes.client.models.V1Secret; import io.kubernetes.client.models.V1SelfSubjectAccessReview; +import io.kubernetes.client.models.V1SelfSubjectRulesReview; import io.kubernetes.client.models.V1Service; import io.kubernetes.client.models.V1ServiceList; import io.kubernetes.client.models.V1Status; @@ -1420,6 +1421,41 @@ public Step createSelfSubjectAccessReviewAsync(V1SelfSubjectAccessReview body, R return createRequestAsync(responseStep, new RequestParams("createSelfSubjectAccessReview", null, null, body), CREATE_SELFSUBJECTACCESSREVIEW); } + /* Self Subject Rules Review */ + + /** + * Create self subject rules review + * @param body Body + * @return Created self subject rules review + * @throws ApiException API Exception + */ + public V1SelfSubjectRulesReview createSelfSubjectRulesReview(V1SelfSubjectRulesReview body) throws ApiException { + ApiClient client = helper.take(); + try { + return new AuthorizationV1Api(client).createSelfSubjectRulesReview(body, pretty); + } finally { + helper.recycle(client); + } + } + + private com.squareup.okhttp.Call createSelfSubjectRulesReviewAsync(ApiClient client, V1SelfSubjectRulesReview body, ApiCallback callback) throws ApiException { + return new AuthorizationV1Api(client).createSelfSubjectRulesReviewAsync(body, pretty, callback); + } + + private final CallFactory CREATE_SELFSUBJECTRULESREVIEW = (requestParams, usage, cont, callback) -> { + return createSelfSubjectRulesReviewAsync(usage, (V1SelfSubjectRulesReview) requestParams.body, callback); + }; + + /** + * Asynchronous step for creating self subject rules review + * @param body Body + * @param responseStep Response step for when call completes + * @return Asynchronous step + */ + public Step createSelfSubjectRulesReviewAsync(V1SelfSubjectRulesReview body, ResponseStep responseStep) { + return createRequestAsync(responseStep, new RequestParams("createSelfSubjectRulesReview", null, null, body), CREATE_SELFSUBJECTRULESREVIEW); + } + /* Token Review */ /** diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/HealthCheckHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/HealthCheckHelper.java index 1d60b1a2a4c..d8e0c1078ed 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/HealthCheckHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/HealthCheckHelper.java @@ -6,6 +6,9 @@ import io.kubernetes.client.ApiException; import io.kubernetes.client.models.V1PersistentVolume; import io.kubernetes.client.models.V1PersistentVolumeList; +import io.kubernetes.client.models.V1ResourceRule; +import io.kubernetes.client.models.V1SelfSubjectRulesReview; +import io.kubernetes.client.models.V1SubjectRulesReviewStatus; import io.kubernetes.client.models.VersionInfo; import oracle.kubernetes.weblogic.domain.v1.Domain; import oracle.kubernetes.weblogic.domain.v1.DomainList; @@ -130,10 +133,10 @@ public void performNonSecurityChecks() throws ApiException { /** * Verify Access. - * + * @param version Kubernetes version * @throws ApiException exception for k8s API **/ - public void performSecurityChecks() throws ApiException { + public void performSecurityChecks(KubernetesVersion version) throws ApiException { // Validate namespace if (DEFAULT_NAMESPACE.equals(operatorNamespace)) { @@ -143,7 +146,51 @@ public void performSecurityChecks() throws ApiException { // Validate RBAC or ABAC policies allow service account to perform required operations AuthorizationProxy ap = new AuthorizationProxy(); LOGGER.info(MessageKeys.VERIFY_ACCESS_START); - + + if (version.major > 1 || version.minor >= 8) { + boolean rulesReviewSuccessful = true; + for (String ns : targetNamespaces) { + V1SelfSubjectRulesReview review = ap.review(ns); + if (review == null) { + rulesReviewSuccessful = false; + break; + } + + V1SubjectRulesReviewStatus status = review.getStatus(); + List rules = status.getResourceRules(); + + for (AuthorizationProxy.Resource r : namespaceAccessChecks.keySet()) { + for (AuthorizationProxy.Operation op : namespaceAccessChecks.get(r)) { + check(rules, r, op); + } + } + for (AuthorizationProxy.Resource r : clusterAccessChecks.keySet()) { + for (AuthorizationProxy.Operation op : clusterAccessChecks.get(r)) { + check(rules, r, op); + } + } + } + if (!targetNamespaces.contains("default")) { + V1SelfSubjectRulesReview review = ap.review("default"); + if (review == null) { + rulesReviewSuccessful = false; + } else { + V1SubjectRulesReviewStatus status = review.getStatus(); + List rules = status.getResourceRules(); + + for (AuthorizationProxy.Resource r : clusterAccessChecks.keySet()) { + for (AuthorizationProxy.Operation op : clusterAccessChecks.get(r)) { + check(rules, r, op); + } + } + } + } + + if (rulesReviewSuccessful) { + return; + } + } + for (AuthorizationProxy.Resource r : namespaceAccessChecks.keySet()) { for (AuthorizationProxy.Operation op : namespaceAccessChecks.get(r)) { @@ -164,7 +211,38 @@ public void performSecurityChecks() throws ApiException { } } } + + private void check(List rules, AuthorizationProxy.Resource r, AuthorizationProxy.Operation op) { + String verb = op.name(); + String apiGroup = r.getAPIGroup(); + String resource = r.getResource(); + String sub = r.getSubResource(); + if (sub != null && !sub.isEmpty()) { + resource = resource + "/" + sub; + } + for (V1ResourceRule rule : rules) { + List ruleApiGroups = rule.getApiGroups(); + if (apiGroupMatch(ruleApiGroups, apiGroup)) { + List ruleResources = rule.getResources(); + if (ruleResources != null && ruleResources.contains(resource)) { + List ruleVerbs = rule.getVerbs(); + if (ruleVerbs != null && ruleVerbs.contains(verb)) { + return; + } + } + } + } + + LOGGER.warning(MessageKeys.VERIFY_ACCESS_DENIED, op, r.getResource()); + } + private boolean apiGroupMatch(List ruleApiGroups, String apiGroup) { + if (apiGroup == null || apiGroup.isEmpty()) { + return ruleApiGroups == null || ruleApiGroups.isEmpty() || ruleApiGroups.contains(""); + } + return ruleApiGroups != null && ruleApiGroups.contains(apiGroup); + } + /** * Major and minor version of Kubernetes API Server * diff --git a/operator/src/test/java/oracle/kubernetes/operator/HealthCheckHelperTest.java b/operator/src/test/java/oracle/kubernetes/operator/HealthCheckHelperTest.java index d81a2f88c58..3e1c72cab74 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/HealthCheckHelperTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/HealthCheckHelperTest.java @@ -12,7 +12,10 @@ import io.kubernetes.client.models.V1ServiceAccount; import io.kubernetes.client.models.V1beta1ClusterRole; import io.kubernetes.client.models.V1beta1ClusterRoleBinding; +import io.kubernetes.client.models.V1beta1PolicyRule; import io.kubernetes.client.models.V1beta1RoleBinding; +import io.kubernetes.client.models.V1beta1RoleRef; +import io.kubernetes.client.models.V1beta1Subject; import io.kubernetes.client.util.ClientBuilder; import io.kubernetes.client.util.Config; import oracle.kubernetes.TestUtils; @@ -25,6 +28,7 @@ import oracle.kubernetes.operator.helpers.ClientFactory; import oracle.kubernetes.operator.helpers.ClientPool; import oracle.kubernetes.operator.helpers.HealthCheckHelper; +import oracle.kubernetes.operator.helpers.HealthCheckHelper.KubernetesVersion; import oracle.kubernetes.operator.logging.LoggingFacade; import oracle.kubernetes.operator.logging.LoggingFactory; import oracle.kubernetes.operator.logging.LoggingFormatter; @@ -51,6 +55,7 @@ public class HealthCheckHelperTest { + private KubernetesVersion version; private HealthCheckHelper unitHealthCheckHelper; private final static String UNIT_NAMESPACE = "unit-test-namespace"; @@ -76,7 +81,7 @@ public void setUp() throws Exception { createNamespace(UNIT_NAMESPACE); unitHealthCheckHelper = new HealthCheckHelper(UNIT_NAMESPACE, Collections.singleton(UNIT_NAMESPACE)); - + userProjects = UserProjects.createUserProjectsDirectory(); } @@ -115,6 +120,8 @@ public void testAccountNoPrivs() throws Exception { String token = new String(secret.getData().get("token"), StandardCharsets.UTF_8); */ + applyMinimumSecurity(apiClient, UNIT_NAMESPACE, "alice"); + ContainerResolver.getInstance().getContainer().getComponents().put( ProcessingConstants.MAIN_COMPONENT_NAME, Component.createFor( @@ -136,7 +143,8 @@ public ApiClient get() { ClientPool.getInstance().drain(); - unitHealthCheckHelper.performSecurityChecks(); + version = unitHealthCheckHelper.performK8sVersionCheck(); + unitHealthCheckHelper.performSecurityChecks(version); hdlr.flush(); String logOutput = bos.toString(); @@ -186,7 +194,8 @@ public ApiClient get() { ClientPool.getInstance().drain(); - unitHealthCheckHelper.performSecurityChecks(); + version = unitHealthCheckHelper.performK8sVersionCheck(); + unitHealthCheckHelper.performSecurityChecks(version); hdlr.flush(); String logOutput = bos.toString(); @@ -195,6 +204,46 @@ public ApiClient get() { bos.reset(); } + private void applyMinimumSecurity(ApiClient apiClient, String namespace, String sa) throws Exception { + V1beta1ClusterRole clusterRole = new V1beta1ClusterRole(); + clusterRole.setMetadata(new V1ObjectMeta() + .name("test-role") + .putLabelsItem("weblogic.operatorName", namespace)); + clusterRole.addRulesItem(new V1beta1PolicyRule().addNonResourceURLsItem("/version/*").addVerbsItem("get")); + clusterRole.addRulesItem(new V1beta1PolicyRule().addResourcesItem("selfsubjectrulesreviews") + .addApiGroupsItem("authorization.k8s.io").addVerbsItem("create")); + V1beta1ClusterRoleBinding clusterRoleBinding = new V1beta1ClusterRoleBinding(); + clusterRoleBinding.setMetadata(new V1ObjectMeta() + .name(namespace + "-test-role") + .putLabelsItem("weblogic.operatorName", namespace)); + clusterRoleBinding.addSubjectsItem(new V1beta1Subject() + .kind("ServiceAccount") + .apiGroup("") + .name(sa).namespace(namespace)); + clusterRoleBinding.roleRef(new V1beta1RoleRef() + .kind("ClusterRole") + .apiGroup("rbac.authorization.k8s.io") + .name("test-role")); + RbacAuthorizationV1beta1Api rbac = new RbacAuthorizationV1beta1Api(apiClient); + + try { + rbac.createClusterRole(clusterRole, "false"); + } catch (ApiException api) { + if (api.getCode() != CallBuilder.CONFLICT) { + throw api; + } + rbac.replaceClusterRole(clusterRole.getMetadata().getName(), clusterRole, "false"); + } + try { + rbac.createClusterRoleBinding(clusterRoleBinding, "false"); + } catch (ApiException api) { + if (api.getCode() != CallBuilder.CONFLICT) { + throw api; + } + rbac.replaceClusterRoleBinding(clusterRoleBinding.getMetadata().getName(), clusterRoleBinding, "false"); + } + } + private void applySecurity(ApiClient apiClient, String namespace, String sa) throws Exception { CreateOperatorInputs inputs = readDefaultInputsFile() .namespace(namespace) From 12a6941df2c5d3efd1d495e58b3ec1c21058d75d Mon Sep 17 00:00:00 2001 From: Lily He Date: Wed, 18 Apr 2018 15:38:33 -0700 Subject: [PATCH 052/186] add loop to verify voyager resources --- kubernetes/internal/create-weblogic-domain.sh | 59 ++++++++++++++----- 1 file changed, 44 insertions(+), 15 deletions(-) diff --git a/kubernetes/internal/create-weblogic-domain.sh b/kubernetes/internal/create-weblogic-domain.sh index 8bbcfd00b10..a98984e8255 100755 --- a/kubernetes/internal/create-weblogic-domain.sh +++ b/kubernetes/internal/create-weblogic-domain.sh @@ -611,25 +611,54 @@ function createDomain { # Deploy Voyager/HAProxy load balancer # function setupVoyagerLoadBalancer { - # deploy Voyager Ingress controller - kubectl create namespace voyager - curl -fsSL https://raw.githubusercontent.com/appscode/voyager/6.0.0/hack/deploy/voyager.sh \ - | bash -s -- --provider=baremetal --namespace=voyager + # only deploy Voyager Ingress Controller the first time + local vcon=`kubectl get namespace voyager| grep voyager | wc | awk ' { print $1; } '` + if [ "$vcon" == "0" ]; then + kubectl create namespace voyager + curl -fsSL https://raw.githubusercontent.com/appscode/voyager/6.0.0/hack/deploy/voyager.sh \ + | bash -s -- --provider=baremetal --namespace=voyager + fi + + # verify Voyager controller pod is ready + local ready=`kubectl -n voyager get pod | grep voyager-operator | awk ' { print $2; } '` + if [ "${ready}" != "1/1" ] ; then + fail "Voyager Ingress Controller is not ready" + fi # deploy Voyager Ingress resource kubectl apply -f ${voyagerOutput} - echo Checking Voyager deploy - vdep=`kubectl get deploy -n ${namespace} | grep voyager | wc | awk ' { print $1; } '` - if [ "$vdep" != "1" ]; then - fail "The deployment of Voyager Ingress was not created" - fi - -echo Checking Voyager service - vscv=`kubectl get service ${domainUID}-voyager-stats -n ${namespace} | grep voyager-stats | wc | awk ' { print $1; } '` - if [ "$vscv" != "1" ]; then - fail "The service voyager-stats was not created" - fi + echo Checking Voyager deploy + local maxwaitsecs=100 + local mstart=`date +%s` + while : ; do + local mnow=`date +%s` + local vdep=`kubectl get deploy -n ${namespace} | grep ${domainUID}-voyager | wc | awk ' { print $1; } '` + if [ "$vdep" = "1" ]; then + echo 'The deployment ${domainUID}-voyager is created successful.' + break + fi + if [ $((mnow - mstart)) -gt $((maxwaitsecs)) ]; then + fail "The deployment ${domainUID}-voyager was not created." + fi + sleep 5 + done + + echo Checking Voyager service + local maxwaitsecs=100 + local mstart=`date +%s` + while : ; do + local mnow=`date +%s` + local vscv=`kubectl get service ${domainUID}-voyager-stats -n ${namespace} | grep ${domainUID}-voyager-stats | wc | awk ' { print $1; } '` + if [ "$vscv" = "1" ]; then + echo 'The service ${domainUID}-voyager-stats is created successful.' + break + fi + if [ $((mnow - mstart)) -gt $((maxwaitsecs)) ]; then + fail "The service ${domainUID}-voyager-stats was not created." + fi + sleep 5 + done } # # Deploy traefik load balancer From 2a7dbb48ca5a1ff82790f510c95e2b7a00e94a4d Mon Sep 17 00:00:00 2001 From: Russell Gold Date: Thu, 19 Apr 2018 10:58:21 -0400 Subject: [PATCH 053/186] Remove unused code --- .../operator/helpers/CallBuilder.java | 1402 +++-------------- 1 file changed, 194 insertions(+), 1208 deletions(-) diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java index c95f04e0357..02eb263e720 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java @@ -3,17 +3,6 @@ package oracle.kubernetes.operator.helpers; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Consumer; - -import com.squareup.okhttp.Call; - import io.kubernetes.client.ApiCallback; import io.kubernetes.client.ApiClient; import io.kubernetes.client.ApiException; @@ -24,47 +13,35 @@ import io.kubernetes.client.apis.CoreV1Api; import io.kubernetes.client.apis.ExtensionsV1beta1Api; import io.kubernetes.client.apis.VersionApi; -import io.kubernetes.client.models.V1ConfigMap; -import io.kubernetes.client.models.V1ConfigMapList; -import io.kubernetes.client.models.V1DeleteOptions; -import io.kubernetes.client.models.V1Event; -import io.kubernetes.client.models.V1EventList; -import io.kubernetes.client.models.V1Job; -import io.kubernetes.client.models.V1JobList; -import io.kubernetes.client.models.V1ListMeta; -import io.kubernetes.client.models.V1Namespace; -import io.kubernetes.client.models.V1NamespaceList; -import io.kubernetes.client.models.V1PersistentVolume; -import io.kubernetes.client.models.V1PersistentVolumeClaimList; -import io.kubernetes.client.models.V1PersistentVolumeList; -import io.kubernetes.client.models.V1Pod; -import io.kubernetes.client.models.V1PodList; -import io.kubernetes.client.models.V1Secret; -import io.kubernetes.client.models.V1Service; -import io.kubernetes.client.models.V1ServiceList; -import io.kubernetes.client.models.V1Status; -import io.kubernetes.client.models.V1SubjectAccessReview; -import io.kubernetes.client.models.V1TokenReview; -import io.kubernetes.client.models.V1beta1CustomResourceDefinition; -import io.kubernetes.client.models.V1beta1Ingress; -import io.kubernetes.client.models.V1beta1IngressList; -import io.kubernetes.client.models.VersionInfo; -import oracle.kubernetes.weblogic.domain.v1.Domain; -import oracle.kubernetes.weblogic.domain.v1.DomainList; -import oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi; +import io.kubernetes.client.models.*; import oracle.kubernetes.operator.TuningParameters.CallBuilderTuning; import oracle.kubernetes.operator.logging.LoggingFacade; import oracle.kubernetes.operator.logging.LoggingFactory; import oracle.kubernetes.operator.logging.MessageKeys; -import oracle.kubernetes.operator.work.Step; import oracle.kubernetes.operator.work.Component; import oracle.kubernetes.operator.work.NextAction; import oracle.kubernetes.operator.work.Packet; +import oracle.kubernetes.operator.work.Step; +import oracle.kubernetes.weblogic.domain.v1.Domain; +import oracle.kubernetes.weblogic.domain.v1.DomainList; +import oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi; + +import com.squareup.okhttp.Call; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; /** * Simplifies synchronous and asynchronous call patterns to the Kubernetes API Server. * */ +@SuppressWarnings({"WeakerAccess", "UnusedReturnValue"}) public class CallBuilder { private static final LoggingFacade LOGGER = LoggingFactory.getLogger("Operator", "Operator"); @@ -74,10 +51,6 @@ public class CallBuilder { * HTTP status code for "Not Found" */ public static final int NOT_FOUND = 404; - /** - * HTTP status code for "Conflict" - */ - public static final int CONFLICT = 409; public String pretty = "false"; public String fieldSelector = ""; @@ -163,41 +136,7 @@ public VersionInfo readVersionCode() throws ApiException { } /* Namespaces */ - - /** - * List namespaces - * @return List of namespaces - * @throws ApiException API Exception - */ - public V1NamespaceList listNamespace() throws ApiException { - String _continue = ""; - ApiClient client = helper.take(); - try { - return new CoreV1Api(client).listNamespace(pretty, _continue, fieldSelector, - includeUninitialized, labelSelector, limit, resourceVersion, timeoutSeconds, watch); - } finally { - helper.recycle(client); - } - } - private com.squareup.okhttp.Call listNamespaceAsync(ApiClient client, String _continue, ApiCallback callback) throws ApiException { - return new CoreV1Api(client).listNamespaceAsync(pretty, _continue, - fieldSelector, includeUninitialized, labelSelector, limit, resourceVersion, timeoutSeconds, watch, callback); - } - - private final CallFactory LIST_NAMESPACE = (requestParams, usage, cont, callback) -> { - return listNamespaceAsync(usage, cont, callback); - }; - - /** - * Asynchronous step for listing namespaces - * @param responseStep Response step for when call completes - * @return Asynchronous step - */ - public Step listNamespaceAsync(ResponseStep responseStep) { - return createRequestAsync(responseStep, new RequestParams("listNamespace", null, null, null), LIST_NAMESPACE); - } - /** * Read namespace * @param name Name @@ -213,24 +152,6 @@ public V1Namespace readNamespace(String name) throws ApiException { } } - private com.squareup.okhttp.Call readNamespaceAsync(ApiClient client, String name, ApiCallback callback) throws ApiException { - return new CoreV1Api(client).readNamespaceAsync(name, pretty, exact, export, callback); - } - - private final CallFactory READ_NAMESPACE = (requestParams, usage, cont, callback) -> { - return readNamespaceAsync(usage, requestParams.name, callback); - }; - - /** - * Asynchronous step for reading namespace - * @param name Name - * @param responseStep Response step for when call completes - * @return Asynchronous step - */ - public Step readNamespaceAsync(String name, ResponseStep responseStep) { - return createRequestAsync(responseStep, new RequestParams("readNamespace", null, name, null), READ_NAMESPACE); - } - /** * Create namespace * @param body Body @@ -246,60 +167,6 @@ public V1Namespace createNamespace(V1Namespace body) throws ApiException { } } - private com.squareup.okhttp.Call createNamespaceAsync(ApiClient client, V1Namespace body, ApiCallback callback) throws ApiException { - return new CoreV1Api(client).createNamespaceAsync(body, pretty, callback); - } - - private final CallFactory CREATE_NAMESPACE = (requestParams, usage, cont, callback) -> { - return createNamespaceAsync(usage, (V1Namespace) requestParams.body, callback); - }; - - /** - * Asynchronous step for creating namespace - * @param body Body - * @param responseStep Response step for when call completes - * @return Asynchronous step - */ - public Step createNamespaceAsync(V1Namespace body, ResponseStep responseStep) { - return createRequestAsync(responseStep, new RequestParams("createNamespace", null, null, body), CREATE_NAMESPACE); - } - - /** - * Delete namespace - * @param name Name - * @param deleteOptions Delete options - * @return Status of deletion - * @throws ApiException API Exception - */ - public V1Status deleteNamespace(String name, V1DeleteOptions deleteOptions) throws ApiException { - ApiClient client = helper.take(); - try { - return new CoreV1Api(client).deleteNamespace(name, deleteOptions, pretty, gracePeriodSeconds, - orphanDependents, propagationPolicy); - } finally { - helper.recycle(client); - } - } - - private com.squareup.okhttp.Call deleteNamespaceAsync(ApiClient client, String name, V1DeleteOptions deleteOptions, ApiCallback callback) throws ApiException { - return new CoreV1Api(client).deleteNamespaceAsync(name, deleteOptions, pretty, gracePeriodSeconds, - orphanDependents, propagationPolicy, callback); - } - - private final CallFactory DELETE_NAMESPACE = (requestParams, usage, cont, callback) -> { - return deleteNamespaceAsync(usage, requestParams.name, (V1DeleteOptions) requestParams.body, callback); - }; - - /** - * Asynchronous step for deleting service - * @param name Name - * @param responseStep Response step for when call completes - * @return Asynchronous step - */ - public Step deleteNamespaceAsync(String name, ResponseStep responseStep) { - return createRequestAsync(responseStep, new RequestParams("deleteNamespace", null, name, null), DELETE_NAMESPACE); - } - /* Domains */ /** @@ -324,9 +191,8 @@ private com.squareup.okhttp.Call listDomainAsync(ApiClient client, String namesp fieldSelector, includeUninitialized, labelSelector, limit, resourceVersion, timeoutSeconds, watch, callback); } - private final CallFactory LIST_DOMAIN = (requestParams, usage, cont, callback) -> { - return listDomainAsync(usage, requestParams.namespace, cont, callback); - }; + private final CallFactory LIST_DOMAIN = (requestParams, usage, cont, callback) + -> listDomainAsync(usage, requestParams.namespace, cont, callback); /** * Asynchronous step for listing domains @@ -359,9 +225,8 @@ private com.squareup.okhttp.Call replaceDomainAsync(ApiClient client, String nam return new WeblogicApi(client).replaceWebLogicOracleV1NamespacedDomainAsync(name, namespace, body, pretty, callback); } - private final CallFactory REPLACE_DOMAIN = (requestParams, usage, cont, callback) -> { - return replaceDomainAsync(usage, requestParams.name, requestParams.namespace, (Domain) requestParams.body, callback); - }; + private final CallFactory REPLACE_DOMAIN = (requestParams, usage, cont, callback) + -> replaceDomainAsync(usage, requestParams.name, requestParams.namespace, (Domain) requestParams.body, callback); /** * Asynchronous step for replacing domain @@ -374,44 +239,7 @@ private com.squareup.okhttp.Call replaceDomainAsync(ApiClient client, String nam public Step replaceDomainAsync(String name, String namespace, Domain body, ResponseStep responseStep) { return createRequestAsync(responseStep, new RequestParams("replaceDomain", namespace, name, body), REPLACE_DOMAIN); } - - /** - * Replace domain status - * @param name Name - * @param namespace Namespace - * @param body Body - * @return Replaced domain - * @throws ApiException APIException - */ - public Domain replaceDomainStatus(String name, String namespace, Domain body) throws ApiException { - ApiClient client = helper.take(); - try { - return new WeblogicApi(client).replaceWebLogicOracleV1NamespacedDomainStatus(name, namespace, body, pretty); - } finally { - helper.recycle(client); - } - } - - private com.squareup.okhttp.Call replaceDomainStatusAsync(ApiClient client, String name, String namespace, Domain body, ApiCallback callback) throws ApiException { - return new WeblogicApi(client).replaceWebLogicOracleV1NamespacedDomainStatusAsync(name, namespace, body, pretty, callback); - } - private final CallFactory REPLACE_STATUS_DOMAIN = (requestParams, usage, cont, callback) -> { - return replaceDomainStatusAsync(usage, requestParams.name, requestParams.namespace, (Domain) requestParams.body, callback); - }; - - /** - * Asynchronous step for replacing domain status - * @param name Name - * @param namespace Namespace - * @param body Body - * @param responseStep Response step for when call completes - * @return Asynchronous step - */ - public Step replaceDomainStatusAsync(String name, String namespace, Domain body, ResponseStep responseStep) { - return createRequestAsync(responseStep, new RequestParams("replaceDomainStatus", namespace, name, body), REPLACE_STATUS_DOMAIN); - } - /* Custom Resource Definitions */ /** @@ -429,24 +257,6 @@ public V1beta1CustomResourceDefinition readCustomResourceDefinition(String name) } } - private com.squareup.okhttp.Call readCustomResourceDefinitionAsync(ApiClient client, String name, ApiCallback callback) throws ApiException { - return new ApiextensionsV1beta1Api(client).readCustomResourceDefinitionAsync(name, pretty, exact, export, callback); - } - - private final CallFactory READ_CUSTOMRESOURCEDEFINITION = (requestParams, usage, cont, callback) -> { - return readCustomResourceDefinitionAsync(usage, requestParams.name, callback); - }; - - /** - * Asynchronous step for reading custom resource definition - * @param name Name - * @param responseStep Response step for when call completes - * @return Asynchronous step - */ - public Step readCustomResourceDefinitionAsync(String name, ResponseStep responseStep) { - return createRequestAsync(responseStep, new RequestParams("readCustomResourceDefinition", null, name, null), READ_CUSTOMRESOURCEDEFINITION); - } - /** * Create custom resource definition * @param body Body @@ -463,86 +273,14 @@ public V1beta1CustomResourceDefinition createCustomResourceDefinition(V1beta1Cus } } - private com.squareup.okhttp.Call createCustomResourceDefinitionAsync(ApiClient client, V1beta1CustomResourceDefinition body, ApiCallback callback) throws ApiException { - return new ApiextensionsV1beta1Api(client).createCustomResourceDefinitionAsync(body, pretty, callback); - } - - private final CallFactory CREATE_CUSTOMRESOURCEDEFINITION = (requestParams, usage, cont, callback) -> { - return createCustomResourceDefinitionAsync(usage, (V1beta1CustomResourceDefinition) requestParams.body, callback); - }; - - /** - * Asynchronous step for creating custom resource definition - * @param name Name - * @param body Body - * @param responseStep Response step for when call completes - * @return Asynchronous step - */ - public Step createCustomResourceDefinitionAsync(String name, V1beta1CustomResourceDefinition body, ResponseStep responseStep) { - return createRequestAsync(responseStep, new RequestParams("createCustomResourceDefinition", null, name, body), CREATE_CUSTOMRESOURCEDEFINITION); - } - /* Config Maps */ - - /** - * List config maps - * @param namespace Namespace - * @return List of config maps - * @throws ApiException API Exception - */ - public V1ConfigMapList listConfigMap(String namespace) throws ApiException { - String _continue = ""; - ApiClient client = helper.take(); - try { - return new CoreV1Api(client).listNamespacedConfigMap(namespace, pretty, _continue, fieldSelector, - includeUninitialized, labelSelector, limit, resourceVersion, timeoutSeconds, watch); - } finally { - helper.recycle(client); - } - } - - private com.squareup.okhttp.Call listConfigMapAsync(ApiClient client, String namespace, String _continue, ApiCallback callback) throws ApiException { - return new CoreV1Api(client).listNamespacedConfigMapAsync(namespace, pretty, _continue, - fieldSelector, includeUninitialized, labelSelector, limit, resourceVersion, timeoutSeconds, watch, callback); - } - - private final CallFactory LIST_CONFIGMAP = (requestParams, usage, cont, callback) -> { - return listConfigMapAsync(usage, requestParams.namespace, cont, callback); - }; - - /** - * Asynchronous step for listing config maps - * @param namespace Namespace - * @param responseStep Response step for when call completes - * @return Asynchronous step - */ - public Step listConfigMapAsync(String namespace, ResponseStep responseStep) { - return createRequestAsync(responseStep, new RequestParams("listConfigMap", namespace, null, null), LIST_CONFIGMAP); - } - - /** - * Read config map - * @param name Name - * @param namespace Namespace - * @return Read config map - * @throws ApiException API Exception - */ - public V1ConfigMap readConfigMap(String name, String namespace) throws ApiException { - ApiClient client = helper.take(); - try { - return new CoreV1Api(client).readNamespacedConfigMap(name, namespace, pretty, exact, export); - } finally { - helper.recycle(client); - } - } private com.squareup.okhttp.Call readConfigMapAsync(ApiClient client, String name, String namespace, ApiCallback callback) throws ApiException { return new CoreV1Api(client).readNamespacedConfigMapAsync(name, namespace, pretty, exact, export, callback); } - private final CallFactory READ_CONFIGMAP = (requestParams, usage, cont, callback) -> { - return readConfigMapAsync(usage, requestParams.name, requestParams.namespace, callback); - }; + private final CallFactory READ_CONFIGMAP = (requestParams, usage, cont, callback) + -> readConfigMapAsync(usage, requestParams.name, requestParams.namespace, callback); /** * Asynchronous step for reading config map @@ -554,30 +292,13 @@ private com.squareup.okhttp.Call readConfigMapAsync(ApiClient client, String nam public Step readConfigMapAsync(String name, String namespace, ResponseStep responseStep) { return createRequestAsync(responseStep, new RequestParams("readConfigMap", namespace, name, null), READ_CONFIGMAP); } - - /** - * Create config map - * @param namespace Namespace - * @param body Body - * @return Created config map - * @throws ApiException API Exception - */ - public V1ConfigMap createConfigMap(String namespace, V1ConfigMap body) throws ApiException { - ApiClient client = helper.take(); - try { - return new CoreV1Api(client).createNamespacedConfigMap(namespace, body, pretty); - } finally { - helper.recycle(client); - } - } private com.squareup.okhttp.Call createConfigMapAsync(ApiClient client, String namespace, V1ConfigMap body, ApiCallback callback) throws ApiException { return new CoreV1Api(client).createNamespacedConfigMapAsync(namespace, body, pretty, callback); } - private final CallFactory CREATE_CONFIGMAP = (requestParams, usage, cont, callback) -> { - return createConfigMapAsync(usage, requestParams.namespace, (V1ConfigMap) requestParams.body, callback); - }; + private final CallFactory CREATE_CONFIGMAP = (requestParams, usage, cont, callback) + -> createConfigMapAsync(usage, requestParams.namespace, (V1ConfigMap) requestParams.body, callback); /** * Asynchronous step for creating config map @@ -589,970 +310,327 @@ private com.squareup.okhttp.Call createConfigMapAsync(ApiClient client, String n public Step createConfigMapAsync(String namespace, V1ConfigMap body, ResponseStep responseStep) { return createRequestAsync(responseStep, new RequestParams("createConfigMap", namespace, null, body), CREATE_CONFIGMAP); } - - /** - * Replace config map - * @param name Name - * @param namespace Namespace - * @param body Body - * @return Replaced config map - * @throws ApiException API Exception - */ - public V1ConfigMap replaceConfigMap(String name, String namespace, V1ConfigMap body) throws ApiException { - ApiClient client = helper.take(); - try { - return new CoreV1Api(client).replaceNamespacedConfigMap(name, namespace, body, pretty); - } finally { - helper.recycle(client); - } - } - - private com.squareup.okhttp.Call replaceConfigMapAsync(ApiClient client, String name, String namespace, V1ConfigMap body, ApiCallback callback) throws ApiException { - return new CoreV1Api(client).replaceNamespacedConfigMapAsync(name, namespace, body, pretty, callback); - } - - private final CallFactory REPLACE_CONFIGMAP = (requestParams, usage, cont, callback) -> { - return replaceConfigMapAsync(usage, requestParams.name, requestParams.namespace, (V1ConfigMap) requestParams.body, callback); - }; - - /** - * Asynchronous step for replacing config map - * @param name Name - * @param namespace Namespace - * @param body Body - * @param responseStep Response step for when call completes - * @return Asynchronous step - */ - public Step replaceConfigMapAsync(String name, String namespace, V1ConfigMap body, ResponseStep responseStep) { - return createRequestAsync(responseStep, new RequestParams("replaceConfigMap", namespace, name, body), REPLACE_CONFIGMAP); - } - - /** - * Delete config map - * @param name Name - * @param namespace Namespace - * @param deleteOptions Delete options - * @return Status of deletion - * @throws ApiException API Exception - */ - public V1Status deleteConfigMap(String name, String namespace, V1DeleteOptions deleteOptions) throws ApiException { - ApiClient client = helper.take(); - try { - return new CoreV1Api(client).deleteNamespacedConfigMap(name, namespace, deleteOptions, pretty, gracePeriodSeconds, - orphanDependents, propagationPolicy); - } finally { - helper.recycle(client); - } - } - - private com.squareup.okhttp.Call deleteConfigMapAsync(ApiClient client, String name, String namespace, V1DeleteOptions deleteOptions, ApiCallback callback) throws ApiException { - return new CoreV1Api(client).deleteNamespacedConfigMapAsync(name, namespace, deleteOptions, pretty, gracePeriodSeconds, orphanDependents, propagationPolicy, callback); - } - - private final CallFactory DELETE_CONFIGMAP = (requestParams, usage, cont, callback) -> { - return deleteConfigMapAsync(usage, requestParams.name, requestParams.namespace, (V1DeleteOptions) requestParams.body, callback); - }; - - /** - * Asynchronous step for deleting config map - * @param name Name - * @param namespace Namespace - * @param responseStep Response step for when call completes - * @return Asynchronous step - */ - public Step deleteConfigMapAsync(String name, String namespace, ResponseStep responseStep) { - return createRequestAsync(responseStep, new RequestParams("deleteConfigMap", namespace, name, null), DELETE_CONFIGMAP); - } - - /* Pods */ - - /** - * List pods - * @param namespace Namespace - * @return Listed pods - * @throws ApiException API Exception - */ - public V1PodList listPod(String namespace) throws ApiException { - String _continue = ""; - ApiClient client = helper.take(); - try { - return new CoreV1Api(client).listNamespacedPod(namespace, pretty, _continue, fieldSelector, - includeUninitialized, labelSelector, limit, resourceVersion, timeoutSeconds, watch); - } finally { - helper.recycle(client); - } - } - - private com.squareup.okhttp.Call listPodAsync(ApiClient client, String namespace, String _continue, ApiCallback callback) throws ApiException { - return new CoreV1Api(client).listNamespacedPodAsync(namespace, pretty, _continue, - fieldSelector, includeUninitialized, labelSelector, limit, resourceVersion, timeoutSeconds, watch, callback); - } - - private final CallFactory LIST_POD = (requestParams, usage, cont, callback) -> { - return listPodAsync(usage, requestParams.namespace, cont, callback); - }; - - /** - * Asynchronous step for listing pods - * @param namespace Namespace - * @param responseStep Response step for when call completes - * @return Asynchronous step - */ - public Step listPodAsync(String namespace, ResponseStep responseStep) { - return createRequestAsync(responseStep, new RequestParams("listPod", namespace, null, null), LIST_POD); - } - - /** - * Read pod - * @param name Name - * @param namespace Namespace - * @return Read pod - * @throws ApiException API Exception - */ - public V1Pod readPod(String name, String namespace) throws ApiException { - ApiClient client = helper.take(); - try { - return new CoreV1Api(client).readNamespacedPod(name, namespace, pretty, exact, export); - } finally { - helper.recycle(client); - } - } - - private com.squareup.okhttp.Call readPodAsync(ApiClient client, String name, String namespace, ApiCallback callback) throws ApiException { - return new CoreV1Api(client).readNamespacedPodAsync(name, namespace, pretty, exact, export, callback); - } - - private final CallFactory READ_POD = (requestParams, usage, cont, callback) -> { - return readPodAsync(usage, requestParams.name, requestParams.namespace, callback); - }; - - /** - * Asynchronous step for reading pod - * @param name Name - * @param namespace Namespace - * @param responseStep Response step for when call completes - * @return Asynchronous step - */ - public Step readPodAsync(String name, String namespace, ResponseStep responseStep) { - return createRequestAsync(responseStep, new RequestParams("readPod", namespace, name, null), READ_POD); - } - - /** - * Create pod - * @param namespace Namespace - * @param body Body - * @return Created pod - * @throws ApiException API Exception - */ - public V1Pod createPod(String namespace, V1Pod body) throws ApiException { - ApiClient client = helper.take(); - try { - return new CoreV1Api(client).createNamespacedPod(namespace, body, pretty); - } finally { - helper.recycle(client); - } - } - - private com.squareup.okhttp.Call createPodAsync(ApiClient client, String namespace, V1Pod body, ApiCallback callback) throws ApiException { - return new CoreV1Api(client).createNamespacedPodAsync(namespace, body, pretty, callback); - } - - private final CallFactory CREATE_POD = (requestParams, usage, cont, callback) -> { - return createPodAsync(usage, requestParams.namespace, (V1Pod) requestParams.body, callback); - }; - - /** - * Asynchronous step for creating pod - * @param namespace Namespace - * @param body Body - * @param responseStep Response step for when call completes - * @return Asynchronous step - */ - public Step createPodAsync(String namespace, V1Pod body, ResponseStep responseStep) { - return createRequestAsync(responseStep, new RequestParams("createPod", namespace, null, body), CREATE_POD); - } - - /** - * Replace pod - * @param name Name - * @param namespace Namespace - * @param body Body - * @return Replaced pod - * @throws ApiException API Exception - */ - public V1Pod replacePod(String name, String namespace, V1Pod body) throws ApiException { - ApiClient client = helper.take(); - try { - return new CoreV1Api(client).replaceNamespacedPod(name, namespace, body, pretty); - } finally { - helper.recycle(client); - } - } - - private com.squareup.okhttp.Call replacePodAsync(ApiClient client, String name, String namespace, V1Pod body, ApiCallback callback) throws ApiException { - return new CoreV1Api(client).replaceNamespacedPodAsync(name, namespace, body, pretty, callback); - } - - private final CallFactory REPLACE_POD = (requestParams, usage, cont, callback) -> { - return replacePodAsync(usage, requestParams.name, requestParams.namespace, (V1Pod) requestParams.body, callback); - }; - - /** - * Asynchronous step for replacing pod - * @param name Name - * @param namespace Namespace - * @param body Body - * @param responseStep Response step for when call completes - * @return Asynchronous step - */ - public Step replacePodAsync(String name, String namespace, V1Pod body, ResponseStep responseStep) { - return createRequestAsync(responseStep, new RequestParams("replacePod", namespace, name, body), REPLACE_POD); - } - - /** - * Delete pod - * @param name Name - * @param namespace Namespace - * @param deleteOptions Delete options - * @return Status of deletion - * @throws ApiException API Exception - */ - public V1Status deletePod(String name, String namespace, V1DeleteOptions deleteOptions) throws ApiException { - ApiClient client = helper.take(); - try { - return new CoreV1Api(client).deleteNamespacedPod(name, namespace, deleteOptions, pretty, gracePeriodSeconds, - orphanDependents, propagationPolicy); - } finally { - helper.recycle(client); - } - } - - private com.squareup.okhttp.Call deletePodAsync(ApiClient client, String name, String namespace, V1DeleteOptions deleteOptions, ApiCallback callback) throws ApiException { - return new CoreV1Api(client).deleteNamespacedPodAsync(name, namespace, deleteOptions, pretty, gracePeriodSeconds, orphanDependents, propagationPolicy, callback); - } - - private final CallFactory DELETE_POD = (requestParams, usage, cont, callback) -> { - return deletePodAsync(usage, requestParams.name, requestParams.namespace, (V1DeleteOptions) requestParams.body, callback); - }; - - /** - * Asynchronous step for deleting pod - * @param name Name - * @param namespace Namespace - * @param deleteOptions Delete options - * @param responseStep Response step for when call completes - * @return Asynchronous step - */ - public Step deletePodAsync(String name, String namespace, V1DeleteOptions deleteOptions, ResponseStep responseStep) { - return createRequestAsync(responseStep, new RequestParams("deletePod", namespace, name, deleteOptions), DELETE_POD); - } - - /** - * Delete collection of pods - * @param namespace Namespace - * @return Status of deletion - * @throws ApiException API Exception - */ - public V1Status deleteCollectionPod(String namespace) throws ApiException { - String _continue = ""; - ApiClient client = helper.take(); - try { - return new CoreV1Api(client).deleteCollectionNamespacedPod(namespace, pretty, _continue, fieldSelector, - includeUninitialized, labelSelector, limit, resourceVersion, timeoutSeconds, watch); - } finally { - helper.recycle(client); - } - } - - private com.squareup.okhttp.Call deleteCollectionPodAsync(ApiClient client, String namespace, String _continue, ApiCallback callback) throws ApiException { - return new CoreV1Api(client).deleteCollectionNamespacedPodAsync(namespace, pretty, _continue, fieldSelector, - includeUninitialized, labelSelector, limit, resourceVersion, timeoutSeconds, watch, callback); - } - - private final CallFactory DELETECOLLECTION_POD = (requestParams, usage, cont, callback) -> { - return deleteCollectionPodAsync(usage, requestParams.namespace, cont, callback); - }; - - /** - * Asynchronous step for deleting collection of pods - * @param namespace Namespace - * @param responseStep Response step for when call completes - * @return Asynchronous step - */ - public Step deleteCollectionPodAsync(String namespace, ResponseStep responseStep) { - return createRequestAsync(responseStep, new RequestParams("deleteCollection", namespace, null, null), DELETECOLLECTION_POD); - } - - /* Jobs */ - - /** - * List jobs - * @param namespace Namespace - * @return List of jobs - * @throws ApiException API Exception - */ - public V1JobList listJob(String namespace) throws ApiException { - String _continue = ""; - ApiClient client = helper.take(); - try { - return new BatchV1Api(client).listNamespacedJob(namespace, pretty, _continue, fieldSelector, - includeUninitialized, labelSelector, limit, resourceVersion, timeoutSeconds, watch); - } finally { - helper.recycle(client); - } - } - - private com.squareup.okhttp.Call listJobAsync(ApiClient client, String namespace, String _continue, ApiCallback callback) throws ApiException { - return new BatchV1Api(client).listNamespacedJobAsync(namespace, pretty, _continue, - fieldSelector, includeUninitialized, labelSelector, limit, resourceVersion, timeoutSeconds, watch, callback); - } - - private final CallFactory LIST_JOB = (requestParams, usage, cont, callback) -> { - return listJobAsync(usage, requestParams.namespace, cont, callback); - }; - - /** - * Asynchronous step for listing jobs - * @param namespace Namespace - * @param responseStep Response step for when call completes - * @return Asynchronous step - */ - public Step listJobsAsync(String namespace, ResponseStep responseStep) { - return createRequestAsync(responseStep, new RequestParams("listJob", namespace, null, null), LIST_JOB); - } - - /** - * Read job - * @param name Name - * @param namespace Namespace - * @return Read service - * @throws ApiException API Exception - */ - public V1Job readJob(String name, String namespace) throws ApiException { - ApiClient client = helper.take(); - try { - return new BatchV1Api(client).readNamespacedJob(name, namespace, pretty, exact, export); - } finally { - helper.recycle(client); - } - } - - private com.squareup.okhttp.Call readJobAsync(ApiClient client, String name, String namespace, ApiCallback callback) throws ApiException { - return new BatchV1Api(client).readNamespacedJobAsync(name, namespace, pretty, exact, export, callback); - } - - private final CallFactory READ_JOB = (requestParams, usage, cont, callback) -> { - return readJobAsync(usage, requestParams.name, requestParams.namespace, callback); - }; - - /** - * Asynchronous step for reading job - * @param name Name - * @param namespace Namespace - * @param responseStep Response step for when call completes - * @return Asynchronous step - */ - public Step readJobAsync(String name, String namespace, ResponseStep responseStep) { - return createRequestAsync(responseStep, new RequestParams("readJob", namespace, name, null), READ_JOB); - } - - /** - * Create job - * @param namespace Namespace - * @param body Body - * @return Created job - * @throws ApiException API Exception - */ - public V1Job createJob(String namespace, V1Job body) throws ApiException { - ApiClient client = helper.take(); - try { - return new BatchV1Api(client).createNamespacedJob(namespace, body, pretty); - } finally { - helper.recycle(client); - } - } - - private com.squareup.okhttp.Call createJobAsync(ApiClient client, String namespace, V1Job body, ApiCallback callback) throws ApiException { - return new BatchV1Api(client).createNamespacedJobAsync(namespace, body, pretty, callback); - } - - private final CallFactory CREATE_JOB = (requestParams, usage, cont, callback) -> { - return createJobAsync(usage, requestParams.namespace, (V1Job) requestParams.body, callback); - }; - - /** - * Asynchronous step for creating job - * @param namespace Namespace - * @param body Body - * @param responseStep Response step for when call completes - * @return Asynchronous step - */ - public Step createJobAsync(String namespace, V1Job body, ResponseStep responseStep) { - return createRequestAsync(responseStep, new RequestParams("createJob", namespace, null, body), CREATE_JOB); - } - - /** - * Replace job - * @param name Name - * @param namespace Namespace - * @param body Body - * @return Replaced job - * @throws ApiException API Exception - */ - public V1Job replaceJob(String name, String namespace, V1Job body) throws ApiException { - ApiClient client = helper.take(); - try { - return new BatchV1Api(client).replaceNamespacedJob(name, namespace, body, pretty); - } finally { - helper.recycle(client); - } - } - - private com.squareup.okhttp.Call replaceJobAsync(ApiClient client, String name, String namespace, V1Job body, ApiCallback callback) throws ApiException { - return new BatchV1Api(client).replaceNamespacedJobAsync(name, namespace, body, pretty, callback); - } - - private final CallFactory REPLACE_JOB = (requestParams, usage, cont, callback) -> { - return replaceJobAsync(usage, requestParams.name, requestParams.namespace, (V1Job) requestParams.body, callback); - }; - - /** - * Asynchronous step for replacing job - * @param name Name - * @param namespace Namespace - * @param body Body - * @param responseStep Response step for when call completes - * @return Asynchronous step - */ - public Step replaceJobAsync(String name, String namespace, V1Job body, ResponseStep responseStep) { - return createRequestAsync(responseStep, new RequestParams("replaceJob", namespace, name, body), REPLACE_JOB); - } - - /** - * Delete job - * @param name Name - * @param namespace Namespace - * @param body Delete options - * @return Status of deletion - * @throws ApiException API Exception - */ - public V1Status deleteJob(String name, String namespace, V1DeleteOptions body) throws ApiException { - ApiClient client = helper.take(); - try { - return new BatchV1Api(client).deleteNamespacedJob(name, namespace, body, pretty, gracePeriodSeconds, orphanDependents, propagationPolicy); - } finally { - helper.recycle(client); - } - } - - private com.squareup.okhttp.Call deleteJobAsync(ApiClient client, String name, String namespace,V1DeleteOptions body, ApiCallback callback) throws ApiException { - return new BatchV1Api(client).deleteNamespacedJobAsync(name, namespace, body, pretty, gracePeriodSeconds, orphanDependents, propagationPolicy, callback); - } - - private final CallFactory DELETE_JOB = (requestParams, usage, cont, callback) -> { - return deleteJobAsync(usage, requestParams.name, requestParams.namespace, (V1DeleteOptions) requestParams.body, callback); - }; - - /** - * Asynchronous step for deleting job - * @param name Name - * @param namespace Namespace - * @param responseStep Response step for when call completes - * @return Asynchronous step - */ - public Step deleteJobAsync(String name, String namespace, ResponseStep responseStep) { - return createRequestAsync(responseStep, new RequestParams("deleteJob", namespace, name, null), DELETE_JOB); - } - - /* Services */ - - /** - * List services - * @param namespace Namespace - * @return List of services - * @throws ApiException API Exception - */ - public V1ServiceList listService(String namespace) throws ApiException { - String _continue = ""; - ApiClient client = helper.take(); - try { - return new CoreV1Api(client).listNamespacedService(namespace, pretty, _continue, fieldSelector, - includeUninitialized, labelSelector, limit, resourceVersion, timeoutSeconds, watch); - } finally { - helper.recycle(client); - } - } - - private com.squareup.okhttp.Call listServiceAsync(ApiClient client, String namespace, String _continue, ApiCallback callback) throws ApiException { - return new CoreV1Api(client).listNamespacedServiceAsync(namespace, pretty, _continue, - fieldSelector, includeUninitialized, labelSelector, limit, resourceVersion, timeoutSeconds, watch, callback); - } - - private final CallFactory LIST_SERVICE = (requestParams, usage, cont, callback) -> { - return listServiceAsync(usage, requestParams.namespace, cont, callback); - }; - - /** - * Asynchronous step for listing services - * @param namespace Namespace - * @param responseStep Response step for when call completes - * @return Asynchronous step - */ - public Step listServiceAsync(String namespace, ResponseStep responseStep) { - return createRequestAsync(responseStep, new RequestParams("listService", namespace, null, null), LIST_SERVICE); - } - - /** - * Read service - * @param name Name - * @param namespace Namespace - * @return Read service - * @throws ApiException API Exception - */ - public V1Service readService(String name, String namespace) throws ApiException { - ApiClient client = helper.take(); - try { - return new CoreV1Api(client).readNamespacedService(name, namespace, pretty, exact, export); - } finally { - helper.recycle(client); - } - } - - private com.squareup.okhttp.Call readServiceAsync(ApiClient client, String name, String namespace, ApiCallback callback) throws ApiException { - return new CoreV1Api(client).readNamespacedServiceAsync(name, namespace, pretty, exact, export, callback); - } - - private final CallFactory READ_SERVICE = (requestParams, usage, cont, callback) -> { - return readServiceAsync(usage, requestParams.name, requestParams.namespace, callback); - }; - - /** - * Asynchronous step for reading service - * @param name Name - * @param namespace Namespace - * @param responseStep Response step for when call completes - * @return Asynchronous step - */ - public Step readServiceAsync(String name, String namespace, ResponseStep responseStep) { - return createRequestAsync(responseStep, new RequestParams("readService", namespace, name, null), READ_SERVICE); - } - - /** - * Create service - * @param namespace Namespace - * @param body Body - * @return Created service - * @throws ApiException API Exception - */ - public V1Service createService(String namespace, V1Service body) throws ApiException { - ApiClient client = helper.take(); - try { - return new CoreV1Api(client).createNamespacedService(namespace, body, pretty); - } finally { - helper.recycle(client); - } - } - - private com.squareup.okhttp.Call createServiceAsync(ApiClient client, String namespace, V1Service body, ApiCallback callback) throws ApiException { - return new CoreV1Api(client).createNamespacedServiceAsync(namespace, body, pretty, callback); - } - - private final CallFactory CREATE_SERVICE = (requestParams, usage, cont, callback) -> { - return createServiceAsync(usage, requestParams.namespace, (V1Service) requestParams.body, callback); - }; - - /** - * Asynchronous step for creating service - * @param namespace Namespace - * @param body Body - * @param responseStep Response step for when call completes - * @return Asynchronous step - */ - public Step createServiceAsync(String namespace, V1Service body, ResponseStep responseStep) { - return createRequestAsync(responseStep, new RequestParams("createService", namespace, null, body), CREATE_SERVICE); - } - - /** - * Replace service - * @param name Name - * @param namespace Namespace - * @param body Body - * @return Replaced service - * @throws ApiException API Exception - */ - public V1Service replaceService(String name, String namespace, V1Service body) throws ApiException { - ApiClient client = helper.take(); - try { - return new CoreV1Api(client).replaceNamespacedService(name, namespace, body, pretty); - } finally { - helper.recycle(client); - } - } - - private com.squareup.okhttp.Call replaceServiceAsync(ApiClient client, String name, String namespace, V1Service body, ApiCallback callback) throws ApiException { - return new CoreV1Api(client).replaceNamespacedServiceAsync(name, namespace, body, pretty, callback); - } - - private final CallFactory REPLACE_SERVICE = (requestParams, usage, cont, callback) -> { - return replaceServiceAsync(usage, requestParams.name, requestParams.namespace, (V1Service) requestParams.body, callback); - }; - - /** - * Asynchronous step for replacing service - * @param name Name - * @param namespace Namespace - * @param body Body - * @param responseStep Response step for when call completes - * @return Asynchronous step - */ - public Step replaceServiceAsync(String name, String namespace, V1Service body, ResponseStep responseStep) { - return createRequestAsync(responseStep, new RequestParams("replaceService", namespace, name, body), REPLACE_SERVICE); - } - - /** - * Delete service - * @param name Name - * @param namespace Namespace - * @return Status of deletion - * @throws ApiException API Exception - */ - public V1Status deleteService(String name, String namespace) throws ApiException { - ApiClient client = helper.take(); - try { - return new CoreV1Api(client).deleteNamespacedService(name, namespace, pretty); - } finally { - helper.recycle(client); - } - } - private com.squareup.okhttp.Call deleteServiceAsync(ApiClient client, String name, String namespace, ApiCallback callback) throws ApiException { - return new CoreV1Api(client).deleteNamespacedServiceAsync(name, namespace, pretty, callback); + private com.squareup.okhttp.Call replaceConfigMapAsync(ApiClient client, String name, String namespace, V1ConfigMap body, ApiCallback callback) throws ApiException { + return new CoreV1Api(client).replaceNamespacedConfigMapAsync(name, namespace, body, pretty, callback); } - private final CallFactory DELETE_SERVICE = (requestParams, usage, cont, callback) -> { - return deleteServiceAsync(usage, requestParams.name, requestParams.namespace, callback); - }; + private final CallFactory REPLACE_CONFIGMAP = (requestParams, usage, cont, callback) + -> replaceConfigMapAsync(usage, requestParams.name, requestParams.namespace, (V1ConfigMap) requestParams.body, callback); /** - * Asynchronous step for deleting service + * Asynchronous step for replacing config map * @param name Name * @param namespace Namespace + * @param body Body * @param responseStep Response step for when call completes * @return Asynchronous step */ - public Step deleteServiceAsync(String name, String namespace, ResponseStep responseStep) { - return createRequestAsync(responseStep, new RequestParams("deleteService", namespace, name, null), DELETE_SERVICE); - } - - /* Events */ - - /** - * List events - * @param namespace Namespace - * @return List of events - * @throws ApiException API Exception - */ - public V1EventList listEvent(String namespace) throws ApiException { - String _continue = ""; - ApiClient client = helper.take(); - try { - return new CoreV1Api(client).listNamespacedEvent(namespace, pretty, _continue, fieldSelector, - includeUninitialized, labelSelector, limit, resourceVersion, timeoutSeconds, watch); - } finally { - helper.recycle(client); - } + public Step replaceConfigMapAsync(String name, String namespace, V1ConfigMap body, ResponseStep responseStep) { + return createRequestAsync(responseStep, new RequestParams("replaceConfigMap", namespace, name, body), REPLACE_CONFIGMAP); } - private com.squareup.okhttp.Call listEventAsync(ApiClient client, String namespace, String _continue, ApiCallback callback) throws ApiException { - return new CoreV1Api(client).listNamespacedEventAsync(namespace, pretty, _continue, + /* Pods */ + + private com.squareup.okhttp.Call listPodAsync(ApiClient client, String namespace, String _continue, ApiCallback callback) throws ApiException { + return new CoreV1Api(client).listNamespacedPodAsync(namespace, pretty, _continue, fieldSelector, includeUninitialized, labelSelector, limit, resourceVersion, timeoutSeconds, watch, callback); } - private final CallFactory LIST_EVENT = (requestParams, usage, cont, callback) -> { - return listEventAsync(usage, requestParams.namespace, cont, callback); - }; + private final CallFactory LIST_POD = (requestParams, usage, cont, callback) + -> listPodAsync(usage, requestParams.namespace, cont, callback); /** - * Asynchronous step for listing events + * Asynchronous step for listing pods * @param namespace Namespace * @param responseStep Response step for when call completes * @return Asynchronous step */ - public Step listEventAsync(String namespace, ResponseStep responseStep) { - return createRequestAsync(responseStep, new RequestParams("listEvent", namespace, null, null), LIST_EVENT); + public Step listPodAsync(String namespace, ResponseStep responseStep) { + return createRequestAsync(responseStep, new RequestParams("listPod", namespace, null, null), LIST_POD); + } + + private com.squareup.okhttp.Call readPodAsync(ApiClient client, String name, String namespace, ApiCallback callback) throws ApiException { + return new CoreV1Api(client).readNamespacedPodAsync(name, namespace, pretty, exact, export, callback); } + + private final CallFactory READ_POD = (requestParams, usage, cont, callback) + -> readPodAsync(usage, requestParams.name, requestParams.namespace, callback); /** - * Read event + * Asynchronous step for reading pod * @param name Name * @param namespace Namespace - * @return Read event - * @throws ApiException API Exception + * @param responseStep Response step for when call completes + * @return Asynchronous step */ - public V1Event readEvent(String name, String namespace) throws ApiException { - ApiClient client = helper.take(); - try { - return new CoreV1Api(client).readNamespacedEvent(name, namespace, pretty, exact, export); - } finally { - helper.recycle(client); - } + public Step readPodAsync(String name, String namespace, ResponseStep responseStep) { + return createRequestAsync(responseStep, new RequestParams("readPod", namespace, name, null), READ_POD); } - private com.squareup.okhttp.Call readEventAsync(ApiClient client, String name, String namespace, ApiCallback callback) throws ApiException { - return new CoreV1Api(client).readNamespacedEventAsync(name, namespace, pretty, exact, export, callback); + private com.squareup.okhttp.Call createPodAsync(ApiClient client, String namespace, V1Pod body, ApiCallback callback) throws ApiException { + return new CoreV1Api(client).createNamespacedPodAsync(namespace, body, pretty, callback); } - private final CallFactory READ_EVENT = (requestParams, usage, cont, callback) -> { - return readEventAsync(usage, requestParams.name, requestParams.namespace, callback); - }; + private final CallFactory CREATE_POD = (requestParams, usage, cont, callback) + -> createPodAsync(usage, requestParams.namespace, (V1Pod) requestParams.body, callback); /** - * Asynchronous step for reading event - * @param name Name + * Asynchronous step for creating pod * @param namespace Namespace + * @param body Body * @param responseStep Response step for when call completes * @return Asynchronous step */ - public Step readEventAsync(String name, String namespace, ResponseStep responseStep) { - return createRequestAsync(responseStep, new RequestParams("readEvent", namespace, name, null), READ_EVENT); + public Step createPodAsync(String namespace, V1Pod body, ResponseStep responseStep) { + return createRequestAsync(responseStep, new RequestParams("createPod", namespace, null, body), CREATE_POD); + } + + private com.squareup.okhttp.Call deletePodAsync(ApiClient client, String name, String namespace, V1DeleteOptions deleteOptions, ApiCallback callback) throws ApiException { + return new CoreV1Api(client).deleteNamespacedPodAsync(name, namespace, deleteOptions, pretty, gracePeriodSeconds, orphanDependents, propagationPolicy, callback); } + + private final CallFactory DELETE_POD = (requestParams, usage, cont, callback) + -> deletePodAsync(usage, requestParams.name, requestParams.namespace, (V1DeleteOptions) requestParams.body, callback); /** - * Create event + * Asynchronous step for deleting pod + * @param name Name * @param namespace Namespace - * @param body Body - * @return Created service - * @throws ApiException API Exception + * @param deleteOptions Delete options + * @param responseStep Response step for when call completes + * @return Asynchronous step */ - public V1Event createEvent(String namespace, V1Event body) throws ApiException { - ApiClient client = helper.take(); - try { - return new CoreV1Api(client).createNamespacedEvent(namespace, body, pretty); - } finally { - helper.recycle(client); - } + public Step deletePodAsync(String name, String namespace, V1DeleteOptions deleteOptions, ResponseStep responseStep) { + return createRequestAsync(responseStep, new RequestParams("deletePod", namespace, name, deleteOptions), DELETE_POD); } - private com.squareup.okhttp.Call createEventAsync(ApiClient client, String namespace, V1Event body, ApiCallback callback) throws ApiException { - return new CoreV1Api(client).createNamespacedEventAsync(namespace, body, pretty, callback); + private com.squareup.okhttp.Call deleteCollectionPodAsync(ApiClient client, String namespace, String _continue, ApiCallback callback) throws ApiException { + return new CoreV1Api(client).deleteCollectionNamespacedPodAsync(namespace, pretty, _continue, fieldSelector, + includeUninitialized, labelSelector, limit, resourceVersion, timeoutSeconds, watch, callback); } - private final CallFactory CREATE_EVENT = (requestParams, usage, cont, callback) -> { - return createEventAsync(usage, requestParams.namespace, (V1Event) requestParams.body, callback); - }; + private final CallFactory DELETECOLLECTION_POD = (requestParams, usage, cont, callback) + -> deleteCollectionPodAsync(usage, requestParams.namespace, cont, callback); /** - * Asynchronous step for creating event + * Asynchronous step for deleting collection of pods * @param namespace Namespace - * @param body Body * @param responseStep Response step for when call completes * @return Asynchronous step */ - public Step createEventAsync(String namespace, V1Event body, ResponseStep responseStep) { - return createRequestAsync(responseStep, new RequestParams("createEvent", namespace, null, body), CREATE_EVENT); + public Step deleteCollectionPodAsync(String namespace, ResponseStep responseStep) { + return createRequestAsync(responseStep, new RequestParams("deleteCollection", namespace, null, null), DELETECOLLECTION_POD); } + /* Jobs */ + + private com.squareup.okhttp.Call createJobAsync(ApiClient client, String namespace, V1Job body, ApiCallback callback) throws ApiException { + return new BatchV1Api(client).createNamespacedJobAsync(namespace, body, pretty, callback); + } + + private final CallFactory CREATE_JOB = (requestParams, usage, cont, callback) + -> createJobAsync(usage, requestParams.namespace, (V1Job) requestParams.body, callback); + /** - * Delete event - * @param name Name + * Asynchronous step for creating job * @param namespace Namespace - * @param deleteOptions Deletion options - * @return Status of deletion - * @throws ApiException API Exception + * @param body Body + * @param responseStep Response step for when call completes + * @return Asynchronous step */ - public V1Status deleteEvent(String name, String namespace, V1DeleteOptions deleteOptions) throws ApiException { - ApiClient client = helper.take(); - try { - return new CoreV1Api(client).deleteNamespacedEvent(name, namespace, deleteOptions, pretty, gracePeriodSeconds, - orphanDependents, propagationPolicy); - } finally { - helper.recycle(client); - } + public Step createJobAsync(String namespace, V1Job body, ResponseStep responseStep) { + return createRequestAsync(responseStep, new RequestParams("createJob", namespace, null, body), CREATE_JOB); } - private com.squareup.okhttp.Call deleteEventAsync(ApiClient client, String name, String namespace, V1DeleteOptions deleteOptions, ApiCallback callback) throws ApiException { - return new CoreV1Api(client).deleteNamespacedEventAsync(name, namespace, deleteOptions, pretty, gracePeriodSeconds, orphanDependents, propagationPolicy, callback); + private com.squareup.okhttp.Call deleteJobAsync(ApiClient client, String name, String namespace,V1DeleteOptions body, ApiCallback callback) throws ApiException { + return new BatchV1Api(client).deleteNamespacedJobAsync(name, namespace, body, pretty, gracePeriodSeconds, orphanDependents, propagationPolicy, callback); } - private final CallFactory DELETE_EVENT = (requestParams, usage, cont, callback) -> { - return deleteEventAsync(usage, requestParams.name, requestParams.namespace, (V1DeleteOptions) requestParams.body, callback); - }; + private final CallFactory DELETE_JOB = (requestParams, usage, cont, callback) + -> deleteJobAsync(usage, requestParams.name, requestParams.namespace, (V1DeleteOptions) requestParams.body, callback); /** - * Asynchronous step for deleting event + * Asynchronous step for deleting job * @param name Name * @param namespace Namespace * @param responseStep Response step for when call completes * @return Asynchronous step */ - public Step deleteEventAsync(String name, String namespace, ResponseStep responseStep) { - return createRequestAsync(responseStep, new RequestParams("deleteEvent", namespace, name, null), DELETE_EVENT); + public Step deleteJobAsync(String name, String namespace, ResponseStep responseStep) { + return createRequestAsync(responseStep, new RequestParams("deleteJob", namespace, name, null), DELETE_JOB); } - /* Persistent Volumes */ + /* Services */ /** - * List persistent volumes - * @return List of persistent volumes + * List services + * @param namespace Namespace + * @return List of services * @throws ApiException API Exception */ - public V1PersistentVolumeList listPersistentVolume() throws ApiException { + public V1ServiceList listService(String namespace) throws ApiException { String _continue = ""; ApiClient client = helper.take(); try { - return new CoreV1Api(client).listPersistentVolume(pretty, _continue, fieldSelector, + return new CoreV1Api(client).listNamespacedService(namespace, pretty, _continue, fieldSelector, includeUninitialized, labelSelector, limit, resourceVersion, timeoutSeconds, watch); } finally { helper.recycle(client); } } - private com.squareup.okhttp.Call listPersistentVolumeAsync(ApiClient client, String _continue, ApiCallback callback) throws ApiException { - return new CoreV1Api(client).listPersistentVolumeAsync(pretty, _continue, + private com.squareup.okhttp.Call listServiceAsync(ApiClient client, String namespace, String _continue, ApiCallback callback) throws ApiException { + return new CoreV1Api(client).listNamespacedServiceAsync(namespace, pretty, _continue, fieldSelector, includeUninitialized, labelSelector, limit, resourceVersion, timeoutSeconds, watch, callback); } - private final CallFactory LIST_PERSISTENT_VOLUME = (requestParams, usage, cont, callback) -> { - return listPersistentVolumeAsync(usage, cont, callback); - }; + private final CallFactory LIST_SERVICE = (requestParams, usage, cont, callback) + -> listServiceAsync(usage, requestParams.namespace, cont, callback); /** - * Asynchronous step for listing persistent volumes + * Asynchronous step for listing services + * @param namespace Namespace * @param responseStep Response step for when call completes * @return Asynchronous step */ - public Step listPersistentVolumeAsync(String namespace, ResponseStep responseStep) { - return createRequestAsync(responseStep, new RequestParams("listPersistentVolume", null, null, null), LIST_PERSISTENT_VOLUME); + public Step listServiceAsync(String namespace, ResponseStep responseStep) { + return createRequestAsync(responseStep, new RequestParams("listService", namespace, null, null), LIST_SERVICE); } /** - * Read persistent volume + * Read service * @param name Name - * @return Read persistent volume + * @param namespace Namespace + * @return Read service * @throws ApiException API Exception */ - public V1PersistentVolume readEvent(String name) throws ApiException { + public V1Service readService(String name, String namespace) throws ApiException { ApiClient client = helper.take(); try { - return new CoreV1Api(client).readPersistentVolume(name, pretty, exact, export); + return new CoreV1Api(client).readNamespacedService(name, namespace, pretty, exact, export); } finally { helper.recycle(client); } } - private com.squareup.okhttp.Call readPersistentVolumeAsync(ApiClient client, String name, ApiCallback callback) throws ApiException { - return new CoreV1Api(client).readPersistentVolumeAsync(name, pretty, exact, export, callback); + private com.squareup.okhttp.Call readServiceAsync(ApiClient client, String name, String namespace, ApiCallback callback) throws ApiException { + return new CoreV1Api(client).readNamespacedServiceAsync(name, namespace, pretty, exact, export, callback); } - private final CallFactory READ_PERSISTENT_VOLUME = (requestParams, usage, cont, callback) -> { - return readPersistentVolumeAsync(usage, requestParams.name, callback); - }; + private final CallFactory READ_SERVICE = (requestParams, usage, cont, callback) + -> readServiceAsync(usage, requestParams.name, requestParams.namespace, callback); /** - * Asynchronous step for reading persistent volume + * Asynchronous step for reading service * @param name Name + * @param namespace Namespace * @param responseStep Response step for when call completes * @return Asynchronous step */ - public Step readPersistentVolumeAsync(String name, ResponseStep responseStep) { - return createRequestAsync(responseStep, new RequestParams("readEvent", null, name, null), READ_PERSISTENT_VOLUME); - } - - /** - * Create persistent volume - * @param body Body - * @return Created persistent volume - * @throws ApiException API Exception - */ - public V1PersistentVolume createPersistentVolume(V1PersistentVolume body) throws ApiException { - ApiClient client = helper.take(); - try { - return new CoreV1Api(client).createPersistentVolume(body, pretty); - } finally { - helper.recycle(client); - } + public Step readServiceAsync(String name, String namespace, ResponseStep responseStep) { + return createRequestAsync(responseStep, new RequestParams("readService", namespace, name, null), READ_SERVICE); } - private com.squareup.okhttp.Call createPersistentVolumeAsync(ApiClient client, V1PersistentVolume body, ApiCallback callback) throws ApiException { - return new CoreV1Api(client).createPersistentVolumeAsync(body, pretty, callback); + private com.squareup.okhttp.Call createServiceAsync(ApiClient client, String namespace, V1Service body, ApiCallback callback) throws ApiException { + return new CoreV1Api(client).createNamespacedServiceAsync(namespace, body, pretty, callback); } - private final CallFactory CREATE_PERSISTENT_VOLUME = (requestParams, usage, cont, callback) -> { - return createPersistentVolumeAsync(usage, (V1PersistentVolume) requestParams.body, callback); - }; + private final CallFactory CREATE_SERVICE = (requestParams, usage, cont, callback) + -> createServiceAsync(usage, requestParams.namespace, (V1Service) requestParams.body, callback); /** - * Asynchronous step for creating persistent volume + * Asynchronous step for creating service + * @param namespace Namespace * @param body Body * @param responseStep Response step for when call completes * @return Asynchronous step */ - public Step createPersistentVolumeAsync(V1PersistentVolume body, ResponseStep responseStep) { - return createRequestAsync(responseStep, new RequestParams("createPersistentVolume", null, null, body), CREATE_PERSISTENT_VOLUME); + public Step createServiceAsync(String namespace, V1Service body, ResponseStep responseStep) { + return createRequestAsync(responseStep, new RequestParams("createService", namespace, null, body), CREATE_SERVICE); } - + /** - * Delete persistent volume + * Delete service * @param name Name - * @param deleteOptions Deletion options + * @param namespace Namespace * @return Status of deletion * @throws ApiException API Exception */ - public V1Status deletePersistentVolume(String name, V1DeleteOptions deleteOptions) throws ApiException { + public V1Status deleteService(String name, String namespace) throws ApiException { ApiClient client = helper.take(); try { - return new CoreV1Api(client).deletePersistentVolume(name, deleteOptions, pretty, gracePeriodSeconds, - orphanDependents, propagationPolicy); + return new CoreV1Api(client).deleteNamespacedService(name, namespace, pretty); } finally { helper.recycle(client); } } - private com.squareup.okhttp.Call deletePersistentVolumeAsync(ApiClient client, String name, V1DeleteOptions deleteOptions, ApiCallback callback) throws ApiException { - return new CoreV1Api(client).deletePersistentVolumeAsync(name, deleteOptions, pretty, gracePeriodSeconds, orphanDependents, propagationPolicy, callback); + private com.squareup.okhttp.Call deleteServiceAsync(ApiClient client, String name, String namespace, ApiCallback callback) throws ApiException { + return new CoreV1Api(client).deleteNamespacedServiceAsync(name, namespace, pretty, callback); } - private final CallFactory DELETE_PERSISTENT_VOLUME = (requestParams, usage, cont, callback) -> { - return deletePersistentVolumeAsync(usage, requestParams.name, (V1DeleteOptions) requestParams.body, callback); - }; + private final CallFactory DELETE_SERVICE = (requestParams, usage, cont, callback) + -> deleteServiceAsync(usage, requestParams.name, requestParams.namespace, callback); /** - * Asynchronous step for deleting persistent volume + * Asynchronous step for deleting service * @param name Name + * @param namespace Namespace * @param responseStep Response step for when call completes * @return Asynchronous step */ - public Step deletePersistentVolumeAsync(String name, ResponseStep responseStep) { - return createRequestAsync(responseStep, new RequestParams("deletePersistentVolume", null, name, null), DELETE_PERSISTENT_VOLUME); + public Step deleteServiceAsync(String name, String namespace, ResponseStep responseStep) { + return createRequestAsync(responseStep, new RequestParams("deleteService", namespace, name, null), DELETE_SERVICE); } - /* Persistent Volume Claims */ + /* Events */ + + private com.squareup.okhttp.Call listEventAsync(ApiClient client, String namespace, String _continue, ApiCallback callback) throws ApiException { + return new CoreV1Api(client).listNamespacedEventAsync(namespace, pretty, _continue, + fieldSelector, includeUninitialized, labelSelector, limit, resourceVersion, timeoutSeconds, watch, callback); + } + + private final CallFactory LIST_EVENT = (requestParams, usage, cont, callback) + -> listEventAsync(usage, requestParams.namespace, cont, callback); /** - * List persistent volume claims + * Asynchronous step for listing events * @param namespace Namespace - * @return List of persistent volume claims + * @param responseStep Response step for when call completes + * @return Asynchronous step + */ + public Step listEventAsync(String namespace, ResponseStep responseStep) { + return createRequestAsync(responseStep, new RequestParams("listEvent", namespace, null, null), LIST_EVENT); + } + + /* Persistent Volumes */ + + /** + * List persistent volumes + * @return List of persistent volumes * @throws ApiException API Exception */ - public V1PersistentVolumeClaimList listPersistentVolumeClaim(String namespace) throws ApiException { + public V1PersistentVolumeList listPersistentVolume() throws ApiException { String _continue = ""; ApiClient client = helper.take(); try { - return new CoreV1Api(client).listNamespacedPersistentVolumeClaim(namespace, pretty, _continue, fieldSelector, + return new CoreV1Api(client).listPersistentVolume(pretty, _continue, fieldSelector, includeUninitialized, labelSelector, limit, resourceVersion, timeoutSeconds, watch); } finally { helper.recycle(client); } } + /* Persistent Volume Claims */ + private com.squareup.okhttp.Call listPersistentVolumeClaimAsync(ApiClient client, String namespace, String _continue, ApiCallback callback) throws ApiException { return new CoreV1Api(client).listNamespacedPersistentVolumeClaimAsync(namespace, pretty, _continue, fieldSelector, includeUninitialized, labelSelector, limit, resourceVersion, timeoutSeconds, watch, callback); } - private final CallFactory LIST_PERSISTENTVOLUMECLAIM = (requestParams, usage, cont, callback) -> { - return listPersistentVolumeClaimAsync(usage, requestParams.namespace, cont, callback); - }; + private final CallFactory LIST_PERSISTENTVOLUMECLAIM = (requestParams, usage, cont, callback) + -> listPersistentVolumeClaimAsync(usage, requestParams.namespace, cont, callback); /** * Asynchronous step for listing persistent volume claims @@ -1586,9 +664,8 @@ private com.squareup.okhttp.Call readSecretAsync(ApiClient client, String name, return new CoreV1Api(client).readNamespacedSecretAsync(name, namespace, pretty, exact, export, callback); } - private final CallFactory READ_SECRET = (requestParams, usage, cont, callback) -> { - return readSecretAsync(usage, requestParams.name, requestParams.namespace, callback); - }; + private final CallFactory READ_SECRET = (requestParams, usage, cont, callback) + -> readSecretAsync(usage, requestParams.name, requestParams.namespace, callback); /** * Create secret @@ -1652,24 +729,6 @@ public V1SubjectAccessReview createSubjectAccessReview(V1SubjectAccessReview bod } } - private com.squareup.okhttp.Call createSubjectAccessReviewAsync(ApiClient client, V1SubjectAccessReview body, ApiCallback callback) throws ApiException { - return new AuthorizationV1Api(client).createSubjectAccessReviewAsync(body, pretty, callback); - } - - private final CallFactory CREATE_SUBJECTACCESSREVIEW = (requestParams, usage, cont, callback) -> { - return createSubjectAccessReviewAsync(usage, (V1SubjectAccessReview) requestParams.body, callback); - }; - - /** - * Asynchronous step for creating subject access review - * @param body Body - * @param responseStep Response step for when call completes - * @return Asynchronous step - */ - public Step createSubjectAccessReviewAsync(V1SubjectAccessReview body, ResponseStep responseStep) { - return createRequestAsync(responseStep, new RequestParams("createSubjectAccessReview", null, null, body), CREATE_SUBJECTACCESSREVIEW); - } - /* Token Review */ /** @@ -1687,51 +746,15 @@ public V1TokenReview createTokenReview(V1TokenReview body) throws ApiException { } } - private com.squareup.okhttp.Call createTokenReviewAsync(ApiClient client, V1TokenReview body, ApiCallback callback) throws ApiException { - return new AuthenticationV1Api(client).createTokenReviewAsync(body, pretty, callback); - } - - private final CallFactory CREATE_TOKENREVIEW = (requestParams, usage, cont, callback) -> { - return createTokenReviewAsync(usage, (V1TokenReview) requestParams.body, callback); - }; - - /** - * Asynchronous step for creating token review - * @param body Body - * @param responseStep Response step for when call completes - * @return Asynchronous step - */ - public Step createTokenReviewAsync(V1TokenReview body, ResponseStep responseStep) { - return createRequestAsync(responseStep, new RequestParams("createTokenReview", null, null, body), CREATE_TOKENREVIEW); - } - /* Ingress */ - - /** - * List ingress - * @param namespace Namespace - * @return Listed ingress - * @throws ApiException API Exception - */ - public V1beta1IngressList listIngress(String namespace) throws ApiException { - String _continue = ""; - ApiClient client = helper.take(); - try { - return new ExtensionsV1beta1Api(client).listNamespacedIngress(namespace, pretty, _continue, fieldSelector, - includeUninitialized, labelSelector, limit, resourceVersion, timeoutSeconds, watch); - } finally { - helper.recycle(client); - } - } private com.squareup.okhttp.Call listIngressAsync(ApiClient client, String namespace, String _continue, ApiCallback callback) throws ApiException { return new ExtensionsV1beta1Api(client).listNamespacedIngressAsync(namespace, pretty, _continue, fieldSelector, includeUninitialized, labelSelector, limit, resourceVersion, timeoutSeconds, watch, callback); } - private final CallFactory LIST_INGRESS = (requestParams, usage, cont, callback) -> { - return listIngressAsync(usage, requestParams.namespace, cont, callback); - }; + private final CallFactory LIST_INGRESS = (requestParams, usage, cont, callback) + -> listIngressAsync(usage, requestParams.namespace, cont, callback); /** * Asynchronous step for listing ingress @@ -1763,9 +786,8 @@ private com.squareup.okhttp.Call readIngressAsync(ApiClient client, String name, return new ExtensionsV1beta1Api(client).readNamespacedIngressAsync(name, namespace, pretty, exact, export, callback); } - private final CallFactory READ_INGRESS = (requestParams, usage, cont, callback) -> { - return readIngressAsync(usage, requestParams.name, requestParams.namespace, callback); - }; + private final CallFactory READ_INGRESS = (requestParams, usage, cont, callback) + -> readIngressAsync(usage, requestParams.name, requestParams.namespace, callback); /** * Asynchronous step for reading ingress @@ -1777,30 +799,13 @@ private com.squareup.okhttp.Call readIngressAsync(ApiClient client, String name, public Step readIngressAsync(String name, String namespace, ResponseStep responseStep) { return createRequestAsync(responseStep, new RequestParams("readIngress", namespace, name, null), READ_INGRESS); } - - /** - * Create ingress - * @param namespace Namespace - * @param body Body - * @return Created ingress - * @throws ApiException API Exception - */ - public V1beta1Ingress createIngress(String namespace, V1beta1Ingress body) throws ApiException { - ApiClient client = helper.take(); - try { - return new ExtensionsV1beta1Api(client).createNamespacedIngress(namespace, body, pretty); - } finally { - helper.recycle(client); - } - } private com.squareup.okhttp.Call createIngressAsync(ApiClient client, String namespace, V1beta1Ingress body, ApiCallback callback) throws ApiException { return new ExtensionsV1beta1Api(client).createNamespacedIngressAsync(namespace, body, pretty, callback); } - private final CallFactory CREATE_INGRESS = (requestParams, usage, cont, callback) -> { - return createIngressAsync(usage, requestParams.namespace, (V1beta1Ingress) requestParams.body, callback); - }; + private final CallFactory CREATE_INGRESS = (requestParams, usage, cont, callback) + -> createIngressAsync(usage, requestParams.namespace, (V1beta1Ingress) requestParams.body, callback); /** * Asynchronous step for creating ingress @@ -1812,31 +817,13 @@ private com.squareup.okhttp.Call createIngressAsync(ApiClient client, String nam public Step createIngressAsync(String namespace, V1beta1Ingress body, ResponseStep responseStep) { return createRequestAsync(responseStep, new RequestParams("createIngress", namespace, null, body), CREATE_INGRESS); } - - /** - * Replace ingress - * @param name Name - * @param namespace Namespace - * @param body Body - * @return Replaced ingress - * @throws ApiException API Exception - */ - public V1beta1Ingress replaceIngress(String name, String namespace, V1beta1Ingress body) throws ApiException { - ApiClient client = helper.take(); - try { - return new ExtensionsV1beta1Api(client).replaceNamespacedIngress(name, namespace, body, pretty); - } finally { - helper.recycle(client); - } - } private com.squareup.okhttp.Call replaceIngressAsync(ApiClient client, String name, String namespace, V1beta1Ingress body, ApiCallback callback) throws ApiException { return new ExtensionsV1beta1Api(client).replaceNamespacedIngressAsync(name, namespace, body, pretty, callback); } - private final CallFactory REPLACE_INGRESS = (requestParams, usage, cont, callback) -> { - return replaceIngressAsync(usage, requestParams.name, requestParams.namespace, (V1beta1Ingress) requestParams.body, callback); - }; + private final CallFactory REPLACE_INGRESS = (requestParams, usage, cont, callback) + -> replaceIngressAsync(usage, requestParams.name, requestParams.namespace, (V1beta1Ingress) requestParams.body, callback); /** * Asynchronous step for replacing ingress @@ -1872,9 +859,8 @@ private com.squareup.okhttp.Call deleteIngressAsync(ApiClient client, String nam return new ExtensionsV1beta1Api(client).deleteNamespacedIngressAsync(name, namespace, deleteOptions, pretty, gracePeriodSeconds, orphanDependents, propagationPolicy, callback); } - private final CallFactory DELETE_INGRESS = (requestParams, usage, cont, callback) -> { - return deleteIngressAsync(usage, requestParams.name, requestParams.namespace, (V1DeleteOptions) requestParams.body, callback); - }; + private final CallFactory DELETE_INGRESS = (requestParams, usage, cont, callback) + -> deleteIngressAsync(usage, requestParams.name, requestParams.namespace, (V1DeleteOptions) requestParams.body, callback); /** * Asynchronous step for deleting ingress From 89dfda398f2ec7ff0cb85a0551865d8b5cce2019 Mon Sep 17 00:00:00 2001 From: Russell Gold Date: Fri, 20 Apr 2018 10:24:18 -0400 Subject: [PATCH 054/186] Prepare for unit testing --- .../java/oracle/kubernetes/operator/Main.java | 5 +- .../kubernetes/operator/TuningParameters.java | 6 +- .../operator/TuningParametersImpl.java | 28 +- .../operator/calls/AsyncRequestStep.java | 242 ++++++++++++ .../operator/calls/CallFactory.java | 13 + .../operator/calls/CallResponse.java | 23 ++ .../operator/calls/CallWrapper.java | 23 ++ .../operator/calls/CancelableCall.java | 15 + .../operator/calls/RequestParams.java | 18 + .../operator/calls/RetryStrategy.java | 43 +++ .../operator/helpers/CallBuilder.java | 357 +++--------------- .../operator/helpers/CallBuilderFactory.java | 4 +- .../operator/helpers/ResponseStep.java | 15 +- .../operator/HealthCheckHelperTest.java | 22 +- .../kubernetes/operator/SecretHelperTest.java | 16 +- .../operator/ServiceHelperTest.java | 16 +- .../operator/helpers/CallBuilderTest.java | 20 +- .../operator/helpers/IngressHelperTest.java | 16 +- 18 files changed, 511 insertions(+), 371 deletions(-) create mode 100644 operator/src/main/java/oracle/kubernetes/operator/calls/AsyncRequestStep.java create mode 100644 operator/src/main/java/oracle/kubernetes/operator/calls/CallFactory.java create mode 100644 operator/src/main/java/oracle/kubernetes/operator/calls/CallResponse.java create mode 100644 operator/src/main/java/oracle/kubernetes/operator/calls/CallWrapper.java create mode 100644 operator/src/main/java/oracle/kubernetes/operator/calls/CancelableCall.java create mode 100644 operator/src/main/java/oracle/kubernetes/operator/calls/RequestParams.java create mode 100644 operator/src/main/java/oracle/kubernetes/operator/calls/RetryStrategy.java diff --git a/operator/src/main/java/oracle/kubernetes/operator/Main.java b/operator/src/main/java/oracle/kubernetes/operator/Main.java index 9d1e1b32823..d99eaecc4c0 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/Main.java +++ b/operator/src/main/java/oracle/kubernetes/operator/Main.java @@ -105,13 +105,14 @@ public class Main { private static final TuningParameters tuningAndConfig; static { try { - tuningAndConfig = TuningParameters.initializeInstance(factory, "/operator/config"); + TuningParameters.initializeInstance(factory, "/operator/config"); + tuningAndConfig = TuningParameters.getInstance(); } catch (IOException e) { LOGGER.warning(MessageKeys.EXCEPTION, e); throw new RuntimeException(e); } } - static final CallBuilderFactory callBuilderFactory = new CallBuilderFactory(tuningAndConfig); + static final CallBuilderFactory callBuilderFactory = new CallBuilderFactory(); private static final Container container = new Container(); private static final ScheduledExecutorService wrappedExecutorService = Engine.wrappedExecutorService("operator", diff --git a/operator/src/main/java/oracle/kubernetes/operator/TuningParameters.java b/operator/src/main/java/oracle/kubernetes/operator/TuningParameters.java index 32a5d5ed607..73c8aa1f28d 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/TuningParameters.java +++ b/operator/src/main/java/oracle/kubernetes/operator/TuningParameters.java @@ -9,10 +9,14 @@ public interface TuningParameters extends Map { - public static TuningParameters initializeInstance( + static TuningParameters initializeInstance( ThreadFactory factory, String mountPoint) throws IOException { return TuningParametersImpl.initializeInstance(factory, mountPoint); } + + public static TuningParameters getInstance() { + return TuningParametersImpl.getInstance(); + } public static class MainTuning { public final int statusUpdateTimeoutSeconds; diff --git a/operator/src/main/java/oracle/kubernetes/operator/TuningParametersImpl.java b/operator/src/main/java/oracle/kubernetes/operator/TuningParametersImpl.java index 87315ba9e1e..840a830409d 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/TuningParametersImpl.java +++ b/operator/src/main/java/oracle/kubernetes/operator/TuningParametersImpl.java @@ -3,19 +3,19 @@ package oracle.kubernetes.operator; -import java.io.IOException; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; - import oracle.kubernetes.operator.helpers.ConfigMapConsumer; import oracle.kubernetes.operator.logging.LoggingFacade; import oracle.kubernetes.operator.logging.LoggingFactory; import oracle.kubernetes.operator.logging.MessageKeys; +import java.io.IOException; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + public class TuningParametersImpl extends ConfigMapConsumer implements TuningParameters { private static final LoggingFacade LOGGER = LoggingFactory.getLogger("Operator", "Operator"); - private static TuningParametersImpl INSTANCE = null; + private static TuningParameters INSTANCE = null; private final ReadWriteLock lock = new ReentrantReadWriteLock(); private MainTuning main = null; @@ -23,24 +23,26 @@ public class TuningParametersImpl extends ConfigMapConsumer implements TuningPar private WatchTuning watch = null; private PodTuning pod = null; - public synchronized static TuningParameters initializeInstance( - ThreadFactory factory, String mountPoint) throws IOException { + synchronized static TuningParameters initializeInstance( + ThreadFactory factory, String mountPoint) throws IOException { if (INSTANCE == null) { INSTANCE = new TuningParametersImpl(factory, mountPoint); return INSTANCE; } throw new IllegalStateException(); } + + public synchronized static TuningParameters getInstance() { + return INSTANCE; + } private TuningParametersImpl(ThreadFactory factory, String mountPoint) throws IOException { - super(factory, mountPoint, () -> { - updateTuningParameters(); - }); + super(factory, mountPoint, TuningParametersImpl::updateTuningParameters); update(); } - public static void updateTuningParameters() { - INSTANCE.update(); + private static void updateTuningParameters() { + ((TuningParametersImpl) INSTANCE).update(); } private void update() { diff --git a/operator/src/main/java/oracle/kubernetes/operator/calls/AsyncRequestStep.java b/operator/src/main/java/oracle/kubernetes/operator/calls/AsyncRequestStep.java new file mode 100644 index 00000000000..68f8934741d --- /dev/null +++ b/operator/src/main/java/oracle/kubernetes/operator/calls/AsyncRequestStep.java @@ -0,0 +1,242 @@ +// Copyright 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.calls; + +import io.kubernetes.client.ApiCallback; +import io.kubernetes.client.ApiClient; +import io.kubernetes.client.ApiException; +import io.kubernetes.client.models.V1ListMeta; +import oracle.kubernetes.operator.helpers.CallBuilder; +import oracle.kubernetes.operator.helpers.ClientPool; +import oracle.kubernetes.operator.helpers.ResponseStep; +import oracle.kubernetes.operator.logging.LoggingFacade; +import oracle.kubernetes.operator.logging.LoggingFactory; +import oracle.kubernetes.operator.logging.MessageKeys; +import oracle.kubernetes.operator.work.Component; +import oracle.kubernetes.operator.work.NextAction; +import oracle.kubernetes.operator.work.Packet; +import oracle.kubernetes.operator.work.Step; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * A Step driven by an asynchronous call to the Kubernetes API, which results in a series of callbacks until canceled. + */ +public class AsyncRequestStep extends Step { + public static final String RESPONSE_COMPONENT_NAME = "response"; + private static final Random R = new Random(); + private static final int HIGH = 200; + private static final int LOW = 10; + private static final int SCALE = 100; + private static final int MAX = 10000; + + private final LoggingFacade LOGGER = LoggingFactory.getLogger("Operator", "Operator"); + private final ClientPool helper; + private final RequestParams requestParams; + private final CallFactory factory; + private int timeoutSeconds; + private final int maxRetryCount; + private final String fieldSelector; + private final String labelSelector; + private final String resourceVersion; + + public AsyncRequestStep(ResponseStep next, RequestParams requestParams, CallFactory factory, ClientPool helper, int timeoutSeconds, int maxRetryCount, String fieldSelector, String labelSelector, String resourceVersion) { + super(next); + this.helper = helper; + this.requestParams = requestParams; + this.factory = factory; + this.timeoutSeconds = timeoutSeconds; + this.maxRetryCount = maxRetryCount; + this.fieldSelector = fieldSelector; + this.labelSelector = labelSelector; + this.resourceVersion = resourceVersion; + next.setPrevious(this); + } + + @Override + public NextAction apply(Packet packet) { + // clear out earlier results + String cont = null; + RetryStrategy retry = null; + Component oldResponse = packet.getComponents().remove(RESPONSE_COMPONENT_NAME); + if (oldResponse != null) { + @SuppressWarnings("unchecked") + CallResponse old = oldResponse.getSPI(CallResponse.class); + if (old != null && old.result != null) { + // called again, access continue value, if available + cont = accessContinue(old.result); + } + + retry = oldResponse.getSPI(RetryStrategy.class); + } + String _continue = (cont != null) ? cont : ""; + if (retry == null) { + retry = new DefaultRetryStrategy(); + retry.setRetryStep(this); + } + RetryStrategy _retry = retry; + + LOGGER.fine(MessageKeys.ASYNC_REQUEST, requestParams.call, requestParams.namespace, requestParams.name, requestParams.body, fieldSelector, labelSelector, resourceVersion); + + AtomicBoolean didResume = new AtomicBoolean(false); + AtomicBoolean didRecycle = new AtomicBoolean(false); + ApiClient client = helper.take(); + return doSuspend((fiber) -> { + ApiCallback callback = new BaseApiCallback() { + @Override + public void onFailure(ApiException e, int statusCode, Map> responseHeaders) { + if (didResume.compareAndSet(false, true)) { + if (statusCode != CallBuilder.NOT_FOUND) { + LOGGER.info(MessageKeys.ASYNC_FAILURE, e, statusCode, responseHeaders, requestParams.call, requestParams.namespace, requestParams.name, requestParams.body, fieldSelector, labelSelector, resourceVersion); + } + + if (didRecycle.compareAndSet(false, true)) { + helper.recycle(client); + } + packet.getComponents().put(RESPONSE_COMPONENT_NAME, Component.createFor(RetryStrategy.class, _retry, new CallResponse(null, e, statusCode, responseHeaders))); + fiber.resume(packet); + } + } + + @Override + public void onSuccess(T result, int statusCode, Map> responseHeaders) { + if (didResume.compareAndSet(false, true)) { + LOGGER.fine(MessageKeys.ASYNC_SUCCESS, result, statusCode, responseHeaders); + + if (didRecycle.compareAndSet(false, true)) { + helper.recycle(client); + } + packet.getComponents().put(RESPONSE_COMPONENT_NAME, Component.createFor(new CallResponse<>(result, null, statusCode, responseHeaders))); + fiber.resume(packet); + } + } + }; + + try { + CancelableCall c = factory.generate(requestParams, client, _continue, callback); + + // timeout handling + fiber.owner.getExecutor().schedule(() -> { + if (didRecycle.compareAndSet(false, true)) { + // don't recycle on timeout because state is unknown + // usage.recycle(); + } + if (didResume.compareAndSet(false, true)) { + try { + c.cancel(); + } finally { + LOGGER.info(MessageKeys.ASYNC_TIMEOUT, requestParams.call, requestParams.namespace, requestParams.name, requestParams.body, fieldSelector, labelSelector, resourceVersion); + packet.getComponents().put(RESPONSE_COMPONENT_NAME, Component.createFor(RetryStrategy.class, _retry)); + fiber.resume(packet); + } + } + }, timeoutSeconds, TimeUnit.SECONDS); + } catch (Throwable t) { + LOGGER.warning(MessageKeys.ASYNC_FAILURE, t, 0, null, requestParams, requestParams.namespace, requestParams.name, requestParams.body, fieldSelector, labelSelector, resourceVersion); + if (didRecycle.compareAndSet(false, true)) { + // don't recycle on throwable because state is unknown + // usage.recycle(); + } + if (didResume.compareAndSet(false, true)) { + packet.getComponents().put(RESPONSE_COMPONENT_NAME, Component.createFor(RetryStrategy.class, _retry)); + fiber.resume(packet); + } + } + }); + } + + private static String accessContinue(Object result) { + String cont = ""; + if (result != null) { + try { + Method m = result.getClass().getMethod("getMetadata"); + Object meta = m.invoke(result); + if (meta instanceof V1ListMeta) { + return ((V1ListMeta) meta).getContinue(); + } + } catch (NoSuchMethodException | SecurityException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { + // no-op, no-log + } + } + return cont; + } + + private final class DefaultRetryStrategy implements RetryStrategy { + private long retryCount = 0; + private Step retryStep = null; + + @Override + public void setRetryStep(Step retryStep) { + this.retryStep = retryStep; + } + + @Override + public NextAction doPotentialRetry(Step conflictStep, Packet packet, ApiException e, int statusCode, + Map> responseHeaders) { + // Check statusCode, many statuses should not be retried + // https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md#http-status-codes + if (statusCode == 0 /* simple timeout */ || + statusCode == 429 /* StatusTooManyRequests */ || + statusCode == 500 /* StatusInternalServerError */ || + statusCode == 503 /* StatusServiceUnavailable */ || + statusCode == 504 /* StatusServerTimeout */) { + + // exponential back-off + long waitTime = Math.min((2 << ++retryCount) * SCALE, MAX) + (R.nextInt(HIGH - LOW) + LOW); + + if (statusCode == 0 || statusCode == 504 /* StatusServerTimeout */) { + // increase server timeout + timeoutSeconds *= 2; + } + + NextAction na = new NextAction(); + if (statusCode == 0 && retryCount <= maxRetryCount) { + na.invoke(retryStep, packet); + } else { + LOGGER.info(MessageKeys.ASYNC_RETRY, String.valueOf(waitTime)); + na.delay(retryStep, packet, waitTime, TimeUnit.MILLISECONDS); + } + return na; + } else if (statusCode == 409 /* Conflict */ && conflictStep != null) { + // Conflict is an optimistic locking failure. Therefore, we can't + // simply retry the request. Instead, application code needs to rebuild + // the request based on latest contents. If provided, a conflict step will do that. + + // exponential back-off + long waitTime = Math.min((2 << ++retryCount) * SCALE, MAX) + (R.nextInt(HIGH - LOW) + LOW); + + LOGGER.info(MessageKeys.ASYNC_RETRY, String.valueOf(waitTime)); + NextAction na = new NextAction(); + na.delay(conflictStep, packet, waitTime, TimeUnit.MILLISECONDS); + return na; + } + + // otherwise, we will not retry + return null; + } + + @Override + public void reset() { + retryCount = 0; + } + } + + private static abstract class BaseApiCallback implements ApiCallback { + @Override + public void onDownloadProgress(long bytesRead, long contentLength, boolean done) { + // no-op + } + + @Override + public void onUploadProgress(long bytesWritten, long contentLength, boolean done) { + // no-op + } + } +} diff --git a/operator/src/main/java/oracle/kubernetes/operator/calls/CallFactory.java b/operator/src/main/java/oracle/kubernetes/operator/calls/CallFactory.java new file mode 100644 index 00000000000..b7555e8f03f --- /dev/null +++ b/operator/src/main/java/oracle/kubernetes/operator/calls/CallFactory.java @@ -0,0 +1,13 @@ +// Copyright 2017,2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.calls; + +import io.kubernetes.client.ApiCallback; +import io.kubernetes.client.ApiClient; +import io.kubernetes.client.ApiException; + +@FunctionalInterface +public interface CallFactory { + CancelableCall generate(RequestParams requestParams, ApiClient client, String cont, ApiCallback callback) throws ApiException; +} diff --git a/operator/src/main/java/oracle/kubernetes/operator/calls/CallResponse.java b/operator/src/main/java/oracle/kubernetes/operator/calls/CallResponse.java new file mode 100644 index 00000000000..33ac6850e28 --- /dev/null +++ b/operator/src/main/java/oracle/kubernetes/operator/calls/CallResponse.java @@ -0,0 +1,23 @@ +// Copyright 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.calls; + +import io.kubernetes.client.ApiException; + +import java.util.List; +import java.util.Map; + +public final class CallResponse { + public final T result; + public final ApiException e; + public final int statusCode; + public final Map> responseHeaders; + + CallResponse(T result, ApiException e, int statusCode, Map> responseHeaders) { + this.result = result; + this.e = e; + this.statusCode = statusCode; + this.responseHeaders = responseHeaders; + } +} diff --git a/operator/src/main/java/oracle/kubernetes/operator/calls/CallWrapper.java b/operator/src/main/java/oracle/kubernetes/operator/calls/CallWrapper.java new file mode 100644 index 00000000000..235a7c99563 --- /dev/null +++ b/operator/src/main/java/oracle/kubernetes/operator/calls/CallWrapper.java @@ -0,0 +1,23 @@ +// Copyright 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.calls; + +import com.squareup.okhttp.Call; + +/** + * A wrapper for an OKHttp call to isolate its own callers. + */ +public class CallWrapper implements CancelableCall { + + private Call underlyingCall; + + public CallWrapper(Call underlyingCall) { + this.underlyingCall = underlyingCall; + } + + @Override + public void cancel() { + underlyingCall.cancel(); + } +} diff --git a/operator/src/main/java/oracle/kubernetes/operator/calls/CancelableCall.java b/operator/src/main/java/oracle/kubernetes/operator/calls/CancelableCall.java new file mode 100644 index 00000000000..71b8a30d098 --- /dev/null +++ b/operator/src/main/java/oracle/kubernetes/operator/calls/CancelableCall.java @@ -0,0 +1,15 @@ +// Copyright 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.calls; + +/** + * An interface for an asynchronous API invocation that can be canceled + */ +public interface CancelableCall { + + /** + * Cancels the active call. + */ + void cancel(); +} diff --git a/operator/src/main/java/oracle/kubernetes/operator/calls/RequestParams.java b/operator/src/main/java/oracle/kubernetes/operator/calls/RequestParams.java new file mode 100644 index 00000000000..760891b4aa1 --- /dev/null +++ b/operator/src/main/java/oracle/kubernetes/operator/calls/RequestParams.java @@ -0,0 +1,18 @@ +// Copyright 2017, 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.calls; + +public final class RequestParams { + public final String call; + public final String namespace; + public final String name; + public final Object body; + + public RequestParams(String call, String namespace, String name, Object body) { + this.call = call; + this.namespace = namespace; + this.name = name; + this.body = body; + } +} diff --git a/operator/src/main/java/oracle/kubernetes/operator/calls/RetryStrategy.java b/operator/src/main/java/oracle/kubernetes/operator/calls/RetryStrategy.java new file mode 100644 index 00000000000..dada361689f --- /dev/null +++ b/operator/src/main/java/oracle/kubernetes/operator/calls/RetryStrategy.java @@ -0,0 +1,43 @@ +// Copyright 2017, 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.calls; + +import io.kubernetes.client.ApiException; +import oracle.kubernetes.operator.helpers.ResponseStep; +import oracle.kubernetes.operator.work.NextAction; +import oracle.kubernetes.operator.work.Packet; +import oracle.kubernetes.operator.work.Step; + +import java.util.List; +import java.util.Map; + +/** + * Failed or timed-out call retry strategy + * + */ +public interface RetryStrategy { + /** + * Initialization that provides reference to step that should be invoked on a retry attempt + * @param retryStep Retry step + */ + void setRetryStep(Step retryStep); + + /** + * Called during {@link ResponseStep#onFailure(Packet, ApiException, int, Map)} to decide + * if another retry attempt will occur. + * @param conflictStep Conflict step, or null + * @param packet Packet + * @param e ApiException thrown by Kubernetes client; will be null for simple timeout + * @param statusCode HTTP response status code; will be 0 for simple timeout + * @param responseHeaders HTTP response headers; will be null for simple timeout + * @return Desired next action which should specify retryStep. Return null when call will not be retried. + */ + NextAction doPotentialRetry(Step conflictStep, Packet packet, ApiException e, int statusCode, Map> responseHeaders); + + /** + * Called when retry count, or other statistics, should be reset, such as when partial list + * was returned and new request for next portion of list (continue) is invoked. + */ + void reset(); +} diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java index 02eb263e720..28f2d36a6f4 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java @@ -13,14 +13,31 @@ import io.kubernetes.client.apis.CoreV1Api; import io.kubernetes.client.apis.ExtensionsV1beta1Api; import io.kubernetes.client.apis.VersionApi; -import io.kubernetes.client.models.*; +import io.kubernetes.client.models.V1ConfigMap; +import io.kubernetes.client.models.V1DeleteOptions; +import io.kubernetes.client.models.V1EventList; +import io.kubernetes.client.models.V1Job; +import io.kubernetes.client.models.V1Namespace; +import io.kubernetes.client.models.V1PersistentVolumeClaimList; +import io.kubernetes.client.models.V1PersistentVolumeList; +import io.kubernetes.client.models.V1Pod; +import io.kubernetes.client.models.V1PodList; +import io.kubernetes.client.models.V1Secret; +import io.kubernetes.client.models.V1Service; +import io.kubernetes.client.models.V1ServiceList; +import io.kubernetes.client.models.V1Status; +import io.kubernetes.client.models.V1SubjectAccessReview; +import io.kubernetes.client.models.V1TokenReview; +import io.kubernetes.client.models.V1beta1CustomResourceDefinition; +import io.kubernetes.client.models.V1beta1Ingress; +import io.kubernetes.client.models.V1beta1IngressList; +import io.kubernetes.client.models.VersionInfo; import oracle.kubernetes.operator.TuningParameters.CallBuilderTuning; -import oracle.kubernetes.operator.logging.LoggingFacade; -import oracle.kubernetes.operator.logging.LoggingFactory; -import oracle.kubernetes.operator.logging.MessageKeys; -import oracle.kubernetes.operator.work.Component; -import oracle.kubernetes.operator.work.NextAction; -import oracle.kubernetes.operator.work.Packet; +import oracle.kubernetes.operator.calls.AsyncRequestStep; +import oracle.kubernetes.operator.calls.CallFactory; +import oracle.kubernetes.operator.calls.CallWrapper; +import oracle.kubernetes.operator.calls.CancelableCall; +import oracle.kubernetes.operator.calls.RequestParams; import oracle.kubernetes.operator.work.Step; import oracle.kubernetes.weblogic.domain.v1.Domain; import oracle.kubernetes.weblogic.domain.v1.DomainList; @@ -28,13 +45,6 @@ import com.squareup.okhttp.Call; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; /** @@ -43,9 +53,6 @@ */ @SuppressWarnings({"WeakerAccess", "UnusedReturnValue"}) public class CallBuilder { - private static final LoggingFacade LOGGER = LoggingFactory.getLogger("Operator", "Operator"); - - static final String RESPONSE_COMPONENT_NAME = "response"; /** * HTTP status code for "Not Found" @@ -192,8 +199,12 @@ private com.squareup.okhttp.Call listDomainAsync(ApiClient client, String namesp } private final CallFactory LIST_DOMAIN = (requestParams, usage, cont, callback) - -> listDomainAsync(usage, requestParams.namespace, cont, callback); - + -> wrap(listDomainAsync(usage, requestParams.namespace, cont, callback)); + + private CancelableCall wrap(Call call) { + return new CallWrapper(call); + } + /** * Asynchronous step for listing domains * @param namespace Namespace @@ -226,7 +237,7 @@ private com.squareup.okhttp.Call replaceDomainAsync(ApiClient client, String nam } private final CallFactory REPLACE_DOMAIN = (requestParams, usage, cont, callback) - -> replaceDomainAsync(usage, requestParams.name, requestParams.namespace, (Domain) requestParams.body, callback); + -> wrap(replaceDomainAsync(usage, requestParams.name, requestParams.namespace, (Domain) requestParams.body, callback)); /** * Asynchronous step for replacing domain @@ -280,7 +291,7 @@ private com.squareup.okhttp.Call readConfigMapAsync(ApiClient client, String nam } private final CallFactory READ_CONFIGMAP = (requestParams, usage, cont, callback) - -> readConfigMapAsync(usage, requestParams.name, requestParams.namespace, callback); + -> wrap(readConfigMapAsync(usage, requestParams.name, requestParams.namespace, callback)); /** * Asynchronous step for reading config map @@ -298,7 +309,7 @@ private com.squareup.okhttp.Call createConfigMapAsync(ApiClient client, String n } private final CallFactory CREATE_CONFIGMAP = (requestParams, usage, cont, callback) - -> createConfigMapAsync(usage, requestParams.namespace, (V1ConfigMap) requestParams.body, callback); + -> wrap(createConfigMapAsync(usage, requestParams.namespace, (V1ConfigMap) requestParams.body, callback)); /** * Asynchronous step for creating config map @@ -316,7 +327,7 @@ private com.squareup.okhttp.Call replaceConfigMapAsync(ApiClient client, String } private final CallFactory REPLACE_CONFIGMAP = (requestParams, usage, cont, callback) - -> replaceConfigMapAsync(usage, requestParams.name, requestParams.namespace, (V1ConfigMap) requestParams.body, callback); + -> wrap(replaceConfigMapAsync(usage, requestParams.name, requestParams.namespace, (V1ConfigMap) requestParams.body, callback)); /** * Asynchronous step for replacing config map @@ -338,7 +349,7 @@ private com.squareup.okhttp.Call listPodAsync(ApiClient client, String namespace } private final CallFactory LIST_POD = (requestParams, usage, cont, callback) - -> listPodAsync(usage, requestParams.namespace, cont, callback); + -> wrap(listPodAsync(usage, requestParams.namespace, cont, callback)); /** * Asynchronous step for listing pods @@ -355,7 +366,7 @@ private com.squareup.okhttp.Call readPodAsync(ApiClient client, String name, Str } private final CallFactory READ_POD = (requestParams, usage, cont, callback) - -> readPodAsync(usage, requestParams.name, requestParams.namespace, callback); + -> wrap(readPodAsync(usage, requestParams.name, requestParams.namespace, callback)); /** * Asynchronous step for reading pod @@ -373,7 +384,7 @@ private com.squareup.okhttp.Call createPodAsync(ApiClient client, String namespa } private final CallFactory CREATE_POD = (requestParams, usage, cont, callback) - -> createPodAsync(usage, requestParams.namespace, (V1Pod) requestParams.body, callback); + -> wrap(createPodAsync(usage, requestParams.namespace, (V1Pod) requestParams.body, callback)); /** * Asynchronous step for creating pod @@ -391,7 +402,7 @@ private com.squareup.okhttp.Call deletePodAsync(ApiClient client, String name, S } private final CallFactory DELETE_POD = (requestParams, usage, cont, callback) - -> deletePodAsync(usage, requestParams.name, requestParams.namespace, (V1DeleteOptions) requestParams.body, callback); + -> wrap(deletePodAsync(usage, requestParams.name, requestParams.namespace, (V1DeleteOptions) requestParams.body, callback)); /** * Asynchronous step for deleting pod @@ -411,7 +422,7 @@ private com.squareup.okhttp.Call deleteCollectionPodAsync(ApiClient client, Stri } private final CallFactory DELETECOLLECTION_POD = (requestParams, usage, cont, callback) - -> deleteCollectionPodAsync(usage, requestParams.namespace, cont, callback); + -> wrap(deleteCollectionPodAsync(usage, requestParams.namespace, cont, callback)); /** * Asynchronous step for deleting collection of pods @@ -430,7 +441,7 @@ private com.squareup.okhttp.Call createJobAsync(ApiClient client, String namespa } private final CallFactory CREATE_JOB = (requestParams, usage, cont, callback) - -> createJobAsync(usage, requestParams.namespace, (V1Job) requestParams.body, callback); + -> wrap(createJobAsync(usage, requestParams.namespace, (V1Job) requestParams.body, callback)); /** * Asynchronous step for creating job @@ -448,7 +459,7 @@ private com.squareup.okhttp.Call deleteJobAsync(ApiClient client, String name, S } private final CallFactory DELETE_JOB = (requestParams, usage, cont, callback) - -> deleteJobAsync(usage, requestParams.name, requestParams.namespace, (V1DeleteOptions) requestParams.body, callback); + -> wrap(deleteJobAsync(usage, requestParams.name, requestParams.namespace, (V1DeleteOptions) requestParams.body, callback)); /** * Asynchronous step for deleting job @@ -486,7 +497,7 @@ private com.squareup.okhttp.Call listServiceAsync(ApiClient client, String names } private final CallFactory LIST_SERVICE = (requestParams, usage, cont, callback) - -> listServiceAsync(usage, requestParams.namespace, cont, callback); + -> wrap(listServiceAsync(usage, requestParams.namespace, cont, callback)); /** * Asynchronous step for listing services @@ -519,7 +530,7 @@ private com.squareup.okhttp.Call readServiceAsync(ApiClient client, String name, } private final CallFactory READ_SERVICE = (requestParams, usage, cont, callback) - -> readServiceAsync(usage, requestParams.name, requestParams.namespace, callback); + -> wrap(readServiceAsync(usage, requestParams.name, requestParams.namespace, callback)); /** * Asynchronous step for reading service @@ -537,7 +548,7 @@ private com.squareup.okhttp.Call createServiceAsync(ApiClient client, String nam } private final CallFactory CREATE_SERVICE = (requestParams, usage, cont, callback) - -> createServiceAsync(usage, requestParams.namespace, (V1Service) requestParams.body, callback); + -> wrap(createServiceAsync(usage, requestParams.namespace, (V1Service) requestParams.body, callback)); /** * Asynchronous step for creating service @@ -571,7 +582,7 @@ private com.squareup.okhttp.Call deleteServiceAsync(ApiClient client, String nam } private final CallFactory DELETE_SERVICE = (requestParams, usage, cont, callback) - -> deleteServiceAsync(usage, requestParams.name, requestParams.namespace, callback); + -> wrap(deleteServiceAsync(usage, requestParams.name, requestParams.namespace, callback)); /** * Asynchronous step for deleting service @@ -592,7 +603,7 @@ private com.squareup.okhttp.Call listEventAsync(ApiClient client, String namespa } private final CallFactory LIST_EVENT = (requestParams, usage, cont, callback) - -> listEventAsync(usage, requestParams.namespace, cont, callback); + -> wrap(listEventAsync(usage, requestParams.namespace, cont, callback)); /** * Asynchronous step for listing events @@ -630,7 +641,7 @@ private com.squareup.okhttp.Call listPersistentVolumeClaimAsync(ApiClient client } private final CallFactory LIST_PERSISTENTVOLUMECLAIM = (requestParams, usage, cont, callback) - -> listPersistentVolumeClaimAsync(usage, requestParams.namespace, cont, callback); + -> wrap(listPersistentVolumeClaimAsync(usage, requestParams.namespace, cont, callback)); /** * Asynchronous step for listing persistent volume claims @@ -665,7 +676,7 @@ private com.squareup.okhttp.Call readSecretAsync(ApiClient client, String name, } private final CallFactory READ_SECRET = (requestParams, usage, cont, callback) - -> readSecretAsync(usage, requestParams.name, requestParams.namespace, callback); + -> wrap(readSecretAsync(usage, requestParams.name, requestParams.namespace, callback)); /** * Create secret @@ -754,7 +765,7 @@ private com.squareup.okhttp.Call listIngressAsync(ApiClient client, String names } private final CallFactory LIST_INGRESS = (requestParams, usage, cont, callback) - -> listIngressAsync(usage, requestParams.namespace, cont, callback); + -> wrap(listIngressAsync(usage, requestParams.namespace, cont, callback)); /** * Asynchronous step for listing ingress @@ -787,7 +798,7 @@ private com.squareup.okhttp.Call readIngressAsync(ApiClient client, String name, } private final CallFactory READ_INGRESS = (requestParams, usage, cont, callback) - -> readIngressAsync(usage, requestParams.name, requestParams.namespace, callback); + -> wrap(readIngressAsync(usage, requestParams.name, requestParams.namespace, callback)); /** * Asynchronous step for reading ingress @@ -805,7 +816,7 @@ private com.squareup.okhttp.Call createIngressAsync(ApiClient client, String nam } private final CallFactory CREATE_INGRESS = (requestParams, usage, cont, callback) - -> createIngressAsync(usage, requestParams.namespace, (V1beta1Ingress) requestParams.body, callback); + -> wrap(createIngressAsync(usage, requestParams.namespace, (V1beta1Ingress) requestParams.body, callback)); /** * Asynchronous step for creating ingress @@ -823,7 +834,7 @@ private com.squareup.okhttp.Call replaceIngressAsync(ApiClient client, String na } private final CallFactory REPLACE_INGRESS = (requestParams, usage, cont, callback) - -> replaceIngressAsync(usage, requestParams.name, requestParams.namespace, (V1beta1Ingress) requestParams.body, callback); + -> wrap(replaceIngressAsync(usage, requestParams.name, requestParams.namespace, (V1beta1Ingress) requestParams.body, callback)); /** * Asynchronous step for replacing ingress @@ -860,7 +871,7 @@ private com.squareup.okhttp.Call deleteIngressAsync(ApiClient client, String nam } private final CallFactory DELETE_INGRESS = (requestParams, usage, cont, callback) - -> deleteIngressAsync(usage, requestParams.name, requestParams.namespace, (V1DeleteOptions) requestParams.body, callback); + -> wrap(deleteIngressAsync(usage, requestParams.name, requestParams.namespace, (V1DeleteOptions) requestParams.body, callback)); /** * Asynchronous step for deleting ingress @@ -873,269 +884,11 @@ private com.squareup.okhttp.Call deleteIngressAsync(ApiClient client, String nam public Step deleteIngressAsync(String name, String namespace, V1DeleteOptions deleteOptions, ResponseStep responseStep) { return createRequestAsync(responseStep, new RequestParams("deleteIngress", namespace, name, deleteOptions), DELETE_INGRESS); } - - private static abstract class BaseApiCallback implements ApiCallback { - @Override - public void onDownloadProgress(long bytesRead, long contentLength, boolean done) { - // no-op - } - @Override - public void onUploadProgress(long bytesWritten, long contentLength, boolean done) { - // no-op - } - } - @FunctionalInterface - interface CallFactory { - public Call generate(RequestParams requestParams, ApiClient client, String cont, ApiCallback callback) throws ApiException; - } - - static final class RequestParams { - public final String call; - public final String namespace; - public final String name; - public final Object body; - - public RequestParams(String call, String namespace, String name, Object body) { - this.call = call; - this.namespace = namespace; - this.name = name; - this.body = body; - } - } - - static final class CallResponse { - public final T result; - public final ApiException e; - public final int statusCode; - public final Map> responseHeaders; - - public CallResponse(T result, ApiException e, int statusCode, Map> responseHeaders) { - this.result = result; - this.e = e; - this.statusCode = statusCode; - this.responseHeaders = responseHeaders; - } - } - - /** - * Failed or timed-out call retry strategy - * - */ - public interface RetryStrategy { - /** - * Initialization that provides reference to step that should be invoked on a retry attempt - * @param retryStep Retry step - */ - public void setRetryStep(Step retryStep); - - /** - * Called during {@link ResponseStep#onFailure(Packet, ApiException, int, Map)} to decide - * if another retry attempt will occur. - * @param conflictStep Conflict step, or null - * @param packet Packet - * @param e ApiException thrown by Kubernetes client; will be null for simple timeout - * @param statusCode HTTP response status code; will be 0 for simple timeout - * @param responseHeaders HTTP response headers; will be null for simple timeout - * @return Desired next action which should specify retryStep. Return null when call will not be retried. - */ - public NextAction doPotentialRetry(Step conflictStep, Packet packet, ApiException e, int statusCode, Map> responseHeaders); - - /** - * Called when retry count, or other statistics, should be reset, such as when partial list - * was returned and new request for next portion of list (continue) is invoked. - */ - public void reset(); - } - - private static final Random R = new Random(); - private static final int HIGH = 200; - private static final int LOW = 10; - private static final int SCALE = 100; - private static final int MAX = 10000; - - private final class DefaultRetryStrategy implements RetryStrategy { - private long retryCount = 0; - private Step retryStep = null; - - @Override - public void setRetryStep(Step retryStep) { - this.retryStep = retryStep; - } - - @Override - public NextAction doPotentialRetry(Step conflictStep, Packet packet, ApiException e, int statusCode, - Map> responseHeaders) { - // Check statusCode, many statuses should not be retried - // https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md#http-status-codes - if (statusCode == 0 /* simple timeout */ || - statusCode == 429 /* StatusTooManyRequests */ || - statusCode == 500 /* StatusInternalServerError */ || - statusCode == 503 /* StatusServiceUnavailable */ || - statusCode == 504 /* StatusServerTimeout */) { - - // exponential back-off - long waitTime = Math.min((2 << ++retryCount) * SCALE, MAX) + (R.nextInt(HIGH - LOW) + LOW); - - if (statusCode == 0 || statusCode == 504 /* StatusServerTimeout */) { - // increase server timeout - timeoutSeconds *= 2; - } - - NextAction na = new NextAction(); - if (statusCode == 0 && retryCount <= maxRetryCount) { - na.invoke(retryStep, packet); - } else { - LOGGER.info(MessageKeys.ASYNC_RETRY, String.valueOf(waitTime)); - na.delay(retryStep, packet, waitTime, TimeUnit.MILLISECONDS); - } - return na; - } else if (statusCode == 409 /* Conflict */ && conflictStep != null) { - // Conflict is an optimistic locking failure. Therefore, we can't - // simply retry the request. Instead, application code needs to rebuild - // the request based on latest contents. If provided, a conflict step will do that. - - // exponential back-off - long waitTime = Math.min((2 << ++retryCount) * SCALE, MAX) + (R.nextInt(HIGH - LOW) + LOW); - - LOGGER.info(MessageKeys.ASYNC_RETRY, String.valueOf(waitTime)); - NextAction na = new NextAction(); - na.delay(conflictStep, packet, waitTime, TimeUnit.MILLISECONDS); - return na; - } - - // otherwise, we will not retry - return null; - } - - @Override - public void reset() { - retryCount = 0; - } - } - - private class AsyncRequestStep extends Step { - private final RequestParams requestParams; - private final CallFactory factory; - - public AsyncRequestStep(ResponseStep next, RequestParams requestParams, CallFactory factory) { - super(next); - this.requestParams = requestParams; - this.factory = factory; - next.setPrevious(this); - } - - @Override - public NextAction apply(Packet packet) { - // clear out earlier results - String cont = null; - RetryStrategy retry = null; - Component oldResponse = packet.getComponents().remove(RESPONSE_COMPONENT_NAME); - if (oldResponse != null) { - @SuppressWarnings("unchecked") - CallResponse old = oldResponse.getSPI(CallResponse.class); - if (old != null && old.result != null) { - // called again, access continue value, if available - cont = accessContinue(old.result); - } - - retry = oldResponse.getSPI(RetryStrategy.class); - } - String _continue = (cont != null) ? cont : ""; - if (retry == null) { - retry = new DefaultRetryStrategy(); - retry.setRetryStep(this); - } - RetryStrategy _retry = retry; - - LOGGER.fine(MessageKeys.ASYNC_REQUEST, requestParams.call, requestParams.namespace, requestParams.name, requestParams.body, fieldSelector, labelSelector, resourceVersion); - - AtomicBoolean didResume = new AtomicBoolean(false); - AtomicBoolean didRecycle = new AtomicBoolean(false); - ApiClient client = helper.take(); - return doSuspend((fiber) -> { - ApiCallback callback = new BaseApiCallback() { - @Override - public void onFailure(ApiException e, int statusCode, Map> responseHeaders) { - if (didResume.compareAndSet(false, true)) { - if (statusCode != NOT_FOUND) { - LOGGER.info(MessageKeys.ASYNC_FAILURE, e, statusCode, responseHeaders, requestParams.call, requestParams.namespace, requestParams.name, requestParams.body, fieldSelector, labelSelector, resourceVersion); - } - - if (didRecycle.compareAndSet(false, true)) { - helper.recycle(client); - } - packet.getComponents().put(RESPONSE_COMPONENT_NAME, Component.createFor(RetryStrategy.class, _retry, new CallResponse(null, e, statusCode, responseHeaders))); - fiber.resume(packet); - } - } - - @Override - public void onSuccess(T result, int statusCode, Map> responseHeaders) { - if (didResume.compareAndSet(false, true)) { - LOGGER.fine(MessageKeys.ASYNC_SUCCESS, result, statusCode, responseHeaders); - - if (didRecycle.compareAndSet(false, true)) { - helper.recycle(client); - } - packet.getComponents().put(RESPONSE_COMPONENT_NAME, Component.createFor(new CallResponse(result, null, statusCode, responseHeaders))); - fiber.resume(packet); - } - } - }; - - try { - Call c = factory.generate(requestParams, client, _continue, callback); - - // timeout handling - fiber.owner.getExecutor().schedule(() -> { - if (didRecycle.compareAndSet(false, true)) { - // don't recycle on timeout because state is unknown - // usage.recycle(); - } - if (didResume.compareAndSet(false, true)) { - try { - c.cancel(); - } finally { - LOGGER.info(MessageKeys.ASYNC_TIMEOUT, requestParams.call, requestParams.namespace, requestParams.name, requestParams.body, fieldSelector, labelSelector, resourceVersion); - packet.getComponents().put(RESPONSE_COMPONENT_NAME, Component.createFor(RetryStrategy.class, _retry)); - fiber.resume(packet); - } - } - }, timeoutSeconds, TimeUnit.SECONDS); - } catch (Throwable t) { - LOGGER.warning(MessageKeys.ASYNC_FAILURE, t, 0, null, requestParams, requestParams.namespace, requestParams.name, requestParams.body, fieldSelector, labelSelector, resourceVersion); - if (didRecycle.compareAndSet(false, true)) { - // don't recycle on throwable because state is unknown - // usage.recycle(); - } - if (didResume.compareAndSet(false, true)) { - packet.getComponents().put(RESPONSE_COMPONENT_NAME, Component.createFor(RetryStrategy.class, _retry)); - fiber.resume(packet); - } - } - }); - } - } - private Step createRequestAsync(ResponseStep next, RequestParams requestParams, CallFactory factory) { - return new AsyncRequestStep(next, requestParams, factory); + return new AsyncRequestStep<>(next, requestParams, factory, helper, timeoutSeconds, maxRetryCount, fieldSelector, labelSelector, resourceVersion); } - private static String accessContinue(Object result) { - String cont = ""; - if (result != null) { - try { - Method m = result.getClass().getMethod("getMetadata"); - Object meta = m.invoke(result); - if (meta instanceof V1ListMeta) { - return ((V1ListMeta) meta).getContinue(); - } - } catch (NoSuchMethodException | SecurityException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { - // no-op, no-log - } - } - return cont; - } + } \ No newline at end of file diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilderFactory.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilderFactory.java index 78bd937abe8..ee207f899f6 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilderFactory.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilderFactory.java @@ -8,8 +8,8 @@ public class CallBuilderFactory { private final TuningParameters tuning; - public CallBuilderFactory(TuningParameters tuning) { - this.tuning = tuning; + public CallBuilderFactory() { + this.tuning = TuningParameters.getInstance(); } public CallBuilder create() { diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/ResponseStep.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/ResponseStep.java index 28eaa7aecc7..12c722d7064 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/ResponseStep.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/ResponseStep.java @@ -3,12 +3,10 @@ package oracle.kubernetes.operator.helpers; -import java.util.List; -import java.util.Map; - import io.kubernetes.client.ApiException; -import oracle.kubernetes.operator.helpers.CallBuilder.CallResponse; -import oracle.kubernetes.operator.helpers.CallBuilder.RetryStrategy; +import oracle.kubernetes.operator.calls.AsyncRequestStep; +import oracle.kubernetes.operator.calls.CallResponse; +import oracle.kubernetes.operator.calls.RetryStrategy; import oracle.kubernetes.operator.logging.LoggingFacade; import oracle.kubernetes.operator.logging.LoggingFactory; import oracle.kubernetes.operator.logging.MessageKeys; @@ -16,6 +14,9 @@ import oracle.kubernetes.operator.work.Packet; import oracle.kubernetes.operator.work.Step; +import java.util.List; +import java.util.Map; + /** * Step to receive response of Kubernetes API server call. * @@ -36,7 +37,7 @@ public ResponseStep(Step nextStep) { super(nextStep); } - final void setPrevious(Step previousStep) { + public final void setPrevious(Step previousStep) { this.previousStep = previousStep; } @@ -67,7 +68,7 @@ public final NextAction apply(Packet packet) { if (previousStep != nextAction.getNext()) { // not a retry, clear out old response - packet.getComponents().remove(CallBuilder.RESPONSE_COMPONENT_NAME); + packet.getComponents().remove(AsyncRequestStep.RESPONSE_COMPONENT_NAME); } return nextAction; diff --git a/operator/src/test/java/oracle/kubernetes/operator/HealthCheckHelperTest.java b/operator/src/test/java/oracle/kubernetes/operator/HealthCheckHelperTest.java index a400087bb72..e5e9cbceca1 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/HealthCheckHelperTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/HealthCheckHelperTest.java @@ -14,6 +14,14 @@ import oracle.kubernetes.operator.work.Component; import oracle.kubernetes.operator.work.ContainerResolver; +import java.io.ByteArrayOutputStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.logging.Handler; +import java.util.logging.SimpleFormatter; +import java.util.logging.StreamHandler; + import org.junit.After; import org.junit.Assert; import org.junit.Assume; @@ -21,12 +29,6 @@ import org.junit.Ignore; import org.junit.Test; -import java.io.ByteArrayOutputStream; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.logging.*; - public class HealthCheckHelperTest { private HealthCheckHelper defaultHealthCheckHelper; @@ -65,7 +67,7 @@ public void tearDown() throws Exception { public void testDefaultNamespace() throws Exception { ContainerResolver.getInstance().getContainer().getComponents().put( ProcessingConstants.MAIN_COMPONENT_NAME, - Component.createFor(new CallBuilderFactory(null))); + Component.createFor(new CallBuilderFactory())); defaultHealthCheckHelper.performSecurityChecks("default"); hdlr.flush(); @@ -85,7 +87,7 @@ public void testDefaultNamespace() throws Exception { public void testUnitTestNamespace() throws Exception { ContainerResolver.getInstance().getContainer().getComponents().put( ProcessingConstants.MAIN_COMPONENT_NAME, - Component.createFor(new CallBuilderFactory(null))); + Component.createFor(new CallBuilderFactory())); unitHealthCheckHelper.performSecurityChecks("weblogic-operator-account"); hdlr.flush(); @@ -106,7 +108,7 @@ public void testAccountNoPrivs() throws Exception { ContainerResolver.getInstance().getContainer().getComponents().put( ProcessingConstants.MAIN_COMPONENT_NAME, - Component.createFor(new CallBuilderFactory(null))); + Component.createFor(new CallBuilderFactory())); unitHealthCheckHelper.performSecurityChecks("unit-test-svc-account-no-privs"); hdlr.flush(); @@ -119,7 +121,7 @@ public void testAccountNoPrivs() throws Exception { // Create a named namespace private V1Namespace createNamespace(String name) throws Exception { - CallBuilderFactory factory = new CallBuilderFactory(null); + CallBuilderFactory factory = new CallBuilderFactory(); try { V1Namespace existing = factory.create().readNamespace(name); if (existing != null) diff --git a/operator/src/test/java/oracle/kubernetes/operator/SecretHelperTest.java b/operator/src/test/java/oracle/kubernetes/operator/SecretHelperTest.java index 0fab21541dc..e1e838fd879 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/SecretHelperTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/SecretHelperTest.java @@ -13,15 +13,15 @@ import oracle.kubernetes.operator.helpers.CallBuilderFactory; import oracle.kubernetes.operator.helpers.SecretHelper; +import java.util.HashMap; +import java.util.Map; + import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; -import java.util.HashMap; -import java.util.Map; - @Ignore public class SecretHelperTest { @@ -37,7 +37,7 @@ public class SecretHelperTest { @Before public void setUp() throws Exception { - CallBuilderFactory factory = new CallBuilderFactory(null); + CallBuilderFactory factory = new CallBuilderFactory(); // Determine if 1.8 since some bugs with kubernetes-client / java and secrets VersionInfo verInfo = factory.create().readVersionCode(); if ("1".equals(verInfo.getMajor()) && "8".equals(verInfo.getMinor())) { @@ -137,7 +137,7 @@ public void testUnitTestNamespace() throws Exception { // Create a named secret with username / password in specified name private V1Secret createSecret(String name, String namespace) throws Exception { - CallBuilderFactory factory = new CallBuilderFactory(null); + CallBuilderFactory factory = new CallBuilderFactory(); try { V1Secret existing = factory.create().readSecret(name, namespace); if (existing != null) @@ -177,7 +177,7 @@ private V1Secret createSecret(String name, String namespace) throws Exception { // Create a named secret with no username / password in specified namespace private V1Secret createInvalidSecret(String name, String namespace) throws Exception { - CallBuilderFactory factory = new CallBuilderFactory(null); + CallBuilderFactory factory = new CallBuilderFactory(); try { V1Secret existing = factory.create().readSecret(name, namespace); if (existing != null) @@ -210,14 +210,14 @@ private V1Status deleteSecret(String name, String namespace) throws Exception { if (isVersion18) return null; - CallBuilderFactory factory = new CallBuilderFactory(null); + CallBuilderFactory factory = new CallBuilderFactory(); return factory.create().deleteSecret(name, namespace, new V1DeleteOptions()); } // Create a named namespace private V1Namespace createNamespace(String name) throws Exception { - CallBuilderFactory factory = new CallBuilderFactory(null); + CallBuilderFactory factory = new CallBuilderFactory(); try { V1Namespace existing = factory.create().readNamespace(name); if (existing != null) diff --git a/operator/src/test/java/oracle/kubernetes/operator/ServiceHelperTest.java b/operator/src/test/java/oracle/kubernetes/operator/ServiceHelperTest.java index 411742466dd..227f52e2c2a 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/ServiceHelperTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/ServiceHelperTest.java @@ -8,8 +8,6 @@ import io.kubernetes.client.models.V1Service; import io.kubernetes.client.models.V1ServiceList; import io.kubernetes.client.models.V1ServicePort; -import oracle.kubernetes.weblogic.domain.v1.Domain; -import oracle.kubernetes.weblogic.domain.v1.DomainSpec; import oracle.kubernetes.operator.helpers.CallBuilder; import oracle.kubernetes.operator.helpers.CallBuilderFactory; import oracle.kubernetes.operator.helpers.DomainPresenceInfo; @@ -19,6 +17,11 @@ import oracle.kubernetes.operator.work.Fiber; import oracle.kubernetes.operator.work.Packet; import oracle.kubernetes.operator.work.Step; +import oracle.kubernetes.weblogic.domain.v1.Domain; +import oracle.kubernetes.weblogic.domain.v1.DomainSpec; + +import java.util.List; +import java.util.Map; import org.junit.After; import org.junit.Assert; @@ -26,9 +29,6 @@ import org.junit.Ignore; import org.junit.Test; -import java.util.List; -import java.util.Map; - @Ignore public class ServiceHelperTest { @@ -38,7 +38,7 @@ public class ServiceHelperTest { @Before public void startClean() throws Exception { - CallBuilderFactory factory = new CallBuilderFactory(null); + CallBuilderFactory factory = new CallBuilderFactory(); // Delete the service if left around. System.out.println("Deleting service pre-test"); @@ -54,7 +54,7 @@ public void startClean() throws Exception { @After public void tearDown() throws Exception { - CallBuilderFactory factory = new CallBuilderFactory(null); + CallBuilderFactory factory = new CallBuilderFactory(); // Delete the service if we created one. if (serviceCreated) { @@ -66,7 +66,7 @@ public void tearDown() throws Exception { @Test public void createReadListUpdate() throws Exception { - CallBuilderFactory factory = new CallBuilderFactory(null); + CallBuilderFactory factory = new CallBuilderFactory(); // Domain Domain dom = new Domain(); diff --git a/operator/src/test/java/oracle/kubernetes/operator/helpers/CallBuilderTest.java b/operator/src/test/java/oracle/kubernetes/operator/helpers/CallBuilderTest.java index ee5169487eb..a8176c70dc0 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/helpers/CallBuilderTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/helpers/CallBuilderTest.java @@ -3,8 +3,13 @@ package oracle.kubernetes.operator.helpers; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import io.kubernetes.client.ApiException; +import oracle.kubernetes.operator.work.Engine; +import oracle.kubernetes.operator.work.Fiber.CompletionCallback; +import oracle.kubernetes.operator.work.NextAction; +import oracle.kubernetes.operator.work.Packet; +import oracle.kubernetes.operator.work.Step; +import oracle.kubernetes.weblogic.domain.v1.DomainList; import java.util.ArrayList; import java.util.Collections; @@ -17,13 +22,8 @@ import org.junit.Ignore; import org.junit.Test; -import io.kubernetes.client.ApiException; -import oracle.kubernetes.weblogic.domain.v1.DomainList; -import oracle.kubernetes.operator.work.Engine; -import oracle.kubernetes.operator.work.Packet; -import oracle.kubernetes.operator.work.Step; -import oracle.kubernetes.operator.work.Fiber.CompletionCallback; -import oracle.kubernetes.operator.work.NextAction; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; // TODO fix this test so that it will not fail with 404 not found when run against a server // that does not have the domain CRD defined. @@ -80,7 +80,7 @@ public SetupStep(Step next) { public NextAction apply(Packet packet) { String namespace = "default"; - CallBuilderFactory factory = new CallBuilderFactory(null); + CallBuilderFactory factory = new CallBuilderFactory(); Step list = factory.create().listDomainAsync(namespace, new ResponseStep(null) { @SuppressWarnings("unchecked") diff --git a/operator/src/test/java/oracle/kubernetes/operator/helpers/IngressHelperTest.java b/operator/src/test/java/oracle/kubernetes/operator/helpers/IngressHelperTest.java index 1e4f8331ae9..cfe94ffc770 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/helpers/IngressHelperTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/helpers/IngressHelperTest.java @@ -16,8 +16,6 @@ import io.kubernetes.client.models.V1beta1IngressRule; import io.kubernetes.client.models.V1beta1IngressSpec; import oracle.kubernetes.operator.ProcessingConstants; -import oracle.kubernetes.weblogic.domain.v1.Domain; -import oracle.kubernetes.weblogic.domain.v1.DomainSpec; import oracle.kubernetes.operator.wlsconfig.WlsClusterConfig; import oracle.kubernetes.operator.wlsconfig.WlsDomainConfig; import oracle.kubernetes.operator.wlsconfig.WlsServerConfig; @@ -27,6 +25,12 @@ import oracle.kubernetes.operator.work.Fiber.CompletionCallback; import oracle.kubernetes.operator.work.Packet; import oracle.kubernetes.operator.work.Step; +import oracle.kubernetes.weblogic.domain.v1.Domain; +import oracle.kubernetes.weblogic.domain.v1.DomainSpec; + +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import org.junit.After; import org.junit.Assert; @@ -34,10 +38,6 @@ import org.junit.Ignore; import org.junit.Test; -import java.util.List; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; - /** * To test Ingress Helper */ @@ -127,7 +127,7 @@ public void setUp() throws ApiException { @After public void tearDown() throws ApiException { - CallBuilderFactory factory = new CallBuilderFactory(null); + CallBuilderFactory factory = new CallBuilderFactory(); try { factory.create().deleteIngress(ingressName, namespace, new V1DeleteOptions()); } catch (ApiException api) { @@ -165,7 +165,7 @@ public void onThrowable(Packet packet, Throwable throwable) { } // Now check - CallBuilderFactory factory = new CallBuilderFactory(null); + CallBuilderFactory factory = new CallBuilderFactory(); V1beta1Ingress v1beta1Ingress = factory.create().readIngress(ingressName, namespace); List v1beta1HTTPIngressPaths = getPathArray(v1beta1Ingress); From 45eb15c3f4122be8adea95cc7e27912b4434dc93 Mon Sep 17 00:00:00 2001 From: Russell Gold Date: Fri, 20 Apr 2018 15:14:19 -0400 Subject: [PATCH 055/186] initial unit tests for AsyncRequestStep --- .../operator/calls/AsyncRequestStepTest.java | 250 ++++++++++++++++++ 1 file changed, 250 insertions(+) create mode 100644 operator/src/test/java/oracle/kubernetes/operator/calls/AsyncRequestStepTest.java diff --git a/operator/src/test/java/oracle/kubernetes/operator/calls/AsyncRequestStepTest.java b/operator/src/test/java/oracle/kubernetes/operator/calls/AsyncRequestStepTest.java new file mode 100644 index 00000000000..c6a6bd70421 --- /dev/null +++ b/operator/src/test/java/oracle/kubernetes/operator/calls/AsyncRequestStepTest.java @@ -0,0 +1,250 @@ +// Copyright 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.calls; + +import io.kubernetes.client.ApiCallback; +import io.kubernetes.client.ApiClient; +import io.kubernetes.client.ApiException; +import oracle.kubernetes.TestUtils; +import oracle.kubernetes.operator.helpers.ClientPool; +import oracle.kubernetes.operator.helpers.ResponseStep; +import oracle.kubernetes.operator.work.Engine; +import oracle.kubernetes.operator.work.Fiber; +import oracle.kubernetes.operator.work.NextAction; +import oracle.kubernetes.operator.work.Packet; + +import com.meterware.simplestub.Memento; + +import java.net.HttpURLConnection; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static com.meterware.simplestub.Stub.createStrictStub; +import static com.meterware.simplestub.Stub.createStub; +import static oracle.kubernetes.operator.calls.AsyncRequestStep.RESPONSE_COMPONENT_NAME; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.notNullValue; +import static org.junit.Assert.assertTrue; + +public class AsyncRequestStepTest { + + private static final int TIMEOUT_SECONDS = 10; + private static final int MAX_RETRY_COUNT = 2; + private Packet packet = new Packet(); + private Schedule schedule = createStrictStub(Schedule.class); + private Engine engine = new Engine(schedule); + private Fiber fiber = engine.createFiber(); + private RequestParams requestParams = new RequestParams("testcall", "junit", "testName", "body"); + private CallFactoryStub callFactory = new CallFactoryStub(); + private CompletionCallbackStub completionCallback = new CompletionCallbackStub(); + private TestStep nextStep = new TestStep(); + private ClientPool helper = ClientPool.getInstance(); + private List mementos = new ArrayList<>(); + + private final AsyncRequestStep asyncRequestStep + = new AsyncRequestStep<>(nextStep, requestParams, callFactory, helper, TIMEOUT_SECONDS, MAX_RETRY_COUNT, null, null, null); + + @Before + public void setUp() throws Exception { + mementos.add(TestUtils.silenceOperatorLogger()); + + fiber.start(asyncRequestStep, packet, completionCallback); + } + + @After + public void tearDown() throws Exception { + for (Memento memento : mementos) memento.revert(); + } + + @Test + public void afterFiberStarted_requestSent() throws Exception { + assertTrue(callFactory.invokedWith(requestParams)); + } + + @Test + public void afterFiber_timeoutStepScheduled() throws Exception { + assertTrue(schedule.containsStepAt(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + } + + @Test + public void afterSuccessfulCallback_nextStepAppliedWithValue() throws Exception { + callFactory.sendSuccessfulCallback(17); + + assertThat(nextStep.result, equalTo(17)); + } + + @Test + public void afterSuccessfulCallback_packetDoesNotContainsResponse() throws Exception { + schedule.execute(() -> callFactory.sendSuccessfulCallback(17)); + + assertThat(packet.getComponents(), not(hasKey(RESPONSE_COMPONENT_NAME))); + } + + @Test + public void afterFailedCallback_packetContainsRetryStrategy() throws Exception { + schedule.execute(() -> + callFactory.sendFailedCallback(new ApiException("test failure"), HttpURLConnection.HTTP_UNAVAILABLE)); + + assertThat(packet.getComponents().get(RESPONSE_COMPONENT_NAME).getSPI(RetryStrategy.class), notNullValue()); + } + + // todo tests + // after timeout, packet contains retry strategy + // after either failure, setting time to before the timeout causes new request + // after new request, success leads to invocation + + + static class TestStep extends ResponseStep { + private Integer result; + + TestStep() { + super(null); + } + + @Override + public NextAction onSuccess(Packet packet, Integer result, int statusCode, Map> responseHeaders) { + this.result = result; + return null; + } + } + + @SuppressWarnings("SameParameterValue") + static class CallFactoryStub implements CallFactory { + + private RequestParams requestParams; + private ApiCallback callback; + + private boolean invokedWith(RequestParams requestParams) { + return requestParams == this.requestParams; + } + + private void sendSuccessfulCallback(Integer callbackValue) { + callback.onSuccess(callbackValue, HttpURLConnection.HTTP_OK, Collections.emptyMap()); + } + + private void sendFailedCallback(ApiException exception, int statusCode) { + callback.onFailure(exception, statusCode, Collections.emptyMap()); + } + + @Override + public CancelableCall generate(RequestParams requestParams, ApiClient client, String cont, ApiCallback callback) throws ApiException { + this.requestParams = requestParams; + this.callback = callback; + + return new CancelableCallStub(); + } + } + + static class CancelableCallStub implements CancelableCall { + private boolean canceled; + + @Override + public void cancel() { + canceled = true; + } + } + + static class CompletionCallbackStub implements Fiber.CompletionCallback { + private Packet packet; + private Throwable throwable; + + @Override + public void onCompletion(Packet packet) { + this.packet = packet; + } + + @Override + public void onThrowable(Packet packet, Throwable throwable) { + this.packet = packet; + this.throwable = throwable; + } + } + + static class ScheduledItem implements Comparable { + private long atTime; + private Runnable runnable; + + ScheduledItem(long atTime, Runnable runnable) { + this.atTime = atTime; + this.runnable = runnable; + } + + @Override + public int compareTo(@Nonnull ScheduledItem o) { + return Long.compare(atTime, o.atTime); + } + } + + static abstract class Schedule implements ScheduledExecutorService { + /** current time in milliseconds. */ + private long currentTime = 0; + + private SortedSet scheduledItems = new TreeSet<>(); + private Queue queue = new ArrayDeque<>(); + private Runnable current; + + @Override + @Nonnull public ScheduledFuture schedule(@Nonnull Runnable command, long delay, @Nonnull TimeUnit unit) { + scheduledItems.add(new ScheduledItem(unit.toMillis(delay), command)); + return createStub(ScheduledFuture.class); + } + + @Override + public void execute(@Nullable Runnable command) { + queue.add(command); + if (current == null) + runNextRunnable(); + } + + private void runNextRunnable() { + while (queue.peek() != null) { + current = queue.poll(); + current.run(); + current = null; + } + + } + + void setTime(long time, TimeUnit unit) { + long newTime = unit.toMillis(time); + if (newTime < currentTime) + throw new IllegalStateException("Attempt to move clock backwards from " + currentTime + " to " + newTime); + + for (Iterator it = scheduledItems.iterator(); it.hasNext();) { + ScheduledItem item = it.next(); + if (item.atTime > newTime) break; + it.remove(); + execute(item.runnable); + } + + currentTime = newTime; + } + + boolean containsStepAt(int timeoutSeconds, TimeUnit unit) { + for (ScheduledItem scheduledItem : scheduledItems) + if (scheduledItem.atTime == unit.toMillis(timeoutSeconds)) return true; + return false; + } + } +} \ No newline at end of file From 95d4a613fd74227298eaf85f35c81cc8ddbc9326 Mon Sep 17 00:00:00 2001 From: Russell Gold Date: Sun, 22 Apr 2018 13:21:31 -0400 Subject: [PATCH 056/186] Added more unit tests --- .../helpers/AsyncRequestStepFactory.java | 14 ++++++ .../operator/helpers/CallBuilder.java | 4 +- .../operator/calls/AsyncRequestStepTest.java | 44 +++++++++++++++---- 3 files changed, 53 insertions(+), 9 deletions(-) create mode 100644 operator/src/main/java/oracle/kubernetes/operator/helpers/AsyncRequestStepFactory.java diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/AsyncRequestStepFactory.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/AsyncRequestStepFactory.java new file mode 100644 index 00000000000..bd3e658f4a8 --- /dev/null +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/AsyncRequestStepFactory.java @@ -0,0 +1,14 @@ +// Copyright 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.helpers; + +import oracle.kubernetes.operator.calls.CallFactory; +import oracle.kubernetes.operator.calls.RequestParams; +import oracle.kubernetes.operator.work.Step; + +public interface AsyncRequestStepFactory { + Step createRequestAsync(ResponseStep next, RequestParams requestParams, CallFactory factory, + ClientPool helper, int timeoutSeconds, int maxRetryCount, + String fieldSelector, String labelSelector, String resourceVersion); +} diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java index bef56b26eba..4065299c0b4 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java @@ -977,8 +977,10 @@ public Step deleteIngressAsync(String name, String namespace, V1DeleteOptions de } + private static final AsyncRequestStepFactory STEP_FACTORY = AsyncRequestStep::new; + private Step createRequestAsync(ResponseStep next, RequestParams requestParams, CallFactory factory) { - return new AsyncRequestStep<>(next, requestParams, factory, helper, timeoutSeconds, maxRetryCount, fieldSelector, labelSelector, resourceVersion); + return STEP_FACTORY.createRequestAsync(next, requestParams, factory, helper, timeoutSeconds, maxRetryCount, fieldSelector, labelSelector, resourceVersion); } diff --git a/operator/src/test/java/oracle/kubernetes/operator/calls/AsyncRequestStepTest.java b/operator/src/test/java/oracle/kubernetes/operator/calls/AsyncRequestStepTest.java index c6a6bd70421..2fd3f8c27bc 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/calls/AsyncRequestStepTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/calls/AsyncRequestStepTest.java @@ -87,6 +87,15 @@ public void afterFiber_timeoutStepScheduled() throws Exception { assertTrue(schedule.containsStepAt(TIMEOUT_SECONDS, TimeUnit.SECONDS)); } + @Test + public void afterTimeout_newRequestSent() throws Exception { + callFactory.clearRequest(); + + schedule.setTime(TIMEOUT_SECONDS, TimeUnit.SECONDS); + + assertTrue(callFactory.invokedWith(requestParams)); + } + @Test public void afterSuccessfulCallback_nextStepAppliedWithValue() throws Exception { callFactory.sendSuccessfulCallback(17); @@ -103,16 +112,31 @@ public void afterSuccessfulCallback_packetDoesNotContainsResponse() throws Excep @Test public void afterFailedCallback_packetContainsRetryStrategy() throws Exception { - schedule.execute(() -> - callFactory.sendFailedCallback(new ApiException("test failure"), HttpURLConnection.HTTP_UNAVAILABLE)); + sendFailedCallback(HttpURLConnection.HTTP_UNAVAILABLE); assertThat(packet.getComponents().get(RESPONSE_COMPONENT_NAME).getSPI(RetryStrategy.class), notNullValue()); } + private void sendFailedCallback(int statusCode) { + schedule.execute(() -> callFactory.sendFailedCallback(new ApiException("test failure"), statusCode)); + } + + @Test + public void afterFailedCallback_retrySentAfterDelay() throws Exception { + sendFailedCallback(HttpURLConnection.HTTP_UNAVAILABLE); + callFactory.clearRequest(); + + schedule.setTime(TIMEOUT_SECONDS-1, TimeUnit.SECONDS); + + assertTrue(callFactory.invokedWith(requestParams)); + } + // todo tests - // after timeout, packet contains retry strategy - // after either failure, setting time to before the timeout causes new request - // after new request, success leads to invocation + // can new request clear timeout action? + // what is accessContinue? + // test CONFLICT (409) status + // no retry if status not handled + // test exceeded retry count static class TestStep extends ResponseStep { @@ -135,15 +159,19 @@ static class CallFactoryStub implements CallFactory { private RequestParams requestParams; private ApiCallback callback; - private boolean invokedWith(RequestParams requestParams) { + void clearRequest() { + requestParams = null; + } + + boolean invokedWith(RequestParams requestParams) { return requestParams == this.requestParams; } - private void sendSuccessfulCallback(Integer callbackValue) { + void sendSuccessfulCallback(Integer callbackValue) { callback.onSuccess(callbackValue, HttpURLConnection.HTTP_OK, Collections.emptyMap()); } - private void sendFailedCallback(ApiException exception, int statusCode) { + void sendFailedCallback(ApiException exception, int statusCode) { callback.onFailure(exception, statusCode, Collections.emptyMap()); } From dfeb204fa44d4c38eeb2a1394e47f0b21b7e384a Mon Sep 17 00:00:00 2001 From: Russell Gold Date: Mon, 23 Apr 2018 17:56:40 -0400 Subject: [PATCH 057/186] Create support for async request unit tests --- .../operator/calls/AsyncRequestStep.java | 18 +- .../operator/calls/CallFactory.java | 2 +- .../operator/calls/CallResponse.java | 6 +- .../operator/calls/CallWrapper.java | 2 +- ...ncelableCall.java => CancellableCall.java} | 2 +- .../operator/helpers/CallBuilder.java | 34 +-- .../operator/steps/DeleteIngressListStep.java | 3 +- .../operator/calls/AsyncRequestStepTest.java | 145 ++------- .../helpers/AsyncCallTestSupport.java | 279 ++++++++++++++++++ .../steps/DeleteIngressListStepTest.java | 85 ++++++ .../operator/work/FiberTestSupport.java | 221 ++++++++++++++ .../operator/work/TerminalStep.java | 15 + 12 files changed, 646 insertions(+), 166 deletions(-) rename operator/src/main/java/oracle/kubernetes/operator/calls/{CancelableCall.java => CancellableCall.java} (91%) create mode 100644 operator/src/test/java/oracle/kubernetes/operator/helpers/AsyncCallTestSupport.java create mode 100644 operator/src/test/java/oracle/kubernetes/operator/steps/DeleteIngressListStepTest.java create mode 100644 operator/src/test/java/oracle/kubernetes/operator/work/FiberTestSupport.java create mode 100644 operator/src/test/java/oracle/kubernetes/operator/work/TerminalStep.java diff --git a/operator/src/main/java/oracle/kubernetes/operator/calls/AsyncRequestStep.java b/operator/src/main/java/oracle/kubernetes/operator/calls/AsyncRequestStep.java index 68f8934741d..e5d2272ae77 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/calls/AsyncRequestStep.java +++ b/operator/src/main/java/oracle/kubernetes/operator/calls/AsyncRequestStep.java @@ -3,6 +3,14 @@ package oracle.kubernetes.operator.calls; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + import io.kubernetes.client.ApiCallback; import io.kubernetes.client.ApiClient; import io.kubernetes.client.ApiException; @@ -18,14 +26,6 @@ import oracle.kubernetes.operator.work.Packet; import oracle.kubernetes.operator.work.Step; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; - /** * A Step driven by an asynchronous call to the Kubernetes API, which results in a series of callbacks until canceled. */ @@ -120,7 +120,7 @@ public void onSuccess(T result, int statusCode, Map> respon }; try { - CancelableCall c = factory.generate(requestParams, client, _continue, callback); + CancellableCall c = factory.generate(requestParams, client, _continue, callback); // timeout handling fiber.owner.getExecutor().schedule(() -> { diff --git a/operator/src/main/java/oracle/kubernetes/operator/calls/CallFactory.java b/operator/src/main/java/oracle/kubernetes/operator/calls/CallFactory.java index b7555e8f03f..cec2040d969 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/calls/CallFactory.java +++ b/operator/src/main/java/oracle/kubernetes/operator/calls/CallFactory.java @@ -9,5 +9,5 @@ @FunctionalInterface public interface CallFactory { - CancelableCall generate(RequestParams requestParams, ApiClient client, String cont, ApiCallback callback) throws ApiException; + CancellableCall generate(RequestParams requestParams, ApiClient client, String cont, ApiCallback callback) throws ApiException; } diff --git a/operator/src/main/java/oracle/kubernetes/operator/calls/CallResponse.java b/operator/src/main/java/oracle/kubernetes/operator/calls/CallResponse.java index 33ac6850e28..9d1fa3f488f 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/calls/CallResponse.java +++ b/operator/src/main/java/oracle/kubernetes/operator/calls/CallResponse.java @@ -3,18 +3,18 @@ package oracle.kubernetes.operator.calls; -import io.kubernetes.client.ApiException; - import java.util.List; import java.util.Map; +import io.kubernetes.client.ApiException; + public final class CallResponse { public final T result; public final ApiException e; public final int statusCode; public final Map> responseHeaders; - CallResponse(T result, ApiException e, int statusCode, Map> responseHeaders) { + public CallResponse(T result, ApiException e, int statusCode, Map> responseHeaders) { this.result = result; this.e = e; this.statusCode = statusCode; diff --git a/operator/src/main/java/oracle/kubernetes/operator/calls/CallWrapper.java b/operator/src/main/java/oracle/kubernetes/operator/calls/CallWrapper.java index 235a7c99563..03273f28688 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/calls/CallWrapper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/calls/CallWrapper.java @@ -8,7 +8,7 @@ /** * A wrapper for an OKHttp call to isolate its own callers. */ -public class CallWrapper implements CancelableCall { +public class CallWrapper implements CancellableCall { private Call underlyingCall; diff --git a/operator/src/main/java/oracle/kubernetes/operator/calls/CancelableCall.java b/operator/src/main/java/oracle/kubernetes/operator/calls/CancellableCall.java similarity index 91% rename from operator/src/main/java/oracle/kubernetes/operator/calls/CancelableCall.java rename to operator/src/main/java/oracle/kubernetes/operator/calls/CancellableCall.java index 71b8a30d098..8dbeaae87be 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/calls/CancelableCall.java +++ b/operator/src/main/java/oracle/kubernetes/operator/calls/CancellableCall.java @@ -6,7 +6,7 @@ /** * An interface for an asynchronous API invocation that can be canceled */ -public interface CancelableCall { +public interface CancellableCall { /** * Cancels the active call. diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java index 4065299c0b4..df17b56b51d 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java @@ -3,6 +3,10 @@ package oracle.kubernetes.operator.helpers; +import java.util.function.Consumer; + +import com.squareup.okhttp.Call; + import io.kubernetes.client.ApiCallback; import io.kubernetes.client.ApiClient; import io.kubernetes.client.ApiException; @@ -13,42 +17,18 @@ import io.kubernetes.client.apis.CoreV1Api; import io.kubernetes.client.apis.ExtensionsV1beta1Api; import io.kubernetes.client.apis.VersionApi; -import io.kubernetes.client.models.V1ConfigMap; -import io.kubernetes.client.models.V1DeleteOptions; -import io.kubernetes.client.models.V1EventList; -import io.kubernetes.client.models.V1Job; -import io.kubernetes.client.models.V1Namespace; -import io.kubernetes.client.models.V1PersistentVolumeClaimList; -import io.kubernetes.client.models.V1PersistentVolumeList; -import io.kubernetes.client.models.V1Pod; -import io.kubernetes.client.models.V1PodList; -import io.kubernetes.client.models.V1Secret; -import io.kubernetes.client.models.V1SelfSubjectAccessReview; -import io.kubernetes.client.models.V1SelfSubjectRulesReview; -import io.kubernetes.client.models.V1Service; -import io.kubernetes.client.models.V1ServiceList; -import io.kubernetes.client.models.V1Status; -import io.kubernetes.client.models.V1SubjectAccessReview; -import io.kubernetes.client.models.V1TokenReview; -import io.kubernetes.client.models.V1beta1CustomResourceDefinition; -import io.kubernetes.client.models.V1beta1Ingress; -import io.kubernetes.client.models.V1beta1IngressList; -import io.kubernetes.client.models.VersionInfo; +import io.kubernetes.client.models.*; import oracle.kubernetes.operator.TuningParameters.CallBuilderTuning; import oracle.kubernetes.operator.calls.AsyncRequestStep; import oracle.kubernetes.operator.calls.CallFactory; import oracle.kubernetes.operator.calls.CallWrapper; -import oracle.kubernetes.operator.calls.CancelableCall; +import oracle.kubernetes.operator.calls.CancellableCall; import oracle.kubernetes.operator.calls.RequestParams; import oracle.kubernetes.operator.work.Step; import oracle.kubernetes.weblogic.domain.v1.Domain; import oracle.kubernetes.weblogic.domain.v1.DomainList; import oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi; -import com.squareup.okhttp.Call; - -import java.util.function.Consumer; - /** * Simplifies synchronous and asynchronous call patterns to the Kubernetes API Server. * @@ -207,7 +187,7 @@ private com.squareup.okhttp.Call listDomainAsync(ApiClient client, String namesp private final CallFactory LIST_DOMAIN = (requestParams, usage, cont, callback) -> wrap(listDomainAsync(usage, requestParams.namespace, cont, callback)); - private CancelableCall wrap(Call call) { + private CancellableCall wrap(Call call) { return new CallWrapper(call); } diff --git a/operator/src/main/java/oracle/kubernetes/operator/steps/DeleteIngressListStep.java b/operator/src/main/java/oracle/kubernetes/operator/steps/DeleteIngressListStep.java index 9a5cc8e713d..8fbde97c9bb 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/steps/DeleteIngressListStep.java +++ b/operator/src/main/java/oracle/kubernetes/operator/steps/DeleteIngressListStep.java @@ -19,7 +19,6 @@ import oracle.kubernetes.operator.logging.LoggingFacade; import oracle.kubernetes.operator.logging.LoggingFactory; import oracle.kubernetes.operator.logging.MessageKeys; -import oracle.kubernetes.operator.work.ContainerResolver; import oracle.kubernetes.operator.work.NextAction; import oracle.kubernetes.operator.work.Packet; import oracle.kubernetes.operator.work.Step; @@ -36,7 +35,7 @@ public DeleteIngressListStep(Collection c, Step next) { @Override public NextAction apply(Packet packet) { - CallBuilderFactory factory = ContainerResolver.getInstance().getContainer().getSPI(CallBuilderFactory.class); + CallBuilderFactory factory = new CallBuilderFactory(); if (it.hasNext()) { V1beta1Ingress v1beta1Ingress = it.next(); diff --git a/operator/src/test/java/oracle/kubernetes/operator/calls/AsyncRequestStepTest.java b/operator/src/test/java/oracle/kubernetes/operator/calls/AsyncRequestStepTest.java index 2fd3f8c27bc..e02cc5d7387 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/calls/AsyncRequestStepTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/calls/AsyncRequestStepTest.java @@ -3,42 +3,29 @@ package oracle.kubernetes.operator.calls; +import java.net.HttpURLConnection; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import com.meterware.simplestub.Memento; + import io.kubernetes.client.ApiCallback; import io.kubernetes.client.ApiClient; import io.kubernetes.client.ApiException; import oracle.kubernetes.TestUtils; import oracle.kubernetes.operator.helpers.ClientPool; import oracle.kubernetes.operator.helpers.ResponseStep; -import oracle.kubernetes.operator.work.Engine; -import oracle.kubernetes.operator.work.Fiber; +import oracle.kubernetes.operator.work.FiberTestSupport; import oracle.kubernetes.operator.work.NextAction; import oracle.kubernetes.operator.work.Packet; -import com.meterware.simplestub.Memento; - -import java.net.HttpURLConnection; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Queue; -import java.util.SortedSet; -import java.util.TreeSet; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - import org.junit.After; import org.junit.Before; import org.junit.Test; -import static com.meterware.simplestub.Stub.createStrictStub; -import static com.meterware.simplestub.Stub.createStub; import static oracle.kubernetes.operator.calls.AsyncRequestStep.RESPONSE_COMPONENT_NAME; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; @@ -51,13 +38,9 @@ public class AsyncRequestStepTest { private static final int TIMEOUT_SECONDS = 10; private static final int MAX_RETRY_COUNT = 2; - private Packet packet = new Packet(); - private Schedule schedule = createStrictStub(Schedule.class); - private Engine engine = new Engine(schedule); - private Fiber fiber = engine.createFiber(); + private FiberTestSupport testSupport = new FiberTestSupport(); private RequestParams requestParams = new RequestParams("testcall", "junit", "testName", "body"); private CallFactoryStub callFactory = new CallFactoryStub(); - private CompletionCallbackStub completionCallback = new CompletionCallbackStub(); private TestStep nextStep = new TestStep(); private ClientPool helper = ClientPool.getInstance(); private List mementos = new ArrayList<>(); @@ -69,7 +52,7 @@ public class AsyncRequestStepTest { public void setUp() throws Exception { mementos.add(TestUtils.silenceOperatorLogger()); - fiber.start(asyncRequestStep, packet, completionCallback); + testSupport.runStep(asyncRequestStep); } @After @@ -83,15 +66,15 @@ public void afterFiberStarted_requestSent() throws Exception { } @Test - public void afterFiber_timeoutStepScheduled() throws Exception { - assertTrue(schedule.containsStepAt(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + public void afterFiberStarted_timeoutStepScheduled() throws Exception { + assertTrue(testSupport.hasItemScheduledAt(TIMEOUT_SECONDS, TimeUnit.SECONDS)); } @Test public void afterTimeout_newRequestSent() throws Exception { callFactory.clearRequest(); - schedule.setTime(TIMEOUT_SECONDS, TimeUnit.SECONDS); + testSupport.setTime(TIMEOUT_SECONDS, TimeUnit.SECONDS); assertTrue(callFactory.invokedWith(requestParams)); } @@ -105,20 +88,20 @@ public void afterSuccessfulCallback_nextStepAppliedWithValue() throws Exception @Test public void afterSuccessfulCallback_packetDoesNotContainsResponse() throws Exception { - schedule.execute(() -> callFactory.sendSuccessfulCallback(17)); + testSupport.schedule(() -> callFactory.sendSuccessfulCallback(17)); - assertThat(packet.getComponents(), not(hasKey(RESPONSE_COMPONENT_NAME))); + assertThat(testSupport.getPacketComponents(), not(hasKey(RESPONSE_COMPONENT_NAME))); } @Test public void afterFailedCallback_packetContainsRetryStrategy() throws Exception { sendFailedCallback(HttpURLConnection.HTTP_UNAVAILABLE); - assertThat(packet.getComponents().get(RESPONSE_COMPONENT_NAME).getSPI(RetryStrategy.class), notNullValue()); + assertThat(testSupport.getPacketComponents().get(RESPONSE_COMPONENT_NAME).getSPI(RetryStrategy.class), notNullValue()); } private void sendFailedCallback(int statusCode) { - schedule.execute(() -> callFactory.sendFailedCallback(new ApiException("test failure"), statusCode)); + testSupport.schedule(() -> callFactory.sendFailedCallback(new ApiException("test failure"), statusCode)); } @Test @@ -126,7 +109,7 @@ public void afterFailedCallback_retrySentAfterDelay() throws Exception { sendFailedCallback(HttpURLConnection.HTTP_UNAVAILABLE); callFactory.clearRequest(); - schedule.setTime(TIMEOUT_SECONDS-1, TimeUnit.SECONDS); + testSupport.setTime(TIMEOUT_SECONDS-1, TimeUnit.SECONDS); assertTrue(callFactory.invokedWith(requestParams)); } @@ -176,15 +159,15 @@ void sendFailedCallback(ApiException exception, int statusCode) { } @Override - public CancelableCall generate(RequestParams requestParams, ApiClient client, String cont, ApiCallback callback) throws ApiException { + public CancellableCall generate(RequestParams requestParams, ApiClient client, String cont, ApiCallback callback) throws ApiException { this.requestParams = requestParams; this.callback = callback; - return new CancelableCallStub(); + return new CancellableCallStub(); } } - static class CancelableCallStub implements CancelableCall { + static class CancellableCallStub implements CancellableCall { private boolean canceled; @Override @@ -193,86 +176,4 @@ public void cancel() { } } - static class CompletionCallbackStub implements Fiber.CompletionCallback { - private Packet packet; - private Throwable throwable; - - @Override - public void onCompletion(Packet packet) { - this.packet = packet; - } - - @Override - public void onThrowable(Packet packet, Throwable throwable) { - this.packet = packet; - this.throwable = throwable; - } - } - - static class ScheduledItem implements Comparable { - private long atTime; - private Runnable runnable; - - ScheduledItem(long atTime, Runnable runnable) { - this.atTime = atTime; - this.runnable = runnable; - } - - @Override - public int compareTo(@Nonnull ScheduledItem o) { - return Long.compare(atTime, o.atTime); - } - } - - static abstract class Schedule implements ScheduledExecutorService { - /** current time in milliseconds. */ - private long currentTime = 0; - - private SortedSet scheduledItems = new TreeSet<>(); - private Queue queue = new ArrayDeque<>(); - private Runnable current; - - @Override - @Nonnull public ScheduledFuture schedule(@Nonnull Runnable command, long delay, @Nonnull TimeUnit unit) { - scheduledItems.add(new ScheduledItem(unit.toMillis(delay), command)); - return createStub(ScheduledFuture.class); - } - - @Override - public void execute(@Nullable Runnable command) { - queue.add(command); - if (current == null) - runNextRunnable(); - } - - private void runNextRunnable() { - while (queue.peek() != null) { - current = queue.poll(); - current.run(); - current = null; - } - - } - - void setTime(long time, TimeUnit unit) { - long newTime = unit.toMillis(time); - if (newTime < currentTime) - throw new IllegalStateException("Attempt to move clock backwards from " + currentTime + " to " + newTime); - - for (Iterator it = scheduledItems.iterator(); it.hasNext();) { - ScheduledItem item = it.next(); - if (item.atTime > newTime) break; - it.remove(); - execute(item.runnable); - } - - currentTime = newTime; - } - - boolean containsStepAt(int timeoutSeconds, TimeUnit unit) { - for (ScheduledItem scheduledItem : scheduledItems) - if (scheduledItem.atTime == unit.toMillis(timeoutSeconds)) return true; - return false; - } - } } \ No newline at end of file diff --git a/operator/src/test/java/oracle/kubernetes/operator/helpers/AsyncCallTestSupport.java b/operator/src/test/java/oracle/kubernetes/operator/helpers/AsyncCallTestSupport.java new file mode 100644 index 00000000000..b40d59effd3 --- /dev/null +++ b/operator/src/test/java/oracle/kubernetes/operator/helpers/AsyncCallTestSupport.java @@ -0,0 +1,279 @@ +// Copyright 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.helpers; + +import java.net.HttpURLConnection; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import com.meterware.simplestub.Memento; +import com.meterware.simplestub.StaticStubSupport; + +import io.kubernetes.client.ApiException; +import oracle.kubernetes.operator.builders.CallParams; +import oracle.kubernetes.operator.calls.CallFactory; +import oracle.kubernetes.operator.calls.CallResponse; +import oracle.kubernetes.operator.calls.RequestParams; +import oracle.kubernetes.operator.work.Component; +import oracle.kubernetes.operator.work.FiberTestSupport; +import oracle.kubernetes.operator.work.NextAction; +import oracle.kubernetes.operator.work.Packet; +import oracle.kubernetes.operator.work.Step; + +import javax.annotation.Nonnull; + +import static oracle.kubernetes.operator.calls.AsyncRequestStep.RESPONSE_COMPONENT_NAME; + +/** + * Support for writing unit tests that use CallBuilder to send requests that expect asynchronous responses. + * + * The setUp should invoke #installRequestStepFactory to modify CallBuilder for unit testing, while capturing + * the memento to clean up during tearDown. + * + * The test must define the simulated responses to the calls it will test, by invoking #createCannedResponse, + * any qualifiers, and the result of the call. For example: + * + * testSupport.createCannedResponse("deleteIngress") + * .withNamespace(namespace).withName(name) + * .failingWith(HttpURLConnection.HTTP_CONFLICT); + * + * will report a conflict failure on an attempt to delete an Ingress with the specified name and namespace. + * + * testSupport.createCannedResponse("listPod") + * .withNamespace(namespace) + * .returning(new V1PodList().items(Arrays.asList(pod1, pod2, pod3); + * + * will return a list of pods after a query with the specified namespace. + */ +@SuppressWarnings("unused") +public class AsyncCallTestSupport extends FiberTestSupport { + + /** + * Installs a factory into CallBuilder to use canned responses. + * @return a memento which can be used to restore the production factory + */ + public Memento installRequestStepFactory() throws NoSuchFieldException { + return StaticStubSupport.install(CallBuilder.class, "STEP_FACTORY", new RequestStepFactory()); + } + + private class RequestStepFactory implements AsyncRequestStepFactory { + + @Override + public Step createRequestAsync(ResponseStep next, RequestParams requestParams, CallFactory factory, ClientPool helper, int timeoutSeconds, int maxRetryCount, String fieldSelector, String labelSelector, String resourceVersion) { + return new CannedResponseStep<>(next, getMatchingResponse(requestParams, null)); + } + + } + + private Map cannedResponses = new HashMap<>(); + + /** + * Primes CallBuilder to expect a request for the specified method. + * @param forMethod the name of the method + * @return a canned response which may be qualified by parameters and defines how CallBuilder should react. + */ + public CannedResponse createCannedResponse(String forMethod) { + CannedResponse cannedResponse = new CannedResponse(forMethod); + cannedResponses.put(cannedResponse, false); + return cannedResponse; + } + + @SuppressWarnings({"unchecked", "SameParameterValue"}) + private CannedResponse getMatchingResponse(RequestParams requestParams, CallParams callParams) { + for (CannedResponse cannedResponse : cannedResponses.keySet()) + if (cannedResponse.matches(requestParams, callParams)) return afterMarking(cannedResponse); + + throw new AssertionError("Unexpected request for " + toString(requestParams, callParams)); + } + + private CannedResponse afterMarking(CannedResponse cannedResponse) { + cannedResponses.put(cannedResponse, true); + return cannedResponse; + } + + private String toString(RequestParams requestParams, CallParams callParams) { + ErrorFormatter formatter = new ErrorFormatter(requestParams.call); + formatter.addDescriptor("namespace", requestParams.namespace); + formatter.addDescriptor("name", requestParams.name); + return formatter.toString(); + } + + /** + * Throws an exception if any of the canned responses were not used. + */ + public void verify() { + List unusedResponses = new ArrayList<>(); + for (CannedResponse cannedResponse : cannedResponses.keySet()) + if (!cannedResponses.get(cannedResponse)) unusedResponses.add(cannedResponse); + + if (unusedResponses.isEmpty()) return; + + StringBuilder sb = new StringBuilder("The following expected calls were not made:").append('\n'); + for (CannedResponse cannedResponse : unusedResponses) + sb.append(" ").append(cannedResponse).append('\n'); + throw new AssertionError(sb.toString()); + } + + private static class ErrorFormatter { + private String call; + private List descriptors = new ArrayList<>(); + + ErrorFormatter(String call) { + this.call = call; + } + + void addDescriptor(String type, String value) { + if (isDefined(value)) + descriptors.add(String.format("%s '%s'", type, value)); + } + + private boolean isDefined(String value) { + return value != null && value.trim().length() > 0; + } + + public String toString() { + StringBuilder sb = new StringBuilder(call); + if (!descriptors.isEmpty()) { + sb.append(" with ").append(descriptors.get(0)); + for (int i = 1; i < descriptors.size()-1; i++) sb.append(", ").append(descriptors.get(i)); + if (descriptors.size() > 1) sb.append(" and ").append(descriptors.get(descriptors.size()-1)); + } + return sb.toString(); + } + } + + /** + * A canned response which may be qualified by parameters and defines how CallBuilder should react. + * @param the type of value to be returned in the step, if it succeeds + */ + public static class CannedResponse { + private String methodName; + private Map requestParamExpectations = new HashMap<>(); + private T result; + private int status; + + private CannedResponse(String methodName) { + this.methodName = methodName; + } + + private CallResponse getCallResponse() { + if (result == null) + return new CallResponse<>(null, new ApiException(), status, Collections.emptyMap()); + else + return new CallResponse<>(result, null, HttpURLConnection.HTTP_OK, Collections.emptyMap()); + } + + private boolean matches(@Nonnull RequestParams requestParams, CallParams callParams) { + return matches(requestParams); + } + + private boolean matches(RequestParams requestParams) { + return Objects.equals(requestParams.call, methodName) + && Objects.equals(requestParams.name, requestParamExpectations.get("name")) + && Objects.equals(requestParams.namespace, requestParamExpectations.get("namespace")); + } + + /** + * Qualifies the canned response to be used only if the namespace matches the value specified + * @param namespace the expected namespace + * @return the updated response + */ + public CannedResponse withNamespace(String namespace) { + requestParamExpectations.put("namespace", namespace); + return this; + } + + /** + * Qualifies the canned response to be used only if the name matches the value specified + * @param name the expected name + * @return the updated response + */ + public CannedResponse withName(String name) { + requestParamExpectations.put("name", name); + return this; + } + + /** + * Specifies the result to be returned by the canned response. + * @param result the response to return + */ + public void returning(T result) { + this.result = result; + } + + /** + * Indicates that the canned response should fail and specifies the HTML status to report. + * @param status the failure status + */ + public void failingWithStatus(int status) { + this.status = status; + } + + @Override + public String toString() { + ErrorFormatter formatter = new ErrorFormatter(methodName); + for (Map.Entry entry : requestParamExpectations.entrySet()) + formatter.addDescriptor(entry.getKey(), entry.getValue()); + + return formatter.toString(); + } + } + + private static class CannedResponseStep extends Step { + private CannedResponse cannedResponse; + + CannedResponseStep(Step next, CannedResponse cannedResponse) { + super(next); + this.cannedResponse = cannedResponse; + } + + @Override + public NextAction apply(Packet packet) { + CannedResponse cannedResponse = this.cannedResponse; + CallResponse callResponse = cannedResponse.getCallResponse(); + packet.getComponents().put(RESPONSE_COMPONENT_NAME, Component.createFor(callResponse)); + + return doNext(packet); + } + + } + + private static class SuccessStep extends Step { + private final T result; + + SuccessStep(T result, Step next) { + super(next); + this.result = result; + } + + @Override + public NextAction apply(Packet packet) { + packet.getComponents().put(RESPONSE_COMPONENT_NAME, + Component.createFor(new CallResponse<>(result, null, HttpURLConnection.HTTP_OK, Collections.emptyMap()))); + + return doNext(packet); + } + } + + private static class FailureStep extends Step { + private final int status; + + FailureStep(int status, Step next) { + super(next); + this.status = status; + } + + @SuppressWarnings("unchecked") + @Override + public NextAction apply(Packet packet) { + packet.getComponents().put(RESPONSE_COMPONENT_NAME, Component.createFor(new CallResponse(null, new ApiException(), status, Collections.emptyMap()))); + + return doNext(packet); + } + } +} diff --git a/operator/src/test/java/oracle/kubernetes/operator/steps/DeleteIngressListStepTest.java b/operator/src/test/java/oracle/kubernetes/operator/steps/DeleteIngressListStepTest.java new file mode 100644 index 00000000000..46e7de2845f --- /dev/null +++ b/operator/src/test/java/oracle/kubernetes/operator/steps/DeleteIngressListStepTest.java @@ -0,0 +1,85 @@ +package oracle.kubernetes.operator.steps; + +import java.net.HttpURLConnection; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import com.meterware.simplestub.Memento; + +import io.kubernetes.client.ApiException; +import io.kubernetes.client.models.V1ObjectMeta; +import io.kubernetes.client.models.V1Status; +import io.kubernetes.client.models.V1beta1Ingress; +import oracle.kubernetes.TestUtils; +import oracle.kubernetes.operator.helpers.AsyncCallTestSupport; +import oracle.kubernetes.operator.work.TerminalStep; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class DeleteIngressListStepTest { + + private Collection ingresses = new ArrayList<>(); + private TerminalStep terminalStep = new TerminalStep(); + + private AsyncCallTestSupport testSupport = new AsyncCallTestSupport(); + private List mementos = new ArrayList<>(); + + @Before + public void setUp() throws Exception { + mementos.add(testSupport.installRequestStepFactory()); + mementos.add(TestUtils.silenceOperatorLogger()); + } + + @After + public void tearDown() throws Exception { + for (Memento memento : mementos) memento.revert(); + testSupport.throwOnCompletionFailure(); + } + + @Test + public void whenCollectionEmpty_makeNoCalls() throws Exception { + runStep(); + } + + private void runStep() { + testSupport.runStep(new DeleteIngressListStep(ingresses, terminalStep)); + } + + @Test + public void whenCollectionContainsItems_invokeDeleteCalls() throws Exception { + defineResponse("namespace1", "name1").returning(new V1Status()); + defineResponse("namespace2", "name2").returning(new V1Status()); + ingresses.add(new V1beta1Ingress().metadata(new V1ObjectMeta().namespace("namespace1").name("name1"))); + ingresses.add(new V1beta1Ingress().metadata(new V1ObjectMeta().namespace("namespace2").name("name2"))); + + runStep(); + + testSupport.verify(); + } + + @SuppressWarnings("unchecked") + private AsyncCallTestSupport.CannedResponse defineResponse(String namespace, String name) { + return testSupport.createCannedResponse("deleteIngress").withNamespace(namespace).withName(name); + } + + @Test + public void onFailureResponse_reportError() throws Exception { + defineResponse("namespace1", "name1").failingWithStatus(HttpURLConnection.HTTP_FORBIDDEN); + ingresses.add(new V1beta1Ingress().metadata(new V1ObjectMeta().namespace("namespace1").name("name1"))); + + runStep(); + + testSupport.verifyCompletionThrowable(ApiException.class); + } + + @Test + public void onNotFoundResponse_dontReportError() throws Exception { + defineResponse("namespace1", "name1").failingWithStatus(HttpURLConnection.HTTP_NOT_FOUND); + ingresses.add(new V1beta1Ingress().metadata(new V1ObjectMeta().namespace("namespace1").name("name1"))); + + runStep(); + } +} \ No newline at end of file diff --git a/operator/src/test/java/oracle/kubernetes/operator/work/FiberTestSupport.java b/operator/src/test/java/oracle/kubernetes/operator/work/FiberTestSupport.java new file mode 100644 index 00000000000..754f0affb34 --- /dev/null +++ b/operator/src/test/java/oracle/kubernetes/operator/work/FiberTestSupport.java @@ -0,0 +1,221 @@ +package oracle.kubernetes.operator.work; + +import java.util.ArrayDeque; +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.Queue; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +import io.kubernetes.client.ApiException; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import static com.meterware.simplestub.Stub.createStrictStub; +import static com.meterware.simplestub.Stub.createStub; + +/** + * Support for writing unit tests that use a fiber to run steps. Such tests can call #runStep to initiate + * fiber execution, which will happen in simulated time. That time starts at zero when an instance is created, + * and can be increased by a call to setTime. This is useful to run steps which are scheduled for the future, + * without using delays. As all steps are run in a single thread, there is no need to add semaphores to coordinate + * them. + * + * The components in the packet used by the embedded fiber may be access via #getPacketComponents. + */ +public class FiberTestSupport { + private CompletionCallbackStub completionCallback = new CompletionCallbackStub(); + private ScheduledExecutorStub schedule = ScheduledExecutorStub.create(); + + private Engine engine = new Engine(schedule); + private Fiber fiber = engine.createFiber(); + private Packet packet = new Packet(); + + /** + * Schedules a runnable to run immediately. In practice, it will run as soon as all previously queued runnables + * have complete. + * @param runnable a runnable to be executed by the scheduler. + */ + public void schedule(Runnable runnable) { + schedule.execute(runnable); + } + + /** + * Returns true if an item is scheduled to run at the specified time. + * @param time the time, in units + * @param unit the unit associated with the time + */ + public boolean hasItemScheduledAt(int time, TimeUnit unit) { + return schedule.containsItemAt(time, unit); + } + + /** + * Sets the simulated time, thus triggering the execution of any runnable items associated with earlier times. + * @param time the time, in units + * @param unit the unit associated with the time + */ + public void setTime(int time, TimeUnit unit) { + schedule.setTime(time, unit); + } + + /** + * Returns an unmodificable map of the components in the test packet. + */ + public Map getPacketComponents() { + return Collections.unmodifiableMap(packet.getComponents()); + } + + /** + * Starts a unit-test fiber with the specified step + * @param step the first step to run + */ + public void runStep(Step step) { + fiber.start(step, packet, completionCallback); + } + + /** + * Verifies that the completion callback's 'onThrowable' method was invoked with a throwable of the specified class. + * Clears the throwable so that #throwOnFailure will not throw the expected exception. + * @param throwableClass the class of the excepted throwable + */ + public void verifyCompletionThrowable(Class throwableClass) { + completionCallback.verifyThrowable(throwableClass); + } + + /** + * If the completion callback's 'onThrowable' method was invoked, throws the specified throwable. + * Note that a call to #verifyCompletionThrowable will consume the throwable, so this method will not throw it. + * @throws Exception the exception reported as a failure + */ + public void throwOnCompletionFailure() throws Exception { + completionCallback.throwOnFailure(); + } + + abstract static class ScheduledExecutorStub implements ScheduledExecutorService { + /** current time in milliseconds. */ + private long currentTime = 0; + + private SortedSet scheduledItems = new TreeSet<>(); + private Queue queue = new ArrayDeque<>(); + private Runnable current; + + public static ScheduledExecutorStub create() { + return createStrictStub(ScheduledExecutorStub.class); + } + + @Override + @Nonnull + public ScheduledFuture schedule(@Nonnull Runnable command, long delay, @Nonnull TimeUnit unit) { + scheduledItems.add(new ScheduledItem(unit.toMillis(delay), command)); + return createStub(ScheduledFuture.class); + } + + @Override + public void execute(@Nullable Runnable command) { + queue.add(command); + if (current == null) + runNextRunnable(); + } + + private void runNextRunnable() { + while (queue.peek() != null) { + current = queue.poll(); + current.run(); + current = null; + } + } + + /** + * Sets the simulated time, thus triggering the execution of any runnable iterms associated with earlier times. + * @param time the time, in units + * @param unit the unit associated with the time + */ + void setTime(long time, TimeUnit unit) { + long newTime = unit.toMillis(time); + if (newTime < currentTime) + throw new IllegalStateException("Attempt to move clock backwards from " + currentTime + " to " + newTime); + + for (Iterator it = scheduledItems.iterator(); it.hasNext();) { + ScheduledItem item = it.next(); + if (item.atTime > newTime) break; + it.remove(); + execute(item.runnable); + } + + currentTime = newTime; + } + + /** + * Returns true if a runnable item has been scheduled for the specified time. + * @param time the time, in units + * @param unit the unit associated with the time + * @return true if such an item exists + */ + boolean containsItemAt(int time, TimeUnit unit) { + for (ScheduledItem scheduledItem : scheduledItems) + if (scheduledItem.atTime == unit.toMillis(time)) return true; + return false; + } + + private static class ScheduledItem implements Comparable { + private long atTime; + private Runnable runnable; + + ScheduledItem(long atTime, Runnable runnable) { + this.atTime = atTime; + this.runnable = runnable; + } + + @Override + public int compareTo(@Nonnull ScheduledItem o) { + return Long.compare(atTime, o.atTime); + } + } + } + + static class CompletionCallbackStub implements Fiber.CompletionCallback { + private Packet packet; + private Throwable throwable; + + @Override + public void onCompletion(Packet packet) { + this.packet = packet; + } + + @Override + public void onThrowable(Packet packet, Throwable throwable) { + this.packet = packet; + this.throwable = throwable; + } + + /** + * Verifies that 'onThrowable' was invoked with a throwable of the specified class. Clears + * the throwable so that #throwOnFailure will not throw the expected exception. + * @param throwableClass the class of the excepted throwable + */ + void verifyThrowable(Class throwableClass) { + Throwable actual = throwable; + throwable = null; + + if (actual == null) throw new AssertionError("Expected exception: " + throwableClass.getName()); + if (!throwableClass.isInstance(actual)) + throw new AssertionError("Expected exception: " + throwableClass.getName() + " but was " + actual); + } + + /** + * If 'onThrowable' was invoked, throws the specified throwable. Note that a call to #verifyThrowable + * will consume the throwable, so this method will not throw it. + * @throws Exception the exception reported as a failure + */ + void throwOnFailure() throws Exception { + if (throwable == null) return; + if (throwable instanceof Error) throw (Error) throwable; + throw (Exception) throwable; + } + } +} diff --git a/operator/src/test/java/oracle/kubernetes/operator/work/TerminalStep.java b/operator/src/test/java/oracle/kubernetes/operator/work/TerminalStep.java new file mode 100644 index 00000000000..83ce8e64d99 --- /dev/null +++ b/operator/src/test/java/oracle/kubernetes/operator/work/TerminalStep.java @@ -0,0 +1,15 @@ +package oracle.kubernetes.operator.work; + +/** + * A do-nothing step that can be used as a base for testing. It has no next step. + */ +public class TerminalStep extends Step { + public TerminalStep() { + super(null); + } + + @Override + public NextAction apply(Packet packet) { + return doNext(packet); + } +} From 6575973b29d09813e199b05982f2d11940567ea5 Mon Sep 17 00:00:00 2001 From: Russell Gold Date: Mon, 23 Apr 2018 18:12:30 -0400 Subject: [PATCH 058/186] Improve test readability --- .../helpers/AsyncCallTestSupport.java | 2 +- .../steps/DeleteIngressListStepTest.java | 21 +++++++++---------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/operator/src/test/java/oracle/kubernetes/operator/helpers/AsyncCallTestSupport.java b/operator/src/test/java/oracle/kubernetes/operator/helpers/AsyncCallTestSupport.java index b40d59effd3..72ee089879a 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/helpers/AsyncCallTestSupport.java +++ b/operator/src/test/java/oracle/kubernetes/operator/helpers/AsyncCallTestSupport.java @@ -106,7 +106,7 @@ private String toString(RequestParams requestParams, CallParams callParams) { /** * Throws an exception if any of the canned responses were not used. */ - public void verify() { + public void verifyAllDefinedResponsesInvoked() { List unusedResponses = new ArrayList<>(); for (CannedResponse cannedResponse : cannedResponses.keySet()) if (!cannedResponses.get(cannedResponse)) unusedResponses.add(cannedResponse); diff --git a/operator/src/test/java/oracle/kubernetes/operator/steps/DeleteIngressListStepTest.java b/operator/src/test/java/oracle/kubernetes/operator/steps/DeleteIngressListStepTest.java index 46e7de2845f..40a55e7f653 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/steps/DeleteIngressListStepTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/steps/DeleteIngressListStepTest.java @@ -2,6 +2,7 @@ import java.net.HttpURLConnection; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.List; @@ -41,23 +42,23 @@ public void tearDown() throws Exception { @Test public void whenCollectionEmpty_makeNoCalls() throws Exception { - runStep(); + runDeleteStep(); } - private void runStep() { - testSupport.runStep(new DeleteIngressListStep(ingresses, terminalStep)); + private void runDeleteStep(V1beta1Ingress... ingresses) { + this.ingresses.addAll(Arrays.asList(ingresses)); + testSupport.runStep(new DeleteIngressListStep(this.ingresses, terminalStep)); } @Test public void whenCollectionContainsItems_invokeDeleteCalls() throws Exception { defineResponse("namespace1", "name1").returning(new V1Status()); defineResponse("namespace2", "name2").returning(new V1Status()); - ingresses.add(new V1beta1Ingress().metadata(new V1ObjectMeta().namespace("namespace1").name("name1"))); - ingresses.add(new V1beta1Ingress().metadata(new V1ObjectMeta().namespace("namespace2").name("name2"))); - runStep(); + runDeleteStep(new V1beta1Ingress().metadata(new V1ObjectMeta().namespace("namespace1").name("name1")), + new V1beta1Ingress().metadata(new V1ObjectMeta().namespace("namespace2").name("name2"))); - testSupport.verify(); + testSupport.verifyAllDefinedResponsesInvoked(); } @SuppressWarnings("unchecked") @@ -68,9 +69,8 @@ private AsyncCallTestSupport.CannedResponse defineResponse(String namespa @Test public void onFailureResponse_reportError() throws Exception { defineResponse("namespace1", "name1").failingWithStatus(HttpURLConnection.HTTP_FORBIDDEN); - ingresses.add(new V1beta1Ingress().metadata(new V1ObjectMeta().namespace("namespace1").name("name1"))); - runStep(); + runDeleteStep(new V1beta1Ingress().metadata(new V1ObjectMeta().namespace("namespace1").name("name1"))); testSupport.verifyCompletionThrowable(ApiException.class); } @@ -78,8 +78,7 @@ public void onFailureResponse_reportError() throws Exception { @Test public void onNotFoundResponse_dontReportError() throws Exception { defineResponse("namespace1", "name1").failingWithStatus(HttpURLConnection.HTTP_NOT_FOUND); - ingresses.add(new V1beta1Ingress().metadata(new V1ObjectMeta().namespace("namespace1").name("name1"))); - runStep(); + runDeleteStep(new V1beta1Ingress().metadata(new V1ObjectMeta().namespace("namespace1").name("name1"))); } } \ No newline at end of file From a649328ac05dd04e7c88bc4795667fc755433704 Mon Sep 17 00:00:00 2001 From: Lily He Date: Mon, 23 Apr 2018 20:14:21 -0700 Subject: [PATCH 059/186] update delete-weblogic-domain-resources.sh to delete voyager res --- kubernetes/create-weblogic-domain-inputs.yaml | 2 +- .../delete-weblogic-domain-resources.sh | 44 +++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/kubernetes/create-weblogic-domain-inputs.yaml b/kubernetes/create-weblogic-domain-inputs.yaml index 41c92bd81cf..45195d08f5d 100644 --- a/kubernetes/create-weblogic-domain-inputs.yaml +++ b/kubernetes/create-weblogic-domain-inputs.yaml @@ -85,7 +85,7 @@ exposeAdminNodePort: false # Name of the domain namespace namespace: default -# Load balancer to deploy. Supported values are:TRAEFIK, NONE +# Load balancer to deploy. Supported values are:TRAEFIK, VOYAGER, NONE loadBalancer: TRAEFIK # Load balancer web port diff --git a/kubernetes/delete-weblogic-domain-resources.sh b/kubernetes/delete-weblogic-domain-resources.sh index ef96974324b..5ceb4cb2b75 100755 --- a/kubernetes/delete-weblogic-domain-resources.sh +++ b/kubernetes/delete-weblogic-domain-resources.sh @@ -53,6 +53,47 @@ cat << EOF EOF } +# +# get voyager related resources of one domain +# +# Usage: +# getVoyagerOfDomain domainName outfilename +function getVoyagerOfDomain { + local voyagerIngressName="ingress.voyager.appscode.com" + local domainName=$1 + local ns=`kubectl get ingress.voyager.appscode.com --all-namespaces | grep $domainName | awk '{ print $1 }'` + if [ -n "$ns" ]; then + echo $voyagerIngressName $domainName-voyager -n $ns >> $2 + echo service $domainName-voyager-stats -n $ns >> $2 + fi +} + +# +# get voyager related resources +# +# Usage: +# getVoyagerRes domainA,domainB,... outfilename +# getVoyagerRes all outfilename +function getVoyagerRes { + if [ "$1" = "all" ]; then + resList=`kubectl get ingress.voyager.appscode.com --all-namespaces | awk '{print $2}'` + for resName in $resList + do + if [ $resName != 'NAME' ]; then + tail="-voyager" + len=${#resName}-${#tail} + domainName=${resName:0:len} + getVoyagerOfDomain $domainName $2 + fi + done + else + IFS=',' read -r -a array <<< "$1" + for domainName in "${array[@]}" + do + getVoyagerOfDomain $domainName $2 + done + fi +} # # getDomainResources domain(s) outfilename @@ -98,6 +139,9 @@ function getDomainResources { -l "$LABEL_SELECTOR" \ -o=jsonpath='{range .items[*]}{.kind}{" "}{.metadata.name}{"\n"}{end}' \ --all-namespaces=true >> $2 + + # get all voyager-related resources + getVoyagerRes $1 $2 } # From 742e38f393692da01003da6277a6d5e0f585a82d Mon Sep 17 00:00:00 2001 From: Lily He Date: Tue, 24 Apr 2018 01:35:58 -0700 Subject: [PATCH 060/186] update cleanup.sh --- .../delete-weblogic-domain-resources.sh | 13 +- src/integration-tests/bash/cleanup.sh | 321 ++++-------------- 2 files changed, 71 insertions(+), 263 deletions(-) diff --git a/kubernetes/delete-weblogic-domain-resources.sh b/kubernetes/delete-weblogic-domain-resources.sh index 5ceb4cb2b75..ac0ea4f4609 100755 --- a/kubernetes/delete-weblogic-domain-resources.sh +++ b/kubernetes/delete-weblogic-domain-resources.sh @@ -64,7 +64,6 @@ function getVoyagerOfDomain { local ns=`kubectl get ingress.voyager.appscode.com --all-namespaces | grep $domainName | awk '{ print $1 }'` if [ -n "$ns" ]; then echo $voyagerIngressName $domainName-voyager -n $ns >> $2 - echo service $domainName-voyager-stats -n $ns >> $2 fi } @@ -116,8 +115,12 @@ function getDomainResources { LABEL_SELECTOR="weblogic.domainUID in ($1)" fi - # first, let's get all namespaced types with -l $LABEL_SELECTOR + # clean the output file + if [ -e $2 ]; then + rm $2 + fi + # first, let's get all namespaced types with -l $LABEL_SELECTOR NAMESPACED_TYPES="pod,job,deploy,rs,service,pvc,ingress,cm,serviceaccount,role,rolebinding,secret" # if domain crd exists, look for domains too: @@ -129,7 +132,7 @@ function getDomainResources { kubectl get $NAMESPACED_TYPES \ -l "$LABEL_SELECTOR" \ -o=jsonpath='{range .items[*]}{.kind}{" "}{.metadata.name}{" -n "}{.metadata.namespace}{"\n"}{end}' \ - --all-namespaces=true > $2 + --all-namespaces=true >> $2 # now, get all non-namespaced types with -l $LABEL_SELECTOR @@ -240,9 +243,9 @@ function deleteDomains { # for each namespace with leftover resources, try delete them cat $tempfile | awk '{ print $4 }' | grep -v "^$" | sort -u | while read line; do if [ "$test_mode" = "true" ]; then - echo kubectl -n $line delete $NAMESPACED_TYPES -l "$LABEL_SELECTOR" + echo kubectl -n $line delete $NAMESPACED_TYPES,ingress.voyager.appscode.com -l "$LABEL_SELECTOR" else - kubectl -n $line delete $NAMESPACED_TYPES -l "$LABEL_SELECTOR" + kubectl -n $line delete $NAMESPACED_TYPES,ingress.voyager.appscode.com -l "$LABEL_SELECTOR" fi done diff --git a/src/integration-tests/bash/cleanup.sh b/src/integration-tests/bash/cleanup.sh index 46b3f72458b..060b14a26d5 100755 --- a/src/integration-tests/bash/cleanup.sh +++ b/src/integration-tests/bash/cleanup.sh @@ -27,16 +27,10 @@ # # The test runs in 4 phases: # -# Phase 1: Delete test kubernetes artifacts in an orderly -# fashion via kubectl delete -f of previous tests's yaml -# plus various direct kubectl deletes. +# Phase 1: Delete domain resources with label 'weblogic.domainUID'. # -# Phase 2: Wait 15 seconds to see if stage 1 succeeded, and -# if not, repeatedly search for all test related kubectl -# artifacts and try delete them directly for up to 60 more -# seconds. This phase has no dependency on the -# previous test run's yaml files. It makes no -# attempt to delete artifacts in a particular order. +# Phase 2: Delete wls operator with lable 'weblogic.operatorName' and +# voyager operator with lable 'app=voyager'. # # Phase 3: Use a kubernetes job to delete the PV directories # on the kubernetes cluster. @@ -47,13 +41,6 @@ # see LEASE_ID above. # -DOMAINS=(domain1 domain2 domain3 domain4 domain5) -DOMAIN_NAMESPACES=(default default test1 test2 default) -DCOUNT=${#DOMAINS[@]} - -OPER_NAMESPACES=(weblogic-operator-1 weblogic-operator-2) -OCOUNT=${#OPER_NAMESPACES[@]} - SCRIPTPATH="$( cd "$(dirname "$0")" > /dev/null 2>&1 ; pwd -P )" PROJECT_ROOT="$SCRIPTPATH/../../.." RESULT_ROOT=${RESULT_ROOT:-/scratch/$USER/wl_k8s_test_results} @@ -63,248 +50,74 @@ USER_PROJECTS_DIR="$RESULT_DIR/user-projects" TMP_DIR="$RESULT_DIR/cleanup_tmp" JOB_NAME="weblogic-command-job" -# function genericDelete -# -# This function is a 'generic kubernetes delete' that takes three arguments: -# -# arg1: Comma separated list of types of kubernetes namespaced types to search/delete. -# example: "all,cm,pvc,ns,roles,rolebindings,secrets" -# -# arg2: Comma separated list of types of kubernetes non-namespaced types to search/delete. -# example: "crd,pv,clusterroles,clusterrolebindings" +function fail { + echo @@ cleanup.sh: Error "$@" + exit 1 +} + +#!/bin/bash # -# arg3: '|' (pipe) separated list of keywords. -# Artifacts with a label or name that contains one -# or more of the keywords are delete candidates. -# example: "logstash|kibana|elastisearch|weblogic|elk|domain" +# getResWithLabel outfilename # -# It runs in two stages: -# In the first, wait to see if artifacts delete on their own. -# In the second, try to delete any leftovers. -# -function genericDelete { - - for iteration in first second; do - # In the first iteration, we wait to see if artifacts delete. - # in the second iteration, we try to delete any leftovers. - - if [ "$iteration" = "first" ]; then - local maxwaitsecs=15 - else - local maxwaitsecs=60 - fi - - echo "@@ Waiting up to $maxwaitsecs seconds for ${1:?} and ${2:?} artifacts that contain string ${3:?} to delete." - - local artcount_no - local artcount_yes - local artcount_total - local resfile_no - local resfile_yes - - local mstart=`date +%s` - - while : ; do - resfile_no="$TMP_DIR/kinv_filtered_nonamespace.out.tmp" - resfile_yes="$TMP_DIR/kinv_filtered_yesnamespace.out.tmp" - - # leftover namespaced artifacts - kubectl get $1 \ - -o=jsonpath='{range .items[*]}{.metadata.namespace}{" "}{.kind}{"/"}{.metadata.name}{"\n"}{end}' \ - --all-namespaces=true 2>&1 \ - | egrep -e "($3)" | sort > $resfile_yes 2>&1 - artcount_yes="`cat $resfile_yes | wc -l`" - - # leftover non-namespaced artifacts - kubectl get $2 \ - -o=jsonpath='{range .items[*]}{.kind}{"/"}{.metadata.name}{"\n"}{end}' \ - --all-namespaces=true 2>&1 \ - | egrep -e "($3)" | sort > $resfile_no 2>&1 - artcount_no="`cat $resfile_no | wc -l`" - - artcount_total=$((artcount_yes + artcount_no)) - - mnow=`date +%s` - - if [ $((artcount_total)) -eq 0 ]; then - echo "@@ No artifacts found." - return 0 - fi - - if [ "$iteration" = "first" ]; then - # in the first iteration we just wait to see if artifacts go away on there own - - echo "@@ Waiting for $artcount_total artifacts to delete. Wait time $((mnow - mstart)) seconds (max=$maxwaitsecs). Waiting for:" - - cat $resfile_yes | awk '{ print "n=" $1 " " $2 }' - cat $resfile_no | awk '{ print $1 }' - - else - # in the second thirty seconds we try to delete remaining artifacts +function getResWithLabel { - echo "@@ Trying to delete ${artcount_total} leftover artifacts, including ${artcount_yes} namespaced artifacts and ${artcount_no} non-namespaced artifacts, wait time $((mnow - mstart)) seconds (max=$maxwaitsecs)." + # first, let's get all namespaced types with -l $LABEL_SELECTOR + kubectl get $NAMESPACED_TYPES \ + -l "$LABEL_SELECTOR" \ + -o=jsonpath='{range .items[*]}{.kind}{" "}{.metadata.name}{" -n "}{.metadata.namespace}{"\n"}{end}' \ + --all-namespaces=true >> $1 - if [ ${artcount_yes} -gt 0 ]; then - cat "$resfile_yes" | while read line; do - local args="`echo \"$line\" | awk '{ print "-n " $1 " delete " $2 " --ignore-not-found" }'`" - echo "kubectl $args" - kubectl $args - done - fi - - if [ ${artcount_no} -gt 0 ]; then - cat "$resfile_no" | while read line; do - echo "kubectl delete $line --ignore-not-found" - kubectl delete $line --ignore-not-found - done - fi - - fi - - if [ $((mnow - mstart)) -gt $((maxwaitsecs)) ]; then - if [ "$iteration" = "first" ]; then - echo "@@ Warning: ${maxwaitsecs} seconds reached. Will try deleting unexpected resources via kubectl delete." - else - echo "@@ Error: ${maxwaitsecs} seconds reached and possibly ${artcount_total} artifacts remaining. Giving up." - fi - break - fi - - sleep 5 - done - done - return 1 -} - -function kubectlDeleteF { - if [ -f "$1" ]; then - kubectl delete -f "$1" --ignore-not-found - else - echo @@ File \"$1\" not found. Skipping kubectl delete -f. - fi + # now, get all non-namespaced types with -l $LABEL_SELECTOR + kubectl get $NOT_NAMESPACED_TYPES \ + -l "$LABEL_SELECTOR" \ + -o=jsonpath='{range .items[*]}{.kind}{" "}{.metadata.name}{"\n"}{end}' \ + --all-namespaces=true >> $1 } # -# This function looks for specific weblogic operator kubernetes artifacts and deletes them +# deleteResWithLabel outputfile # -function orderlyDelete { - # TODO: Some of the cleanup in this function depends on yaml files generated by run.sh which - # probably won't be there in Wercker, and so will be skipped. The 'genericDelete' function that - # is called after this function seems to take care of the leftovers, but we may want to - # have it delete in a certain order. We also want to revisit this orderly delete to make - # it less dependent on yaml and pre-defined lists of domain and operator names & namespaces. - # Eventually, the two methods could converge and one of them could then go away... - - for ((i=0;i&1 --ignore-not-found | grep -v "the server doesn.t have a resource type" - - # Give operator some time to digest the domain deletion - sleep 3 - - echo @@ Deleting create domain ${curdomain} job in namespace $curns - kubectl -n $curns delete job ${curdomain}-create-weblogic-domain-job --ignore-not-found - - echo @@ Deleting domain pv and pvc for domain ${curdomain} in namespace $curns - kubectl delete pv ${curdomain}-weblogic-domain-pv --ignore-not-found - kubectl -n $curns delete pvc ${curdomain}-weblogic-domain-pvc --ignore-not-found - - echo @@ Deleting ${curdomain}-weblogic-credentials secret in namespace $curns - kubectl -n $curns delete secret ${curdomain}-weblogic-credentials --ignore-not-found - - echo @@ Deleting ${curdomain} traefik in namespace $curns - kubectlDeleteF "${USER_PROJECTS_DIR}/weblogic-domains/${curdomain}/weblogic-domain-traefik-cluster-1.yaml" - kubectlDeleteF "${USER_PROJECTS_DIR}/weblogic-domains/${curdomain}/weblogic-domain-traefik-security-cluster-1.yaml" - - echo @@ Deleting configmap ${curdomain}-create-weblogic-domain-job-cm in namespace $curns - kubectl -n $curns delete cm ${curdomain}-create-weblogic-domain-job-cm --ignore-not-found - - kubectl -n $curns delete deploy ${curdomain}-cluster-1-traefik --ignore-not-found=true - kubectl -n $curns delete service ${curdomain}-cluster-1-traefik --ignore-not-found=true - kubectl -n $curns delete service ${curdomain}-cluster-1-traefik-dashboard --ignore-not-found=true - kubectl -n $curns delete cm ${curdomain}-cluster-1-traefik --ignore-not-found=true - kubectl -n $curns delete serviceaccount ${curdomain}-cluster-1-traefik --ignore-not-found=true - kubectl -n $curns delete clusterrole ${curdomain}-cluster-1-traefik --ignore-not-found=true - kubectl -n $curns delete clusterrolebinding ${curdomain}-cluster-1-traefik --ignore-not-found=true - done - - for ((i=0;i Date: Tue, 24 Apr 2018 08:09:29 -0700 Subject: [PATCH 061/186] update cleanup.sh --- kubernetes/internal/create-weblogic-domain.sh | 4 +-- src/integration-tests/bash/cleanup.sh | 29 ++++++++++++------- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/kubernetes/internal/create-weblogic-domain.sh b/kubernetes/internal/create-weblogic-domain.sh index a98984e8255..579be9cc27f 100755 --- a/kubernetes/internal/create-weblogic-domain.sh +++ b/kubernetes/internal/create-weblogic-domain.sh @@ -612,8 +612,8 @@ function createDomain { # function setupVoyagerLoadBalancer { # only deploy Voyager Ingress Controller the first time - local vcon=`kubectl get namespace voyager| grep voyager | wc | awk ' { print $1; } '` - if [ "$vcon" == "0" ]; then + local vpod=`kubectl get pod -n voyager | grep voyager | wc -l` + if [ "$vpod" == "0" ]; then kubectl create namespace voyager curl -fsSL https://raw.githubusercontent.com/appscode/voyager/6.0.0/hack/deploy/voyager.sh \ | bash -s -- --provider=baremetal --namespace=voyager diff --git a/src/integration-tests/bash/cleanup.sh b/src/integration-tests/bash/cleanup.sh index 060b14a26d5..8b9b453195b 100755 --- a/src/integration-tests/bash/cleanup.sh +++ b/src/integration-tests/bash/cleanup.sh @@ -30,7 +30,7 @@ # Phase 1: Delete domain resources with label 'weblogic.domainUID'. # # Phase 2: Delete wls operator with lable 'weblogic.operatorName' and -# voyager operator with lable 'app=voyager'. +# delete voyager controller. # # Phase 3: Use a kubernetes job to delete the PV directories # on the kubernetes cluster. @@ -105,22 +105,27 @@ function deleteResWithLabel { } # -# deleteOperators +# deleteWLSOperators # -function deleteOperators { +function deleteWLSOperators { local tempfile="/tmp/$(basename $0).tmp.$$" # == /tmp/[script-file-name].tmp.[pid] # delete wls operator resources LABEL_SELECTOR="weblogic.operatorName" #getResWithLabel $tempfile deleteResWithLabel $tempfile +} - # delete voyager operator resources - LABEL_SELECTOR="app=voyager" - #getResWithLabel $tempfile - deleteResWithLabel $tempfile +# +# deleteVoyagerController +# +function deleteVoyagerController { + curl -fsSL https://raw.githubusercontent.com/appscode/voyager/6.0.0/hack/deploy/voyager.sh \ + | bash -s -- --provider=baremetal --namespace=voyager --uninstall --purge } echo @@ Starting cleanup. +script="${BASH_SOURCE[0]}" +scriptDir="$( cd "$(dirname "${script}")" > /dev/null 2>&1 ; pwd -P)" echo "@@ RESULT_ROOT=$RESULT_ROOT TMP_DIR=$TMP_DIR RESULT_DIR=$RESULT_DIR PROJECT_ROOT=$PROJECT_ROOT" @@ -130,10 +135,13 @@ NAMESPACED_TYPES="pod,job,deploy,rs,service,pvc,ingress,cm,serviceaccount,role,r NOT_NAMESPACED_TYPES="pv,crd,clusterroles,clusterrolebindings" # Delele domain resources. -../../../kubernetes/delete-weblogic-domain-resources.sh -d all +${scriptDir}/../../../kubernetes/delete-weblogic-domain-resources.sh -d all + +# Delete wls operator +deleteWLSOperators -# Delete wls operator and voyager operator -deleteOperators +# Delete voyager controller +deleteVoyagerController # Delete pv directories using a job (/scratch maps to PV_ROOT on the k8s cluster machines). @@ -166,3 +174,4 @@ fi echo @@ Exiting with status $SUCCESS exit $SUCCESS + From e1c023da02051c212f1b07ee25c9fa2abfc3b0c8 Mon Sep 17 00:00:00 2001 From: Lily He Date: Wed, 25 Apr 2018 02:10:58 -0700 Subject: [PATCH 062/186] update cleanup.sh --- .../delete-weblogic-domain-resources.sh | 54 ++---------- src/integration-tests/bash/cleanup.sh | 86 +++++++++++++------ 2 files changed, 66 insertions(+), 74 deletions(-) diff --git a/kubernetes/delete-weblogic-domain-resources.sh b/kubernetes/delete-weblogic-domain-resources.sh index ac0ea4f4609..d92782bd768 100755 --- a/kubernetes/delete-weblogic-domain-resources.sh +++ b/kubernetes/delete-weblogic-domain-resources.sh @@ -53,47 +53,6 @@ cat << EOF EOF } -# -# get voyager related resources of one domain -# -# Usage: -# getVoyagerOfDomain domainName outfilename -function getVoyagerOfDomain { - local voyagerIngressName="ingress.voyager.appscode.com" - local domainName=$1 - local ns=`kubectl get ingress.voyager.appscode.com --all-namespaces | grep $domainName | awk '{ print $1 }'` - if [ -n "$ns" ]; then - echo $voyagerIngressName $domainName-voyager -n $ns >> $2 - fi -} - -# -# get voyager related resources -# -# Usage: -# getVoyagerRes domainA,domainB,... outfilename -# getVoyagerRes all outfilename -function getVoyagerRes { - if [ "$1" = "all" ]; then - resList=`kubectl get ingress.voyager.appscode.com --all-namespaces | awk '{print $2}'` - for resName in $resList - do - if [ $resName != 'NAME' ]; then - tail="-voyager" - len=${#resName}-${#tail} - domainName=${resName:0:len} - getVoyagerOfDomain $domainName $2 - fi - done - else - IFS=',' read -r -a array <<< "$1" - for domainName in "${array[@]}" - do - getVoyagerOfDomain $domainName $2 - done - fi -} - # # getDomainResources domain(s) outfilename # @@ -129,6 +88,11 @@ function getDomainResources { NAMESPACED_TYPES="domain,$NAMESPACED_TYPES" fi + VOYAGER_ING_NAME="ingresses.voyager.appscode.com" + if [ `kubectl get crd $VOYAGER_ING_NAME |grep $VOYAGER_ING_NAME | wc -l` = 1 ]; then + NAMESPACED_TYPES="$VOYAGER_ING_NAME,$NAMESPACED_TYPES" + fi + kubectl get $NAMESPACED_TYPES \ -l "$LABEL_SELECTOR" \ -o=jsonpath='{range .items[*]}{.kind}{" "}{.metadata.name}{" -n "}{.metadata.namespace}{"\n"}{end}' \ @@ -142,9 +106,6 @@ function getDomainResources { -l "$LABEL_SELECTOR" \ -o=jsonpath='{range .items[*]}{.kind}{" "}{.metadata.name}{"\n"}{end}' \ --all-namespaces=true >> $2 - - # get all voyager-related resources - getVoyagerRes $1 $2 } # @@ -243,9 +204,9 @@ function deleteDomains { # for each namespace with leftover resources, try delete them cat $tempfile | awk '{ print $4 }' | grep -v "^$" | sort -u | while read line; do if [ "$test_mode" = "true" ]; then - echo kubectl -n $line delete $NAMESPACED_TYPES,ingress.voyager.appscode.com -l "$LABEL_SELECTOR" + echo kubectl -n $line delete $NAMESPACED_TYPES -l "$LABEL_SELECTOR" else - kubectl -n $line delete $NAMESPACED_TYPES,ingress.voyager.appscode.com -l "$LABEL_SELECTOR" + kubectl -n $line delete $NAMESPACED_TYPES -l "$LABEL_SELECTOR" fi done @@ -307,3 +268,4 @@ if [ ! -x "$(command -v kubectl)" ]; then fi deleteDomains "${domains}" "${maxwaitsecs:-$default_maxwaitsecs}" + diff --git a/src/integration-tests/bash/cleanup.sh b/src/integration-tests/bash/cleanup.sh index 8b9b453195b..36d4533eeff 100755 --- a/src/integration-tests/bash/cleanup.sh +++ b/src/integration-tests/bash/cleanup.sh @@ -78,6 +78,7 @@ function getResWithLabel { # deleteResWithLabel outputfile # function deleteResWithLabel { + echo @@ Delete resources with label $LABEL_SELECTOR. # clean the output file first if [ -e $1 ]; then rm $1 @@ -86,43 +87,51 @@ function deleteResWithLabel { getResWithLabel $1 # delete namespaced types cat $1 | awk '{ print $4 }' | grep -v "^$" | sort -u | while read line; do - if [ "$test_mode" = "true" ]; then - echo kubectl -n $line delete $NAMESPACED_TYPES -l "$LABEL_SELECTOR" - else - kubectl -n $line delete $NAMESPACED_TYPES -l "$LABEL_SELECTOR" - fi + kubectl -n $line delete $NAMESPACED_TYPES -l "$LABEL_SELECTOR" done # delete non-namespaced types local no_namespace_count=`grep -c -v " -n " $1` if [ ! "$no_namespace_count" = "0" ]; then - if [ "$test_mode" = "true" ]; then - echo kubectl delete $NOT_NAMESPACED_TYPES -l "$LABEL_SELECTOR" + kubectl delete $NOT_NAMESPACED_TYPES -l "$LABEL_SELECTOR" + fi + + echo "@@ Waiting for pods to stop running." + local total=0 + for count in {1..100}; do + pods=($(kubectl get pods --all-namespaces -l weblogic.domainUID -o jsonpath='{range .items[*]}{.metadata.name} {end}')) + total=${#pods[*]} + if [ $total -eq 0 ] ; then + break else - kubectl delete $NOT_NAMESPACED_TYPES -l "$LABEL_SELECTOR" + echo "@@ There are still $total running pods with label $LABEL_SELECTOR." fi - fi -} + sleep 3 + done -# -# deleteWLSOperators -# -function deleteWLSOperators { - local tempfile="/tmp/$(basename $0).tmp.$$" # == /tmp/[script-file-name].tmp.[pid] - # delete wls operator resources - LABEL_SELECTOR="weblogic.operatorName" - #getResWithLabel $tempfile - deleteResWithLabel $tempfile + if [ $total -gt 0 ]; then + echo "Warning: after waiting 300 seconds, there are still $total running pods with label $LABEL_SELECTOR." + fi } -# -# deleteVoyagerController -# function deleteVoyagerController { + curl -fsSL https://raw.githubusercontent.com/appscode/voyager/6.0.0/hack/deploy/voyager.sh \ | bash -s -- --provider=baremetal --namespace=voyager --uninstall --purge + kubectl delete namespace voyager } +# +# deleteNamespaces outputfile +# +function deleteNamespaces { + cat $1 | awk '{ print $4 }' | grep -v "^$" | sort -u | while read line; do + if [ "$line" != "default" ]; then + kubectl delete namespace $line --ignore-not-found + fi + done + +} echo @@ Starting cleanup. script="${BASH_SOURCE[0]}" scriptDir="$( cd "$(dirname "${script}")" > /dev/null 2>&1 ; pwd -P)" @@ -132,16 +141,37 @@ echo "@@ RESULT_ROOT=$RESULT_ROOT TMP_DIR=$TMP_DIR RESULT_DIR=$RESULT_DIR PROJEC mkdir -p $TMP_DIR || fail No permision to create directory $TMP_DIR NAMESPACED_TYPES="pod,job,deploy,rs,service,pvc,ingress,cm,serviceaccount,role,rolebinding,secret" + +HANDLE_VOYAGER="false" +VOYAGER_ING_NAME="ingresses.voyager.appscode.com" +if [ `kubectl get crd $VOYAGER_ING_NAME --ignore-not-found | grep $VOYAGER_ING_NAME | wc -l` = 1 ]; then + NAMESPACED_TYPES="$VOYAGER_ING_NAME,$NAMESPACED_TYPES" + HANDLE_VOYAGER="true" +fi + +DOMAIN_CRD="domains.weblogic.oracle" +if [ `kubectl get crd $DOMAIN_CRD --ignore-not-found | grep $DOMAIN_CRD | wc -l` = 1 ]; then + NAMESPACED_TYPES="$DOMAIN_CRD,$NAMESPACED_TYPES" +fi + NOT_NAMESPACED_TYPES="pv,crd,clusterroles,clusterrolebindings" -# Delele domain resources. -${scriptDir}/../../../kubernetes/delete-weblogic-domain-resources.sh -d all +tempfile="/tmp/$(basename $0).tmp.$$" # == /tmp/[script-file-name].tmp.[pid] + +echo @@ Deleting domain resources. +LABEL_SELECTOR="weblogic.domainUID" +deleteResWithLabel "$tempfile-0" +deleteNamespaces "$tempfile-0" -# Delete wls operator -deleteWLSOperators +echo @@ Deleting wls operator resources. +LABEL_SELECTOR="weblogic.operatorName" +deleteResWithLabel "$tempfile-1" +deleteNamespaces "$tempfile-1" -# Delete voyager controller -deleteVoyagerController +echo @@ Deleting voyager controller. +if [ "$HANDLE_VOYAGER" = "true" ]; then + deleteVoyagerController +fi # Delete pv directories using a job (/scratch maps to PV_ROOT on the k8s cluster machines). From 702006b3c1904a77c511dd09098faad2cf9466ab Mon Sep 17 00:00:00 2001 From: Dongbo Xiao Date: Wed, 25 Apr 2018 03:39:58 -0700 Subject: [PATCH 063/186] Add support in create domain job for Apache load balancer Signed-off-by: Dongbo Xiao --- kubernetes/create-weblogic-domain-inputs.yaml | 9 +- .../delete-weblogic-domain-resources.sh | 4 +- kubernetes/internal/create-weblogic-domain.sh | 135 +++++++++++++++--- ...logic-domain-apache-security-template.yaml | 44 ++++++ .../weblogic-domain-apache-template.yaml | 109 ++++++++++++++ src/integration-tests/bash/cleanup.sh | 12 +- src/integration-tests/bash/run.sh | 19 ++- 7 files changed, 306 insertions(+), 26 deletions(-) create mode 100644 kubernetes/internal/weblogic-domain-apache-security-template.yaml create mode 100755 kubernetes/internal/weblogic-domain-apache-template.yaml diff --git a/kubernetes/create-weblogic-domain-inputs.yaml b/kubernetes/create-weblogic-domain-inputs.yaml index 41c92bd81cf..9df0131c798 100644 --- a/kubernetes/create-weblogic-domain-inputs.yaml +++ b/kubernetes/create-weblogic-domain-inputs.yaml @@ -85,9 +85,16 @@ exposeAdminNodePort: false # Name of the domain namespace namespace: default -# Load balancer to deploy. Supported values are:TRAEFIK, NONE +# Load balancer to deploy. Supported values are:TRAEFIK, APACHE, NONE loadBalancer: TRAEFIK +# Load balancer app prepath +loadBalancerAppPrepath: / + +# Docker volume path for APACHE +# By default, the VolumePath is empty, which will cause the volume mount be disabled +loadBalancerVolumePath: + # Load balancer web port loadBalancerWebPort: 30305 diff --git a/kubernetes/delete-weblogic-domain-resources.sh b/kubernetes/delete-weblogic-domain-resources.sh index ef96974324b..81fb4f25a9e 100755 --- a/kubernetes/delete-weblogic-domain-resources.sh +++ b/kubernetes/delete-weblogic-domain-resources.sh @@ -137,8 +137,8 @@ function deleteDomains { # get a count of all k8s resources with matching domain-uid labels local allcount=`wc -l $tempfile | awk '{ print $1 }'` - # get a count of all WLS pods (any pod with a matching domain-uid label that doesn't have 'traefik' embedded in its name) - local podcount=`grep "^Pod" $tempfile | grep -v traefik | wc -l | awk '{ print $1 }'` + # get a count of all WLS pods (any pod with a matching domain-uid label that doesn't have 'traefik' or 'apache' embedded in its name) + local podcount=`grep "^Pod" $tempfile | grep -v traefik | grep -v apache | wc -l | awk '{ print $1 }'` local mnow=`date +%s` diff --git a/kubernetes/internal/create-weblogic-domain.sh b/kubernetes/internal/create-weblogic-domain.sh index 06f9c295d94..51c85a44f55 100755 --- a/kubernetes/internal/create-weblogic-domain.sh +++ b/kubernetes/internal/create-weblogic-domain.sh @@ -92,6 +92,7 @@ fi # function initAndValidateOutputDir { domainOutputDir="${outputDir}/weblogic-domains/${domainUID}" + validateOutputDir \ ${domainOutputDir} \ ${valuesInputFile} \ @@ -100,6 +101,8 @@ function initAndValidateOutputDir { weblogic-domain-pvc.yaml \ weblogic-domain-traefik-${clusterNameLC}.yaml \ weblogic-domain-traefik-security-${clusterNameLC}.yaml \ + weblogic-domain-apache.yaml \ + weblogic-domain-apache-security.yaml create-weblogic-domain-job.yaml \ domain-custom-resource.yaml } @@ -183,10 +186,12 @@ function validateLoadBalancer { case ${loadBalancer} in "TRAEFIK") ;; + "APACHE") + ;; "NONE") ;; *) - validationError "Invalid value for loadBalancer: ${loadBalancer}. Valid values are TRAEFIK and NONE." + validationError "Invalid value for loadBalancer: ${loadBalancer}. Valid values are TRAEFIK, APACHE and NONE." ;; esac fi @@ -321,6 +326,16 @@ function initialize { validationError "The template file ${traefikInput} for generating the traefik deployment was not found" fi + apacheSecurityInput="${scriptDir}/weblogic-domain-apache-security-template.yaml" + if [ ! -f ${apacheSecurityInput} ]; then + validationError "The file ${apacheSecurityInput} for generating the apache-webtier RBAC was not found" + fi + + apacheInput="${scriptDir}/weblogic-domain-apache-template.yaml" + if [ ! -f ${apacheInput} ]; then + validationError "The template file ${apacheInput} for generating the apache-webtier deployment was not found" + fi + failIfValidationErrors # Parse the commonn inputs file @@ -388,6 +403,9 @@ function createYamlFiles { dcrOutput="${domainOutputDir}/domain-custom-resource.yaml" traefikSecurityOutput="${domainOutputDir}/weblogic-domain-traefik-security-${clusterNameLC}.yaml" traefikOutput="${domainOutputDir}/weblogic-domain-traefik-${clusterNameLC}.yaml" + apacheOutput="${domainOutputDir}/weblogic-domain-apache.yaml" + apacheSecurityOutput="${domainOutputDir}/weblogic-domain-apache-security.yaml" + enabledPrefix="" # uncomment the feature disabledPrefix="# " # comment out the feature @@ -473,25 +491,62 @@ function createYamlFiles { sed -i -e "s:%JAVA_OPTIONS%:${javaOptions}:g" ${dcrOutput} sed -i -e "s:%STARTUP_CONTROL%:${startupControl}:g" ${dcrOutput} - # Traefik file - cp ${traefikInput} ${traefikOutput} - echo Generating ${traefikOutput} - sed -i -e "s:%NAMESPACE%:$namespace:g" ${traefikOutput} - sed -i -e "s:%DOMAIN_UID%:${domainUID}:g" ${traefikOutput} - sed -i -e "s:%DOMAIN_NAME%:${domainName}:g" ${traefikOutput} - sed -i -e "s:%CLUSTER_NAME%:${clusterName}:g" ${traefikOutput} - sed -i -e "s:%CLUSTER_NAME_LC%:${clusterNameLC}:g" ${traefikOutput} - sed -i -e "s:%LOAD_BALANCER_WEB_PORT%:$loadBalancerWebPort:g" ${traefikOutput} - sed -i -e "s:%LOAD_BALANCER_DASHBOARD_PORT%:$loadBalancerDashboardPort:g" ${traefikOutput} - - # Traefik security file - cp ${traefikSecurityInput} ${traefikSecurityOutput} - echo Generating ${traefikSecurityOutput} - sed -i -e "s:%NAMESPACE%:$namespace:g" ${traefikSecurityOutput} - sed -i -e "s:%DOMAIN_UID%:${domainUID}:g" ${traefikSecurityOutput} - sed -i -e "s:%DOMAIN_NAME%:${domainName}:g" ${traefikSecurityOutput} - sed -i -e "s:%CLUSTER_NAME%:${clusterName}:g" ${traefikSecurityOutput} - sed -i -e "s:%CLUSTER_NAME_LC%:${clusterNameLC}:g" ${traefikSecurityOutput} + if [ "${loadBalancer}" = "TRAEFIK" ]; then + # Traefik file + cp ${traefikInput} ${traefikOutput} + echo Generating ${traefikOutput} + sed -i -e "s:%NAMESPACE%:$namespace:g" ${traefikOutput} + sed -i -e "s:%DOMAIN_UID%:${domainUID}:g" ${traefikOutput} + sed -i -e "s:%DOMAIN_NAME%:${domainName}:g" ${traefikOutput} + sed -i -e "s:%CLUSTER_NAME%:${clusterName}:g" ${traefikOutput} + sed -i -e "s:%CLUSTER_NAME_LC%:${clusterNameLC}:g" ${traefikOutput} + sed -i -e "s:%LOAD_BALANCER_WEB_PORT%:$loadBalancerWebPort:g" ${traefikOutput} + sed -i -e "s:%LOAD_BALANCER_DASHBOARD_PORT%:$loadBalancerDashboardPort:g" ${traefikOutput} + + # Traefik security file + cp ${traefikSecurityInput} ${traefikSecurityOutput} + echo Generating ${traefikSecurityOutput} + sed -i -e "s:%NAMESPACE%:$namespace:g" ${traefikSecurityOutput} + sed -i -e "s:%DOMAIN_UID%:${domainUID}:g" ${traefikSecurityOutput} + sed -i -e "s:%DOMAIN_NAME%:${domainName}:g" ${traefikSecurityOutput} + sed -i -e "s:%CLUSTER_NAME%:${clusterName}:g" ${traefikSecurityOutput} + sed -i -e "s:%CLUSTER_NAME_LC%:${clusterNameLC}:g" ${traefikSecurityOutput} + + elif [ "${loadBalancer}" = "APACHE" ]; then + # Apache file + cp ${apacheInput} ${apacheOutput} + echo Generating ${apacheOutput} + sed -i -e "s:%NAMESPACE%:$namespace:g" ${apacheOutput} + sed -i -e "s:%DOMAIN_UID%:${domainUID}:g" ${apacheOutput} + sed -i -e "s:%DOMAIN_NAME%:${domainName}:g" ${apacheOutput} + sed -i -e "s:%CLUSTER_NAME_LC%:${clusterNameLC}:g" ${apacheOutput} + sed -i -e "s:%ADMIN_SERVER_NAME%:${adminServerName}:g" ${apacheOutput} + sed -i -e "s:%ADMIN_PORT%:${adminPort}:g" ${apacheOutput} + sed -i -e "s:%MANAGED_SERVER_PORT%:${managedServerPort}:g" ${apacheOutput} + sed -i -e "s:%LOAD_BALANCER_WEB_PORT%:$loadBalancerWebPort:g" ${apacheOutput} + sed -i -e "s:%WEB_APP_PREPATH%:$loadBalancerAppPrepath:g" ${apacheOutput} + + if [ ${loadBalancerVolumePath} != "" ]; then + sed -i -e "s:%LOAD_BALANCER_VOLUME_PATH%:${loadBalancerVolumePath}:g" ${apacheOutput} + sed -i -e "s:# volumes:volumes:g" ${apacheOutput} + sed -i -e "s:# - name:- name:g" ${apacheOutput} + sed -i -e "s:# hostPath: hostPath:g" ${apacheOutput} + sed -i -e "s:# path: path:g" ${apacheOutput} + sed -i -e "s:# volumeMounts:volumeMounts:g" ${apacheOutput} + sed -i -e "s:# - name:- name:g" ${apacheOutput} + sed -i -e "s:# mountPath: mountPath:g" ${apacheOutput} + fi + + mkdir -p ${loadBalancerVolumePath} + cp ${scriptDir}/custom_mod_wl_apache.conf ${loadBalancerVolumePath}/ + + # Apache security file + cp ${apacheSecurityInput} ${apacheSecurityOutput} + echo Generating ${apacheSecurityOutput} + sed -i -e "s:%NAMESPACE%:$namespace:g" ${apacheSecurityOutput} + sed -i -e "s:%DOMAIN_UID%:${domainUID}:g" ${apacheSecurityOutput} + sed -i -e "s:%DOMAIN_NAME%:${domainName}:g" ${apacheSecurityOutput} + fi # Remove any "...yaml-e" files left over from running sed rm -f ${domainOutputDir}/*.yaml-e @@ -628,6 +683,36 @@ function setupTraefikLoadBalancer { fi } +# +# Deploy Apache load balancer +# +function setupApacheLoadBalancer { + + apacheName="${domainUID}-apache-webtier" + apacheServiceName="${domainUID}-external-apache-webtier-service" + + echo Deploying apache + kubectl apply -f ${apacheOutput} + + echo Checking apache deployment + SS=`kubectl get deployment -n ${namespace} | grep ${apacheName} | wc | awk ' { print $1; } '` + if [ "$SS" != "1" ]; then + fail "The deployment ${apacheName} was not created" + fi + + echo Checking the apache service account + SA=`kubectl get serviceaccount ${apacheName} -n ${namespace} | grep ${apacheName} | wc | awk ' { print $1; } '` + if [ "$SA" != "1" ]; then + fail "The service account ${apacheName} was not created" + fi + + echo Checking apache service + TSVC=`kubectl get services -n ${namespace} | grep ${apacheServiceName} | wc | awk ' { print $1; } '` + if [ "$TSVC" != "1" ]; then + fail "The service ${apacheServiceName} was not created" + fi +} + # # Function to create the domain custom resource # @@ -686,6 +771,9 @@ function outputJobSummary { echo "The load balancer for cluster '${clusterName}' is available at http:${K8S_IP}:${loadBalancerWebPort}/ (add the application path to the URL)" echo "The load balancer dashboard for cluster '${clusterName}' is available at http:${K8S_IP}:${loadBalancerDashboardPort}" echo "" + elif [ "${loadBalancer}" = "APACHE" ]; then + echo "The apache load balancer for '${domainUID}' is available at http:${K8S_IP}:${loadBalancerWebPort}/ (add the application path to the URL)" + fi echo "The following files were generated:" echo " ${domainOutputDir}/create-weblogic-domain-inputs.yaml" @@ -696,6 +784,10 @@ function outputJobSummary { if [ "${loadBalancer}" = "TRAEFIK" ]; then echo " ${traefikSecurityOutput}" echo " ${traefikOutput}" + elif [ "${loadBalancer}" = "APACHE" ]; then + echo " ${apacheSecurityOutput}" + echo " ${apacheOutput}" + fi } @@ -726,6 +818,9 @@ if [ "${generateOnly}" = false ]; then # Setup load balancer if [ "${loadBalancer}" = "TRAEFIK" ]; then setupTraefikLoadBalancer + elif [ "${loadBalancer}" = "APACHE" ]; then + setupApacheLoadBalancer + fi # Create the domain custom resource diff --git a/kubernetes/internal/weblogic-domain-apache-security-template.yaml b/kubernetes/internal/weblogic-domain-apache-security-template.yaml new file mode 100644 index 00000000000..30fa8200a42 --- /dev/null +++ b/kubernetes/internal/weblogic-domain-apache-security-template.yaml @@ -0,0 +1,44 @@ +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: %DOMAIN_UID%-apache-webtier + labels: + weblogic.domainUID: %DOMAIN_UID% + weblogic.domainName: %DOMAIN_NAME% +rules: + - apiGroups: + - "" + resources: + - pods + - services + - endpoints + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - extensions + resources: + - ingresses + verbs: + - get + - list + - watch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: %DOMAIN_UID%-apache-webtier + labels: + weblogic.domainUID: %DOMAIN_UID% + weblogic.domainName: %DOMAIN_NAME% +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: %DOMAIN_UID%-apache-webtier +subjects: +- kind: ServiceAccount + name: %DOMAIN_UID%-apache-webtier + namespace: %NAMESPACE% diff --git a/kubernetes/internal/weblogic-domain-apache-template.yaml b/kubernetes/internal/weblogic-domain-apache-template.yaml new file mode 100755 index 00000000000..f6f821f732a --- /dev/null +++ b/kubernetes/internal/weblogic-domain-apache-template.yaml @@ -0,0 +1,109 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: %DOMAIN_UID%-apache-webtier + namespace: %NAMESPACE% + labels: + weblogic.domainUID: %DOMAIN_UID% + weblogic.domainName: %DOMAIN_NAME% + app: apache-webtier +--- +kind: Deployment +apiVersion: extensions/v1beta1 +metadata: + name: %DOMAIN_UID%-apache-webtier + namespace: %NAMESPACE% + labels: + weblogic.domainUID: %DOMAIN_UID% + weblogic.domainName: %DOMAIN_NAME% + app: apache-webtier +spec: + replicas: 1 + selector: + matchLabels: + weblogic.domainUID: %DOMAIN_UID% + weblogic.domainName: %DOMAIN_NAME% + app: apache-webtier + template: + metadata: + labels: + weblogic.domainUID: %DOMAIN_UID% + weblogic.domainName: %DOMAIN_NAME% + app: apache-webtier + spec: + serviceAccountName: %DOMAIN_UID%-apache-webtier + terminationGracePeriodSeconds: 60 + # volumes: + # - name: "%DOMAIN_UID%-apache-webtier" + # hostPath: + # path: %LOAD_BALANCER_VOLUME_PATH% + containers: + - name: %DOMAIN_UID%-apache-webtier + image: 12213-apache:latest + imagePullPolicy: Never + # volumeMounts: + # - name: "%DOMAIN_UID%-apache-webtier" + # mountPath: "/config" + env: + - name: WEBLOGIC_CLUSTER + value: '%DOMAIN_UID%-cluster-%CLUSTER_NAME_LC%:%MANAGED_SERVER_PORT%' + - name: LOCATION + value: '%WEB_APP_PREPATH%' + - name: WEBLOGIC_HOST + value: '%DOMAIN_UID%-%ADMIN_SERVER_NAME%' + - name: WEBLOGIC_PORT + value: '%ADMIN_PORT%' + readinessProbe: + tcpSocket: + port: 80 + failureThreshold: 1 + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 2 + livenessProbe: + tcpSocket: + port: 80 + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 2 + +--- +apiVersion: v1 +kind: Service +metadata: + name: %DOMAIN_UID%-external-apache-webtier-service + namespace: %NAMESPACE% + labels: + weblogic.domainUID: %DOMAIN_UID% + weblogic.domainName: %DOMAIN_NAME% +spec: + type: NodePort + selector: + weblogic.domainUID: %DOMAIN_UID% + weblogic.domainName: %DOMAIN_NAME% + app: apache-webtier + ports: + - port: 80 + nodePort: %LOAD_BALANCER_WEB_PORT% + name: rest-https + +# --- +# apiVersion: v1 +# kind: Service +# metadata: +# name: %DOMAIN_UID%-internal-apache-webtier-service +# namespace: %NAMESPACE% +# labels: +# weblogic.domainUID: %DOMAIN_UID% +# weblogic.domainName: %DOMAIN_NAME% +# spec: +# type: ClusterIP +# selector: +# app: %DOMAIN_UID%-apache-webtier +# ports: +# - port: %LOAD_BALANCER_WEB_PORT_INTERNAL% +# name: rest-httpe700 diff --git a/src/integration-tests/bash/cleanup.sh b/src/integration-tests/bash/cleanup.sh index 46b3f72458b..52efd5109df 100755 --- a/src/integration-tests/bash/cleanup.sh +++ b/src/integration-tests/bash/cleanup.sh @@ -219,6 +219,10 @@ function orderlyDelete { kubectlDeleteF "${USER_PROJECTS_DIR}/weblogic-domains/${curdomain}/weblogic-domain-traefik-cluster-1.yaml" kubectlDeleteF "${USER_PROJECTS_DIR}/weblogic-domains/${curdomain}/weblogic-domain-traefik-security-cluster-1.yaml" + echo @@ Deleting apache in namespace $curns + kubectlDeleteF "${USER_PROJECTS_DIR}/weblogic-domains/${curdomain}/weblogic-domain-apache.yaml" + kubectlDeleteF "${USER_PROJECTS_DIR}/weblogic-domains/${curdomain}/weblogic-domain-apache-security.yaml" + echo @@ Deleting configmap ${curdomain}-create-weblogic-domain-job-cm in namespace $curns kubectl -n $curns delete cm ${curdomain}-create-weblogic-domain-job-cm --ignore-not-found @@ -229,6 +233,12 @@ function orderlyDelete { kubectl -n $curns delete serviceaccount ${curdomain}-cluster-1-traefik --ignore-not-found=true kubectl -n $curns delete clusterrole ${curdomain}-cluster-1-traefik --ignore-not-found=true kubectl -n $curns delete clusterrolebinding ${curdomain}-cluster-1-traefik --ignore-not-found=true + + kubectl -n $curns delete deploy ${curdomain}-apache-webtier --ignore-not-found=true + kubectl -n $curns delete service ${curdomain}-external-apache-webtier-service --ignore-not-found=true + kubectl -n $curns delete serviceaccount ${curdomain}-apache-webtier --ignore-not-found=true + kubectl -n $curns delete clusterrole ${curdomain}-apache-webtier --ignore-not-found=true + kubectl -n $curns delete clusterrolebinding ${curdomain}-apache-webtier --ignore-not-found=true done for ((i=0;i> $TMP_DIR/describe.ingress.out kubectl describe ingress -n $NAMESPACE >> $TMP_DIR/describe.ingress.out 2>&1 - + local TEST_APP_URL="http://${NODEPORT_HOST}:${LOAD_BALANCER_WEB_PORT}/testwebapp/" + if [ "$LB_TYPE" == "APACHE" ] ; then + TEST_APP_URL="http://${NODEPORT_HOST}:${LOAD_BALANCER_WEB_PORT}/weblogic/testwebapp/" + fi local CURL_RESPONSE_BODY="$TMP_DIR/testapp.response.body" - trace 'wait for test app to become available' + trace "wait for test app to become available on ${TEST_APP_URL}" + local max_count=30 local wait_time=6 local count=0 @@ -2471,6 +2482,10 @@ function test_suite_init { export LEASE_ID="${LEASE_ID}" + if [ -z "$LB_TYPE" ]; then + export LB_TYPE=TRAEFIK + fi + # The following customizable exports are currently only customized by WERCKER export IMAGE_TAG_OPERATOR=${IMAGE_TAG_OPERATOR:-`echo "test_${BRANCH_NAME}" | sed "s#/#_#g"`} export IMAGE_NAME_OPERATOR=${IMAGE_NAME_OPERATOR:-wlsldi-v2.docker.oraclecorp.com/weblogic-operator} From b3f9a0c99dd4479451386f4a5ad694a36d3ccd14 Mon Sep 17 00:00:00 2001 From: Russell Gold Date: Wed, 25 Apr 2018 12:19:43 -0400 Subject: [PATCH 064/186] Prepare for more unit tests --- .../java/oracle/kubernetes/TestUtils.java | 65 ++++++-- .../kubernetes/operator/WatcherTestBase.java | 20 +-- .../steps/DeleteIngressListStepTest.java | 2 +- .../AsyncCallTestSupport.java | 11 +- .../operator/work/InMemoryDatabase.java | 99 ++++++++++++ .../work/InMemoryDatabaseException.java | 14 ++ .../operator/work/InMemoryDatabaseTest.java | 142 ++++++++++++++++++ 7 files changed, 328 insertions(+), 25 deletions(-) rename operator/src/test/java/oracle/kubernetes/operator/{helpers => work}/AsyncCallTestSupport.java (97%) create mode 100644 operator/src/test/java/oracle/kubernetes/operator/work/InMemoryDatabase.java create mode 100644 operator/src/test/java/oracle/kubernetes/operator/work/InMemoryDatabaseException.java create mode 100644 operator/src/test/java/oracle/kubernetes/operator/work/InMemoryDatabaseTest.java diff --git a/operator/src/test/java/oracle/kubernetes/TestUtils.java b/operator/src/test/java/oracle/kubernetes/TestUtils.java index 41f451b7ec3..c3fcb878384 100644 --- a/operator/src/test/java/oracle/kubernetes/TestUtils.java +++ b/operator/src/test/java/oracle/kubernetes/TestUtils.java @@ -3,11 +3,6 @@ package oracle.kubernetes; -import com.meterware.simplestub.Memento; -import oracle.kubernetes.operator.logging.LoggingFactory; -import org.apache.commons.exec.CommandLine; -import org.apache.commons.exec.DefaultExecutor; - import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PrintStream; @@ -15,8 +10,18 @@ import java.util.List; import java.util.logging.ConsoleHandler; import java.util.logging.Handler; +import java.util.logging.LogRecord; import java.util.logging.Logger; +import com.meterware.simplestub.Memento; + +import oracle.kubernetes.operator.logging.LoggingFactory; + +import org.apache.commons.exec.CommandLine; +import org.apache.commons.exec.DefaultExecutor; + +import static com.meterware.simplestub.Stub.createStub; + public class TestUtils { private static Boolean kubernetesStatus; @@ -57,7 +62,7 @@ public static boolean isLinux() { * * @return a collection of the removed handlers */ - public static Memento silenceOperatorLogger() { + public static ExceptionFilteringMemento silenceOperatorLogger() { Logger logger = LoggingFactory.getLogger("Operator", "Operator").getUnderlyingLogger(); List savedHandlers = new ArrayList<>(); for (Handler handler : logger.getHandlers()) { @@ -69,7 +74,33 @@ public static Memento silenceOperatorLogger() { for (Handler handler : savedHandlers) logger.removeHandler(handler); - return new ConsoleHandlerMemento(logger, savedHandlers); + TestLogHandler testHandler = createStub(TestLogHandler.class); + logger.addHandler(testHandler); + + return new ConsoleHandlerMemento(logger, testHandler, savedHandlers); + } + + abstract static class TestLogHandler extends Handler { + private Throwable throwable; + private List ignoredExceptions = new ArrayList<>(); + + @Override + public void publish(LogRecord record) { + if (record.getThrown() != null && !ignoredExceptions.contains(record.getThrown())) + throwable = record.getThrown(); + } + + void throwLoggedThrowable() { + if (throwable == null) return; + + if (throwable instanceof Error) throw (Error) throwable; + if (throwable instanceof RuntimeException) throw (RuntimeException) throwable; + throw new RuntimeException(throwable); + } + + void ignoreLoggedException(Throwable t) { + ignoredExceptions.add(t); + } } /** @@ -102,18 +133,34 @@ public static void restoreConsoleHandlers(Logger logger, List savedHand } } - private static class ConsoleHandlerMemento implements Memento { + public interface ExceptionFilteringMemento extends Memento { + ExceptionFilteringMemento ignoringLoggedExceptions(Throwable... throwables); + } + + private static class ConsoleHandlerMemento implements ExceptionFilteringMemento { private Logger logger; + private TestLogHandler testHandler; private List savedHandlers; - ConsoleHandlerMemento(Logger logger, List savedHandlers) { + ConsoleHandlerMemento(Logger logger, TestLogHandler testHandler, List savedHandlers) { this.logger = logger; + this.testHandler = testHandler; this.savedHandlers = savedHandlers; } + @Override + public ExceptionFilteringMemento ignoringLoggedExceptions(Throwable... throwables) { + for (Throwable throwable : throwables) + testHandler.ignoreLoggedException(throwable); + return this; + } + @Override public void revert() { + logger.removeHandler(testHandler); restoreConsoleHandlers(logger, savedHandlers); + + testHandler.throwLoggedThrowable(); } @Override diff --git a/operator/src/test/java/oracle/kubernetes/operator/WatcherTestBase.java b/operator/src/test/java/oracle/kubernetes/operator/WatcherTestBase.java index 47f28c139df..59484c0ea4b 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/WatcherTestBase.java +++ b/operator/src/test/java/oracle/kubernetes/operator/WatcherTestBase.java @@ -3,18 +3,18 @@ package oracle.kubernetes.operator; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +import com.meterware.simplestub.Memento; + import io.kubernetes.client.models.V1ObjectMeta; import io.kubernetes.client.util.Watch; import oracle.kubernetes.TestUtils; import oracle.kubernetes.operator.builders.StubWatchFactory; import oracle.kubernetes.operator.builders.WatchEvent; -import com.meterware.simplestub.Memento; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; - import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -34,6 +34,7 @@ public abstract class WatcherTestBase implements StubWatchFactory.AllWatchesClos private static final int NEXT_RESOURCE_VERSION = 123456; private static final int INITIAL_RESOURCE_VERSION = 123; private static final String NAMESPACE = "testspace"; + private final RuntimeException hasNextException = new RuntimeException(Watcher.HAS_NEXT_EXCEPTION_MESSAGE); private List mementos = new ArrayList<>(); private List> callBacks = new ArrayList<>(); @@ -57,7 +58,7 @@ void recordCallBack(Watch.Response response) { @Before public void setUp() throws Exception { - mementos.add(TestUtils.silenceOperatorLogger()); + mementos.add(TestUtils.silenceOperatorLogger().ignoringLoggedExceptions(hasNextException)); mementos.add(StubWatchFactory.install()); StubWatchFactory.setListener(this); } @@ -107,7 +108,7 @@ private Watch.Response createHttpGoneErrorResponse(int nextResourceVersion) { @Test public void receivedEvents_areSentToListeners() throws Exception { Object object = createObjectWithMetaData(); - StubWatchFactory.addCallResponses((Watch.Response) createAddResponse(object), (Watch.Response) createModifyResponse(object)); + StubWatchFactory.addCallResponses(createAddResponse(object), (Watch.Response) createModifyResponse(object)); createAndRunWatcher(NAMESPACE, stopping, INITIAL_RESOURCE_VERSION); @@ -158,7 +159,7 @@ public void afterDelete_nextRequestSendsIncrementedResourceVersion() throws Exce @SuppressWarnings("unchecked") @Test public void afterExceptionDuringNext_closeWatchAndTryAgain() throws Exception { - StubWatchFactory.throwExceptionOnNext(new RuntimeException(Watcher.HAS_NEXT_EXCEPTION_MESSAGE)); + StubWatchFactory.throwExceptionOnNext(hasNextException); StubWatchFactory.addCallResponses(createAddResponse(createObjectWithMetaData())); createAndRunWatcher(NAMESPACE, stopping, INITIAL_RESOURCE_VERSION); @@ -166,6 +167,7 @@ public void afterExceptionDuringNext_closeWatchAndTryAgain() throws Exception { assertThat(StubWatchFactory.getNumCloseCalls(), equalTo(2)); } + @SuppressWarnings("SameParameterValue") private V1ObjectMeta createMetaData(String name, String namespace) { return new V1ObjectMeta().name(name).namespace(namespace).resourceVersion(getNextResourceVersion()); } diff --git a/operator/src/test/java/oracle/kubernetes/operator/steps/DeleteIngressListStepTest.java b/operator/src/test/java/oracle/kubernetes/operator/steps/DeleteIngressListStepTest.java index 40a55e7f653..94cb455a0b5 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/steps/DeleteIngressListStepTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/steps/DeleteIngressListStepTest.java @@ -13,7 +13,7 @@ import io.kubernetes.client.models.V1Status; import io.kubernetes.client.models.V1beta1Ingress; import oracle.kubernetes.TestUtils; -import oracle.kubernetes.operator.helpers.AsyncCallTestSupport; +import oracle.kubernetes.operator.work.AsyncCallTestSupport; import oracle.kubernetes.operator.work.TerminalStep; import org.junit.After; diff --git a/operator/src/test/java/oracle/kubernetes/operator/helpers/AsyncCallTestSupport.java b/operator/src/test/java/oracle/kubernetes/operator/work/AsyncCallTestSupport.java similarity index 97% rename from operator/src/test/java/oracle/kubernetes/operator/helpers/AsyncCallTestSupport.java rename to operator/src/test/java/oracle/kubernetes/operator/work/AsyncCallTestSupport.java index 72ee089879a..991cfc74890 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/helpers/AsyncCallTestSupport.java +++ b/operator/src/test/java/oracle/kubernetes/operator/work/AsyncCallTestSupport.java @@ -1,7 +1,7 @@ // Copyright 2018, Oracle Corporation and/or its affiliates. All rights reserved. // Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. -package oracle.kubernetes.operator.helpers; +package oracle.kubernetes.operator.work; import java.net.HttpURLConnection; import java.util.ArrayList; @@ -19,11 +19,10 @@ import oracle.kubernetes.operator.calls.CallFactory; import oracle.kubernetes.operator.calls.CallResponse; import oracle.kubernetes.operator.calls.RequestParams; -import oracle.kubernetes.operator.work.Component; -import oracle.kubernetes.operator.work.FiberTestSupport; -import oracle.kubernetes.operator.work.NextAction; -import oracle.kubernetes.operator.work.Packet; -import oracle.kubernetes.operator.work.Step; +import oracle.kubernetes.operator.helpers.AsyncRequestStepFactory; +import oracle.kubernetes.operator.helpers.CallBuilder; +import oracle.kubernetes.operator.helpers.ClientPool; +import oracle.kubernetes.operator.helpers.ResponseStep; import javax.annotation.Nonnull; diff --git a/operator/src/test/java/oracle/kubernetes/operator/work/InMemoryDatabase.java b/operator/src/test/java/oracle/kubernetes/operator/work/InMemoryDatabase.java new file mode 100644 index 00000000000..d781ad7b5d5 --- /dev/null +++ b/operator/src/test/java/oracle/kubernetes/operator/work/InMemoryDatabase.java @@ -0,0 +1,99 @@ +// Copyright 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.work; + +import java.lang.reflect.InvocationTargetException; +import java.net.HttpURLConnection; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import io.kubernetes.client.models.V1ObjectMeta; + +import javax.annotation.Nonnull; + +public abstract class InMemoryDatabase { + + private Map contents = new HashMap<>(); + + public void create(T item, Map keys) { + T t = contents.get(new DatabaseKey(keys, item)); + if (t != null) throw new InMemoryDatabaseException(HttpURLConnection.HTTP_CONFLICT, "Item already exists"); + + contents.put(new DatabaseKey(keys, item), item); + } + + void delete(Map keys) { + T removed = contents.remove(new DatabaseKey(keys)); + if (removed == null) throw new InMemoryDatabaseException(HttpURLConnection.HTTP_NOT_FOUND, "No such item"); + } + + @SuppressWarnings("unchecked") + L list(Map searchKeys) { + List foundItems = new ArrayList<>(); + for (DatabaseKey key : contents.keySet()) + if (key.matches(searchKeys)) foundItems.add(contents.get(key)); + + return createList(foundItems); + } + + T read(Map keys) { + T t = contents.get(new DatabaseKey(keys)); + if (t == null) throw new InMemoryDatabaseException(HttpURLConnection.HTTP_NOT_FOUND, "No such item"); + return t; + } + + @SuppressWarnings("unchecked") + abstract L createList(List items); + + void replace(T item, Map keys) { + DatabaseKey databaseKey = new DatabaseKey(keys, item); + T t = contents.get(databaseKey); + if (t == null) throw new InMemoryDatabaseException(HttpURLConnection.HTTP_NOT_FOUND, "No such item"); + + contents.put(databaseKey, item); + } + + + private static class DatabaseKey { + private Map keys; + + DatabaseKey(@Nonnull Map keys) { + this.keys = new HashMap<>(keys); + } + + DatabaseKey(@Nonnull Map keys, Object o) { + this(keys); + String name = getName(o); + if (name != null) this.keys.put("name", name); + } + + private String getName(Object o) { + try { + V1ObjectMeta meta = (V1ObjectMeta) o.getClass().getMethod("getMetadata").invoke(o); + return meta.getName(); + } catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { + return null; + } + } + + boolean matches(Map searchKeys) { + for (String key : searchKeys.keySet()) + if (!Objects.equals(searchKeys.get(key), keys.get(key))) return false; + return true; + } + + @Override + public boolean equals(Object o) { + return o == this || ((o instanceof DatabaseKey) && keys.equals(((DatabaseKey) o).keys)); + } + + @Override + public int hashCode() { + return Objects.hash(keys); + } + } +} diff --git a/operator/src/test/java/oracle/kubernetes/operator/work/InMemoryDatabaseException.java b/operator/src/test/java/oracle/kubernetes/operator/work/InMemoryDatabaseException.java new file mode 100644 index 00000000000..0513e98d9df --- /dev/null +++ b/operator/src/test/java/oracle/kubernetes/operator/work/InMemoryDatabaseException.java @@ -0,0 +1,14 @@ +package oracle.kubernetes.operator.work; + +public class InMemoryDatabaseException extends RuntimeException { + private int code; + + InMemoryDatabaseException(int code, String message) { + super(message); + this.code = code; + } + + public int getCode() { + return code; + } +} diff --git a/operator/src/test/java/oracle/kubernetes/operator/work/InMemoryDatabaseTest.java b/operator/src/test/java/oracle/kubernetes/operator/work/InMemoryDatabaseTest.java new file mode 100644 index 00000000000..633bf0490ab --- /dev/null +++ b/operator/src/test/java/oracle/kubernetes/operator/work/InMemoryDatabaseTest.java @@ -0,0 +1,142 @@ +// Copyright 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.work; + +import java.net.HttpURLConnection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import io.kubernetes.client.models.V1ObjectMeta; +import io.kubernetes.client.models.V1beta1Ingress; +import io.kubernetes.client.models.V1beta1IngressList; + +import org.junit.Test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.equalTo; +import static org.junit.Assert.fail; + +public class InMemoryDatabaseTest { + + private static final String NS1 = "namespace1"; + private static final String NS2 = "namespace2"; + private static final String NAME1 = "name1"; + private static final String NAME2 = "name2"; + + private InMemoryDatabase database = new InMemoryDatabase() { + @Override + V1beta1IngressList createList(List items) { + return new V1beta1IngressList().items(items); + } + }; + + @Test + public void whenItemAbsent_readThrowsException() throws Exception { + try { + database.read(keys().name(NAME1).namespace(NS1).map()); + fail("Should have thrown an InMemoryDatabaseException"); + } catch (InMemoryDatabaseException e) { + assertThat(e.getCode(), equalTo(HttpURLConnection.HTTP_NOT_FOUND)); + } + } + + @Test + public void whenItemPresent_createThrowsException() throws Exception { + createItem(NAME1, NS1); + + try { + database.create(new V1beta1Ingress().metadata(new V1ObjectMeta().namespace(NS1).name(NAME1)), keys().namespace(NS1).map()); + fail("Should have thrown an InMemoryDatabaseException"); + } catch (InMemoryDatabaseException e) { + assertThat(e.getCode(), equalTo(HttpURLConnection.HTTP_CONFLICT)); + } + } + + private V1beta1Ingress createItem(String name, String namespace) { + V1beta1Ingress item = new V1beta1Ingress().metadata(new V1ObjectMeta().namespace(namespace).name(name)); + database.create(item, keys().namespace(namespace).map()); + return item; + } + + @Test + public void afterItemCreated_canRetrieveIt() throws Exception { + V1beta1Ingress item = createItem(NAME1, NS1); + + assertThat(database.read(keys().name(NAME1).namespace(NS1).map()), equalTo(item)); + } + + @Test + public void whenItemAbsent_replaceThrowsException() throws Exception { + try { + database.replace(new V1beta1Ingress().metadata(new V1ObjectMeta().namespace(NS1).name(NAME1)), keys().namespace(NS1).map()); + fail("Should have thrown an InMemoryDatabaseException"); + } catch (InMemoryDatabaseException e) { + assertThat(e.getCode(), equalTo(HttpURLConnection.HTTP_NOT_FOUND)); + } + } + + @Test + public void afterReplaceItem_canRetrieveNewItem() throws Exception { + createItem(NAME1, NS1).kind("old item"); + + V1beta1Ingress replacement = new V1beta1Ingress().metadata(new V1ObjectMeta().namespace(NS1).name(NAME1)).kind("new item"); + database.replace(replacement, keys().namespace(NS1).map()); + + assertThat(database.read(keys().name(NAME1).namespace(NS1).map()), equalTo(replacement)); + } + + @Test(expected = InMemoryDatabaseException.class) + public void afterItemDeleted_cannotRetrieveIt() throws Exception { + createItem(NAME1, NS1); + database.delete(keys().name(NAME1).namespace(NS1).map()); + + database.read(keys().name(NAME1).namespace(NS1).map()); + } + + @Test + public void whenItemToDeletedAbsent_throwException() throws Exception { + try { + database.delete(keys().name(NAME1).namespace(NS1).map()); + fail("Should have thrown an InMemoryDatabaseException"); + } catch (InMemoryDatabaseException e) { + assertThat(e.getCode(), equalTo(HttpURLConnection.HTTP_NOT_FOUND)); + } + } + + @Test + public void afterItemsCreated_canListMatches() throws Exception { + V1beta1Ingress item1 = createItem(NAME1, NS1); + V1beta1Ingress item2 = createItem(NAME2, NS1); + V1beta1Ingress item3 = createItem(NAME1, NS2); + + assertThat(database.list(keys().namespace(NS1).map()).getItems(), containsInAnyOrder(item1, item2)); + assertThat(database.list(keys().name(NAME1).map()).getItems(), containsInAnyOrder(item1, item3)); + } + + private MapMaker keys() { + return new MapMaker(); + } + + static class MapMaker { + private Map keys = new HashMap<>(); + + public Map map() { + return keys; + } + + public MapMaker namespace(String namespace) { + keys.put("namespace", namespace); + return this; + } + + public MapMaker name(String name) { + keys.put("name", name); + return this; + } + } + + +} \ No newline at end of file From 189f6d05dad0bce5d8e17c9b1b27521f3ec50edd Mon Sep 17 00:00:00 2001 From: Simon Meng Date: Mon, 23 Apr 2018 04:34:54 -0700 Subject: [PATCH 065/186] simplify operator script maintenance --- .../operator/helpers/ConfigMapHelper.java | 407 +++--------------- .../operator/logging/MessageKeys.java | 1 + .../src/main/resources/Operator.properties | 1 + .../main/resources/scripts/livenessProbe.sh | 16 + .../src/main/resources/scripts/readState.sh | 21 + .../main/resources/scripts/readinessProbe.sh | 26 ++ .../main/resources/scripts/start-server.py | 68 +++ .../src/main/resources/scripts/startServer.sh | 124 ++++++ .../src/main/resources/scripts/stop-server.py | 40 ++ .../src/main/resources/scripts/stopServer.sh | 17 + 10 files changed, 365 insertions(+), 356 deletions(-) create mode 100755 operator/src/main/resources/scripts/livenessProbe.sh create mode 100755 operator/src/main/resources/scripts/readState.sh create mode 100755 operator/src/main/resources/scripts/readinessProbe.sh create mode 100755 operator/src/main/resources/scripts/start-server.py create mode 100755 operator/src/main/resources/scripts/startServer.sh create mode 100755 operator/src/main/resources/scripts/stop-server.py create mode 100755 operator/src/main/resources/scripts/stopServer.sh diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java index ef69c2bd4e4..2b82158db57 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java @@ -3,9 +3,21 @@ package oracle.kubernetes.operator.helpers; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.stream.Stream; import io.kubernetes.client.ApiException; import io.kubernetes.client.models.V1ConfigMap; @@ -25,282 +37,7 @@ public class ConfigMapHelper { private static final LoggingFacade LOGGER = LoggingFactory.getLogger("Operator", "Operator"); - private static final String START_SERVER_SHELL_SCRIPT = "#!/bin/bash\n" + - "\n" + - "domain_uid=$1\n" + - "server_name=$2\n" + - "domain_name=$3\n" + - "as_name=$4\n" + - "as_port=$5\n" + - "as_hostname=$1-$4\n" + - "\n" + - "echo \"debug arguments are $1 $2 $3 $4 $5\"\n" + - "\n" + - "nmProp=\"/u01/nodemanager/nodemanager.properties\"\n" + - "\n" + - "# TODO: parameterize shared home and domain name\n" + - "export DOMAIN_HOME=/shared/domain/$domain_name\n" + - "\n" + - "#\n" + - "# Create a folder\n" + - "# $1 - path of folder to create\n" + - "function createFolder {\n" + - " mkdir -m 777 -p $1\n" + - " if [ ! -d $1 ]; then\n" + - " fail \"Unable to create folder $1\"\n" + - " fi\n" + - "}\n" + - "\n" + - "# Function to create server specific scripts and properties (e.g startup.properties, etc)\n" + - "# $1 - Domain UID\n" + - "# $2 - Server Name\n" + - "# $3 - Domain Name\n" + - "# $4 - Admin Server Hostname (only passed for managed servers)\n" + - "# $5 - Admin Server port (only passed for managed servers)\n" + - "function createServerScriptsProperties() {\n" + - "\n" + - " # Create nodemanager home for the server\n" + - " srvr_nmdir=/u01/nodemanager\n" + - " createFolder ${srvr_nmdir}\n" + - " cp /shared/domain/$3/nodemanager/nodemanager.domains ${srvr_nmdir}\n" + - " cp /shared/domain/$3/bin/startNodeManager.sh ${srvr_nmdir}\n" + - "\n" + - " # Edit the start nodemanager script to use the home for the server\n" + - " sed -i -e \"s:/shared/domain/$3/nodemanager:/u01/nodemanager:g\" ${srvr_nmdir}/startNodeManager.sh\n" + - "\n" + - " # Create startup.properties file\n" + - " datadir=${DOMAIN_HOME}/servers/$2/data/nodemanager\n" + - " nmdir=${DOMAIN_HOME}/nodemgr_home\n" + - " stateFile=${datadir}/$2.state\n" + - " startProp=${datadir}/startup.properties\n" + - " if [ -f \"$startProp\" ]; then\n" + - " echo \"startup.properties already exists\"\n" + - " return 0\n" + - " fi\n" + - "\n" + - " createFolder ${datadir}\n" + - " echo \"# Server startup properties\" > ${startProp}\n" + - " echo \"AutoRestart=true\" >> ${startProp}\n" + - " if [ -n \"$4\" ]; then\n" + - " echo \"AdminURL=http\\://$4\\:$5\" >> ${startProp}\n" + - " fi\n" + - " echo \"RestartMax=2\" >> ${startProp}\n" + - " echo \"RotateLogOnStartup=false\" >> ${startProp}\n" + - " echo \"RotationType=bySize\" >> ${startProp}\n" + - " echo \"RotationTimeStart=00\\:00\" >> ${startProp}\n" + - " echo \"RotatedFileCount=100\" >> ${startProp}\n" + - " echo \"RestartDelaySeconds=0\" >> ${startProp}\n" + - " echo \"FileSizeKB=5000\" >> ${startProp}\n" + - " echo \"FileTimeSpanFactor=3600000\" >> ${startProp}\n" + - " echo \"RestartInterval=3600\" >> ${startProp}\n" + - " echo \"NumberOfFilesLimited=true\" >> ${startProp}\n" + - " echo \"FileTimeSpan=24\" >> ${startProp}\n" + - " echo \"NMHostName=$1-$2\" >> ${startProp}\n" + - "}\n" + - "\n" + - "# Check for stale state file and remove if found\"\n" + - "if [ -f \"$stateFile\" ]; then\n" + - " echo \"Removing stale file $stateFile\"\n" + - " rm ${stateFile}\n" + - "fi\n" + - "\n" + - "# Create nodemanager home directory that is local to the k8s node\n" + - "mkdir -p /u01/nodemanager\n" + - "cp ${DOMAIN_HOME}/nodemanager/* /u01/nodemanager/\n" + - "\n" + - "# Edit the nodemanager properties file to use the home for the server\n" + - "sed -i -e \"s:DomainsFile=.*:DomainsFile=/u01/nodemanager/nodemanager.domains:g\" /u01/nodemanager/nodemanager.properties\n" + - "sed -i -e \"s:NodeManagerHome=.*:NodeManagerHome=/u01/nodemanager:g\" /u01/nodemanager/nodemanager.properties\n" + - "sed -i -e \"s:ListenAddress=.*:ListenAddress=$1-$2:g\" /u01/nodemanager/nodemanager.properties\n" + - "sed -i -e \"s:LogFile=.*:LogFile=/shared/logs/nodemanager-$2.log:g\" /u01/nodemanager/nodemanager.properties\n" + - "\n" + - "export JAVA_PROPERTIES=\"-DLogFile=/shared/logs/nodemanager-$server_name.log -DNodeManagerHome=/u01/nodemanager\"\n" + - "export NODEMGR_HOME=\"/u01/nodemanager\"\n" + - "\n" + - "\n" + - "# Create startup.properties\n" + - "echo \"Create startup.properties\"\n" + - "if [ -n \"$4\" ]; then\n" + - " echo \"this is managed server\"\n" + - " createServerScriptsProperties $domain_uid $server_name $domain_name $as_hostname $as_port\n" + - "else\n" + - " echo \"this is admin server\"\n" + - " createServerScriptsProperties $domain_uid $server_name $domain_name\n" + - "fi\n" + - "\n" + - "echo \"Start the nodemanager\"\n" + - ". ${NODEMGR_HOME}/startNodeManager.sh &\n" + - "\n" + - "echo \"Allow the nodemanager some time to start before attempting to connect\"\n" + - "sleep 15\n" + - "echo \"Finished waiting for the nodemanager to start\"\n" + - "\n" + - "echo \"Update JVM arguments\"\n" + - "echo \"Arguments=${USER_MEM_ARGS} -XX\\:+UnlockExperimentalVMOptions -XX\\:+UseCGroupMemoryLimitForHeap ${JAVA_OPTIONS}\" >> ${startProp}\n" + - "\n" + - "admin_server_t3_url=\n" + - "if [ -n \"$4\" ]; then\n" + - " admin_server_t3_url=t3://$domain_uid-$as_name:$as_port\n" + - "fi\n" + - "\n" + - "echo \"Start the server\"\n" + - "wlst.sh -skipWLSModuleScanning /weblogic-operator/scripts/start-server.py $domain_uid $server_name $domain_name $admin_server_t3_url\n" + - "\n" + - "echo \"Wait indefinitely so that the Kubernetes pod does not exit and try to restart\"\n" + - "while true; do sleep 60; done\n"; - - private static final String START_SERVER_PYTHON_SCRIPT = "import sys;\n" + - "#\n" + - "# +++ Start of common code for reading domain secrets\n" + - "\n" + - "# Read username secret\n" + - "file = open('/weblogic-operator/secrets/username', 'r')\n" + - "admin_username = file.read()\n" + - "file.close()\n" + - "\n" + - "# Read password secret\n" + - "file = open('/weblogic-operator/secrets/password', 'r')\n" + - "admin_password = file.read()\n" + - "file.close()\n" + - "\n" + - "# +++ End of common code for reading domain secrets\n" + - "#\n" + - "domain_uid = sys.argv[1]\n" + - "server_name = sys.argv[2]\n" + - "domain_name = sys.argv[3]\n" + - "if (len(sys.argv) == 5):\n" + - " admin_server_url = sys.argv[4]\n" + - "else:\n" + - " admin_server_url = None\n" + - "\n" + - "domain_path='/shared/domain/%s' % domain_name\n" + - "\n" + - "print 'admin username is %s' % admin_username\n" + - "print 'domain path is %s' % domain_path\n" + - "print 'server name is %s' % server_name\n" + - "print 'admin server url is %s' % admin_server_url\n" + - "\n" + - "# Encrypt the admin username and password\n" + - "adminUsernameEncrypted=encrypt(admin_username, domain_path)\n" + - "adminPasswordEncrypted=encrypt(admin_password, domain_path)\n" + - "\n" + - "print 'Create boot.properties files for this server'\n" + - "\n" + - "# Define the folder path\n" + - "secdir='%s/servers/%s/security' % (domain_path, server_name)\n" + - "\n" + - "# Create the security folder (if it does not already exist)\n" + - "try:\n" + - " os.makedirs(secdir)\n" + - "except OSError:\n" + - " if not os.path.isdir(secdir):\n" + - " raise\n" + - "\n" + - "print 'writing boot.properties to %s/servers/%s/security/boot.properties' % (domain_path, server_name)\n" + - "\n" + - "bpFile=open('%s/servers/%s/security/boot.properties' % (domain_path, server_name), 'w+')\n" + - "bpFile.write(\"username=%s\\n\" % adminUsernameEncrypted)\n" + - "bpFile.write(\"password=%s\\n\" % adminPasswordEncrypted)\n" + - "bpFile.close()\n" + - "\n" + - "service_name = domain_uid + \"-\" + server_name\n" + - "\n" + - /* Commented out as we are not configuring machines for servers - "# Update node manager listen address\n" + - "if admin_server_url is not None:\n" + - " connect(admin_username, admin_password, admin_server_url)\n" + - " serverConfig()\n" + - " server=cmo.lookupServer(server_name)\n" + - " machineName=server.getMachine().getName()\n" + - " print 'Name of machine assigned to server %s is %s' % (server_name, machineName)\n" + - "\n" + - " if machineName is not None:\n" + - " print 'Updating listen address of machine %s' % machineName\n" + - " try:\n" + - " edit()\n" + - " startEdit(120000, 120000, 'true')\n" + - " cd('/')\n" + - " machine=cmo.lookupMachine(machineName)\n" + - " print 'Machine is %s' % machine\n" + - " nm=machine.getNodeManager()\n" + - " nm.setListenAddress(service_name)\n" + - " nm.setNMType('Plain')\n" + - " save()\n" + - " activate()\n" + - " print 'Updated listen address of machine %s to %s' % (machineName, service_name)\n" + - " except:\n" + - " cancelEdit('y')\n" + - " disconnect()\n" + - "\n" + - */ - "# Connect to nodemanager and start server\n" + - "try:\n" + - " nmConnect(admin_username, admin_password, service_name, '5556', domain_name, domain_path, 'plain')\n" + - " nmStart(server_name)\n" + - " nmDisconnect()\n" + - "except WLSTException, e:\n" + - " nmDisconnect()\n" + - " print e\n" + - "\n" + - "# Exit WLST\n" + - "exit()\n"; - - private static final String STOP_SERVER_SHELL_SCRIPT = "#!/bin/bash\n" + - "\n" + - "echo \"Stop the server\"\n" + - "\n" + - "wlst.sh -skipWLSModuleScanning /weblogic-operator/scripts/stop-server.py $1 $2 $3\n" + - "\n" + - "# Return status of 2 means failed to stop a server through the NodeManager.\n" + - "# Look to see if there is a server process that can be killed.\n" + - "if [ $? -eq 2 ]; then\n" + - " pid=$(jps -v | grep '[D]weblogic.Name=$2' | awk '{print $1}')\n" + - " if [ ! -z $pid ]; then\n" + - " echo \"Killing the server process $pid\"\n" + - " kill -15 $pid\n" + - " fi\n" + - "fi\n" + - "\n"; - - private static final String STOP_SERVER_PYTHON_SCRIPT = "#\n" + - "# +++ Start of common code for reading domain secrets\n" + - "\n" + - "# Read username secret\n" + - "file = open('/weblogic-operator/secrets/username', 'r')\n" + - "admin_username = file.read()\n" + - "file.close()\n" + - "\n" + - "# Read password secret\n" + - "file = open('/weblogic-operator/secrets/password', 'r')\n" + - "admin_password = file.read()\n" + - "file.close()\n" + - "\n" + - "# +++ End of common code for reading domain secrets\n" + - "#\n" + - "domain_uid = sys.argv[1]\n" + - "server_name = sys.argv[2]\n" + - "domain_name = sys.argv[3]\n" + - "\n" + - "service_name = domain_uid + \"-\" + server_name\n" + - "domain_path='/shared/domain/%s' % domain_name\n" + - "\n" + - "# Connect to nodemanager and stop server\n" + - "try:\n" + - " nmConnect(admin_username, admin_password, service_name, '5556', domain_name, domain_path, 'plain')\n" + - "except:\n" + - " print('Failed to connect to the NodeManager')\n" + - " exit(exitcode=2)\n" + - "\n" + - "# Kill the server\n" + - "try:\n" + - " nmKill(server_name)\n" + - "except:\n" + - " print('Connected to the NodeManager, but failed to stop the server')\n" + - " exit(exitcode=2)\n" + - "\n" + - "# Exit WLST\n" + - "nmDisconnect()\n" + - "exit()\n"; + private static final String SCRIPT_LOCATION = "/scripts"; private ConfigMapHelper() {} @@ -413,89 +150,47 @@ protected V1ConfigMap computeDomainConfigMap() { metadata.setLabels(labels); cm.setMetadata(metadata); + cm.setData(loadScripts()); - Map data = new HashMap<>(); - - data.put("livenessProbe.sh", - "#!/bin/bash\n" + - "# Kubernetes periodically calls this liveness probe script to determine whether\n" + - "# the pod should be restarted. The script checks a WebLogic Server state file which\n" + - "# is updated by the node manager.\n" + - "DN=${DOMAIN_NAME:-$1}\n" + - "SN=${SERVER_NAME:-$2}\n" + - "STATEFILE=/shared/domain/${DN}/servers/${SN}/data/nodemanager/${SN}.state\n" + - "if [ `jps -l | grep -c \" weblogic.NodeManager\"` -eq 0 ]; then\n" + - " echo \"Error: WebLogic NodeManager process not found.\"\n" + - " exit 1\n" + - "fi\n" + - "if [ -f ${STATEFILE} ] && [ `grep -c \"" + WebLogicConstants.FAILED_NOT_RESTARTABLE_STATE + "\" ${STATEFILE}` -eq 1 ]; then\n" + - " echo \"Error: WebLogic Server state is " + WebLogicConstants.FAILED_NOT_RESTARTABLE_STATE + ".\"\n" + - " exit 1\n" + - "fi\n" + - "exit 0"); - - data.put("readinessProbe.sh", - "#!/bin/bash\n" + - "\n" + - "# Kubernetes periodically calls this readiness probe script to determine whether\n" + - "# the pod should be included in load balancing. The script checks a WebLogic Server state\n" + - "# file which is updated by the node manager.\n" + - "\n" + - "DN=${DOMAIN_NAME:-$1}\n" + - "SN=${SERVER_NAME:-$2}\n" + - "STATEFILE=/shared/domain/${DN}/servers/${SN}/data/nodemanager/${SN}.state\n" + - "\n" + - "if [ `jps -l | grep -c \" weblogic.NodeManager\"` -eq 0 ]; then\n" + - " echo \"Error: WebLogic NodeManager process not found.\"\n" + - " exit 1\n" + - "fi\n" + - "\n" + - "if [ ! -f ${STATEFILE} ]; then\n" + - " echo \"Error: WebLogic Server state file not found.\"\n" + - " exit 2\n" + - "fi\n" + - "\n" + - "state=$(cat ${STATEFILE} | cut -f 1 -d ':')\n" + - "if [ \"$state\" != \"" + WebLogicConstants.RUNNING_STATE + "\" ]; then\n" + - " echo \"" + WebLogicConstants.READINESS_PROBE_NOT_READY_STATE + "${state}\"\n" + - " exit 3\n" + - "fi\n" + - "exit 0"); - - data.put("startServer.sh", START_SERVER_SHELL_SCRIPT); - - data.put("start-server.py", START_SERVER_PYTHON_SCRIPT); - - data.put("stopServer.sh", STOP_SERVER_SHELL_SCRIPT); - - data.put("stop-server.py", STOP_SERVER_PYTHON_SCRIPT); - - data.put("readState.sh", - "#!/bin/bash\n" + - "\n" + - "# Reads the current state of a server. The script checks a WebLogic Server state\n" + - "# file which is updated by the node manager.\n" + - "\n" + - "DN=${DOMAIN_NAME:-$1}\n" + - "SN=${SERVER_NAME:-$2}\n" + - "STATEFILE=/shared/domain/${DN}/servers/${SN}/data/nodemanager/${SN}.state\n" + - "\n" + - "if [ `jps -l | grep -c \" weblogic.NodeManager\"` -eq 0 ]; then\n" + - " echo \"Error: WebLogic NodeManager process not found.\"\n" + - " exit 1\n" + - "fi\n" + - "\n" + - "if [ ! -f ${STATEFILE} ]; then\n" + - " echo \"Error: WebLogic Server state file not found.\"\n" + - " exit 2\n" + - "fi\n" + - "\n" + - "cat ${STATEFILE} | cut -f 1 -d ':'\n" + - "exit 0"); + return cm; + } - cm.setData(data); + private Map loadScripts() { + URI uri = null; + try { + uri = getClass().getResource(SCRIPT_LOCATION).toURI(); + } catch (URISyntaxException e) { + LOGGER.warning(MessageKeys.EXCEPTION, e); + throw new RuntimeException(e); + } + + try { + FileSystem fileSystem = FileSystems.newFileSystem(uri, Collections.emptyMap()); + Stream walk = Files.walk(fileSystem.getPath(SCRIPT_LOCATION), 1); + Map data = new HashMap<>(); + for (Iterator it = walk.iterator(); it.hasNext();) { + Path script = it.next(); + String scriptName = script.toString(); + if (!SCRIPT_LOCATION.equals(scriptName)) { + data.put(script.getFileName().toString(), readScript(getClass().getResourceAsStream(scriptName))); + } + } + LOGGER.info(MessageKeys.SCRIPT_LOADED, domainNamespace); + return data; + } catch (IOException e) { + LOGGER.warning(MessageKeys.EXCEPTION, e); + throw new RuntimeException(e); + } + } - return cm; + private String readScript(InputStream inputStream) throws IOException { + ByteArrayOutputStream result = new ByteArrayOutputStream(); + byte[] buffer = new byte[1024]; + int length; + while ((length = inputStream.read(buffer)) != -1) { + result.write(buffer, 0, length); + } + return result.toString(); } } diff --git a/operator/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java b/operator/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java index 93e1577cc88..18d35ec8e43 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java +++ b/operator/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java @@ -141,4 +141,5 @@ private MessageKeys() {} public static final String WLS_UPDATE_CLUSTER_SIZE_INVALID_CLUSTER = "WLSKO-0131"; public static final String WLS_CLUSTER_SIZE_UPDATED = "WLSKO-0132"; public static final String WLS_SERVER_TEMPLATE_NOT_FOUND = "WLSKO-0133"; + public static final String SCRIPT_LOADED = "WLSKO-0134"; } diff --git a/operator/src/main/resources/Operator.properties b/operator/src/main/resources/Operator.properties index 50b259f4017..7c4a86f11ea 100644 --- a/operator/src/main/resources/Operator.properties +++ b/operator/src/main/resources/Operator.properties @@ -132,3 +132,4 @@ WLSKO-0130=Failed to update WebLogic dynamic cluster size for cluster {0} within WLSKO-0131=Failed to update WebLogic dynamic cluster size for cluster {0}. Cluster is not a dynamic cluster WLSKO-0132=Updated cluster size for WebLogic dynamic cluster {0} to {1}. Time taken {2} ms WLSKO-0133=Cannot find WebLogic server template with name {0} which is referenced by WebLogic cluster {1} +WLSKO-0134=Loading scripts into domain control config map for namespace: {0} diff --git a/operator/src/main/resources/scripts/livenessProbe.sh b/operator/src/main/resources/scripts/livenessProbe.sh new file mode 100755 index 00000000000..332a41141de --- /dev/null +++ b/operator/src/main/resources/scripts/livenessProbe.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# Kubernetes periodically calls this liveness probe script to determine whether +# the pod should be restarted. The script checks a WebLogic Server state file which +# is updated by the node manager. +DN=${DOMAIN_NAME:-$1} +SN=${SERVER_NAME:-$2} +STATEFILE=/shared/domain/${DN}/servers/${SN}/data/nodemanager/${SN}.state +if [ `jps -l | grep -c " weblogic.NodeManager"` -eq 0 ]; then + echo "Error: WebLogic NodeManager process not found." + exit 1 +fi +if [ -f ${STATEFILE} ] && [ `grep -c "FAILED_NOT_RESTARTABLE" ${STATEFILE}` -eq 1 ]; then + echo "Error: WebLogic Server state is FAILED_NOT_RESTARTABLE." + exit 1 +fi +exit 0 diff --git a/operator/src/main/resources/scripts/readState.sh b/operator/src/main/resources/scripts/readState.sh new file mode 100755 index 00000000000..2d46058c0dd --- /dev/null +++ b/operator/src/main/resources/scripts/readState.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# Reads the current state of a server. The script checks a WebLogic Server state +# file which is updated by the node manager. + +DN=${DOMAIN_NAME:-$1} +SN=${SERVER_NAME:-$2} +STATEFILE=/shared/domain/${DN}/servers/${SN}/data/nodemanager/${SN}.state + +if [ `jps -l | grep -c " weblogic.NodeManager"` -eq 0 ]; then + echo "Error: WebLogic NodeManager process not found." + exit 1 +fi + +if [ ! -f ${STATEFILE} ]; then + echo "Error: WebLogic Server state file not found." + exit 2 +fi + +cat ${STATEFILE} | cut -f 1 -d ':' +exit 0 diff --git a/operator/src/main/resources/scripts/readinessProbe.sh b/operator/src/main/resources/scripts/readinessProbe.sh new file mode 100755 index 00000000000..beadb18e2d9 --- /dev/null +++ b/operator/src/main/resources/scripts/readinessProbe.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# Kubernetes periodically calls this readiness probe script to determine whether +# the pod should be included in load balancing. The script checks a WebLogic Server state +# file which is updated by the node manager. + +DN=${DOMAIN_NAME:-$1} +SN=${SERVER_NAME:-$2} +STATEFILE=/shared/domain/${DN}/servers/${SN}/data/nodemanager/${SN}.state + +if [ `jps -l | grep -c " weblogic.NodeManager"` -eq 0 ]; then + echo "Error: WebLogic NodeManager process not found." + exit 1 +fi + +if [ ! -f ${STATEFILE} ]; then + echo "Error: WebLogic Server state file not found." + exit 2 +fi + +state=$(cat ${STATEFILE} | cut -f 1 -d ':') +if [ "$state" != "RUNNING" ]; then + echo "Not ready: WebLogic Server state: ${state}" + exit 3 +fi +exit 0 diff --git a/operator/src/main/resources/scripts/start-server.py b/operator/src/main/resources/scripts/start-server.py new file mode 100755 index 00000000000..4671a74efb9 --- /dev/null +++ b/operator/src/main/resources/scripts/start-server.py @@ -0,0 +1,68 @@ +import sys; +# +# +++ Start of common code for reading domain secrets + +# Read username secret +file = open('/weblogic-operator/secrets/username', 'r') +admin_username = file.read() +file.close() + +# Read password secret +file = open('/weblogic-operator/secrets/password', 'r') +admin_password = file.read() +file.close() + +# +++ End of common code for reading domain secrets +# +domain_uid = sys.argv[1] +server_name = sys.argv[2] +domain_name = sys.argv[3] +if (len(sys.argv) == 5): + admin_server_url = sys.argv[4] +else: + admin_server_url = None + +domain_path='/shared/domain/%s' % domain_name + +print 'admin username is %s' % admin_username +print 'domain path is %s' % domain_path +print 'server name is %s' % server_name +print 'admin server url is %s' % admin_server_url + +# Encrypt the admin username and password +adminUsernameEncrypted=encrypt(admin_username, domain_path) +adminPasswordEncrypted=encrypt(admin_password, domain_path) + +print 'Create boot.properties files for this server' + +# Define the folder path +secdir='%s/servers/%s/security' % (domain_path, server_name) + +# Create the security folder (if it does not already exist) +try: + os.makedirs(secdir) +except OSError: + if not os.path.isdir(secdir): + raise + +print 'writing boot.properties to %s/servers/%s/security/boot.properties' % (domain_path, server_name) + +bpFile=open('%s/servers/%s/security/boot.properties' % (domain_path, server_name), 'w+') +bpFile.write("username=%s\n" % adminUsernameEncrypted) +bpFile.write("password=%s\n" % adminPasswordEncrypted) +bpFile.close() + +service_name = domain_uid + "-" + server_name + +# Connect to nodemanager and start server +try: + nmConnect(admin_username, admin_password, service_name, '5556', domain_name, domain_path, 'plain') + nmStart(server_name) + nmDisconnect() +except WLSTException, e: + nmDisconnect() + print e + +# Exit WLST +exit() + diff --git a/operator/src/main/resources/scripts/startServer.sh b/operator/src/main/resources/scripts/startServer.sh new file mode 100755 index 00000000000..4018eae1546 --- /dev/null +++ b/operator/src/main/resources/scripts/startServer.sh @@ -0,0 +1,124 @@ +#!/bin/bash + +domain_uid=$1 +server_name=$2 +domain_name=$3 +as_name=$4 +as_port=$5 +as_hostname=$1-$4 + +echo "debug arguments are $1 $2 $3 $4 $5" + +nmProp="/u01/nodemanager/nodemanager.properties" + +# TODO: parameterize shared home and domain name +export DOMAIN_HOME=/shared/domain/$domain_name + +# +# Create a folder +# $1 - path of folder to create +function createFolder { + mkdir -m 777 -p $1 + if [ ! -d $1 ]; then + fail "Unable to create folder $1" + fi +} + +# Function to create server specific scripts and properties (e.g startup.properties, etc) +# $1 - Domain UID +# $2 - Server Name +# $3 - Domain Name +# $4 - Admin Server Hostname (only passed for managed servers) +# $5 - Admin Server port (only passed for managed servers) +function createServerScriptsProperties() { + + # Create nodemanager home for the server + srvr_nmdir=/u01/nodemanager + createFolder ${srvr_nmdir} + cp /shared/domain/$3/nodemanager/nodemanager.domains ${srvr_nmdir} + cp /shared/domain/$3/bin/startNodeManager.sh ${srvr_nmdir} + + # Edit the start nodemanager script to use the home for the server + sed -i -e "s:/shared/domain/$3/nodemanager:/u01/nodemanager:g" ${srvr_nmdir}/startNodeManager.sh + + # Create startup.properties file + datadir=${DOMAIN_HOME}/servers/$2/data/nodemanager + nmdir=${DOMAIN_HOME}/nodemgr_home + stateFile=${datadir}/$2.state + startProp=${datadir}/startup.properties + if [ -f "$startProp" ]; then + echo "startup.properties already exists" + return 0 + fi + + createFolder ${datadir} + echo "# Server startup properties" > ${startProp} + echo "AutoRestart=true" >> ${startProp} + if [ -n "$4" ]; then + echo "AdminURL=http\://$4\:$5" >> ${startProp} + fi + echo "RestartMax=2" >> ${startProp} + echo "RotateLogOnStartup=false" >> ${startProp} + echo "RotationType=bySize" >> ${startProp} + echo "RotationTimeStart=00\:00" >> ${startProp} + echo "RotatedFileCount=100" >> ${startProp} + echo "RestartDelaySeconds=0" >> ${startProp} + echo "FileSizeKB=5000" >> ${startProp} + echo "FileTimeSpanFactor=3600000" >> ${startProp} + echo "RestartInterval=3600" >> ${startProp} + echo "NumberOfFilesLimited=true" >> ${startProp} + echo "FileTimeSpan=24" >> ${startProp} + echo "NMHostName=$1-$2" >> ${startProp} +} + +# Check for stale state file and remove if found" +if [ -f "$stateFile" ]; then + echo "Removing stale file $stateFile" + rm ${stateFile} +fi + +# Create nodemanager home directory that is local to the k8s node +mkdir -p /u01/nodemanager +cp ${DOMAIN_HOME}/nodemanager/* /u01/nodemanager/ + +# Edit the nodemanager properties file to use the home for the server +sed -i -e "s:DomainsFile=.*:DomainsFile=/u01/nodemanager/nodemanager.domains:g" /u01/nodemanager/nodemanager.properties +sed -i -e "s:NodeManagerHome=.*:NodeManagerHome=/u01/nodemanager:g" /u01/nodemanager/nodemanager.properties +sed -i -e "s:ListenAddress=.*:ListenAddress=$1-$2:g" /u01/nodemanager/nodemanager.properties +sed -i -e "s:LogFile=.*:LogFile=/shared/logs/nodemanager-$2.log:g" /u01/nodemanager/nodemanager.properties + +export JAVA_PROPERTIES="-DLogFile=/shared/logs/nodemanager-$server_name.log -DNodeManagerHome=/u01/nodemanager" +export NODEMGR_HOME="/u01/nodemanager" + + +# Create startup.properties +echo "Create startup.properties" +if [ -n "$4" ]; then + echo "this is managed server" + createServerScriptsProperties $domain_uid $server_name $domain_name $as_hostname $as_port +else + echo "this is admin server" + createServerScriptsProperties $domain_uid $server_name $domain_name +fi + +echo "Start the nodemanager" +. ${NODEMGR_HOME}/startNodeManager.sh & + +echo "Allow the nodemanager some time to start before attempting to connect" +sleep 15 +echo "Finished waiting for the nodemanager to start" + +echo "Update JVM arguments" +echo "Arguments=${USER_MEM_ARGS} -XX\:+UnlockExperimentalVMOptions -XX\:+UseCGroupMemoryLimitForHeap ${JAVA_OPTIONS}" >> ${startProp} + +admin_server_t3_url= +if [ -n "$4" ]; then + admin_server_t3_url=t3://$domain_uid-$as_name:$as_port +fi + +echo "Start the server" +wlst.sh -skipWLSModuleScanning /weblogic-operator/scripts/start-server.py $domain_uid $server_name $domain_name $admin_server_t3_url + +echo "Wait indefinitely so that the Kubernetes pod does not exit and try to restart" +while true; do sleep 60; done + diff --git a/operator/src/main/resources/scripts/stop-server.py b/operator/src/main/resources/scripts/stop-server.py new file mode 100755 index 00000000000..8e69ba384c5 --- /dev/null +++ b/operator/src/main/resources/scripts/stop-server.py @@ -0,0 +1,40 @@ +# +# +++ Start of common code for reading domain secrets + +# Read username secret +file = open('/weblogic-operator/secrets/username', 'r') +admin_username = file.read() +file.close() + +# Read password secret +file = open('/weblogic-operator/secrets/password', 'r') +admin_password = file.read() +file.close() + +# +++ End of common code for reading domain secrets +# +domain_uid = sys.argv[1] +server_name = sys.argv[2] +domain_name = sys.argv[3] + +service_name = domain_uid + "-" + server_name +domain_path='/shared/domain/%s' % domain_name + +# Connect to nodemanager and stop server +try: + nmConnect(admin_username, admin_password, service_name, '5556', domain_name, domain_path, 'plain') +except: + print('Failed to connect to the NodeManager') + exit(exitcode=2) + +# Kill the server +try: + nmKill(server_name) +except: + print('Connected to the NodeManager, but failed to stop the server') + exit(exitcode=2) + +# Exit WLST +nmDisconnect() +exit() + diff --git a/operator/src/main/resources/scripts/stopServer.sh b/operator/src/main/resources/scripts/stopServer.sh new file mode 100755 index 00000000000..5fb32ed2573 --- /dev/null +++ b/operator/src/main/resources/scripts/stopServer.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +echo "Stop the server" + +wlst.sh -skipWLSModuleScanning /weblogic-operator/scripts/stop-server.py $1 $2 $3 + +# Return status of 2 means failed to stop a server through the NodeManager. +# Look to see if there is a server process that can be killed. +if [ $? -eq 2 ]; then + pid=$(jps -v | grep '[D]weblogic.Name=$2' | awk '{print $1}') + if [ ! -z $pid ]; then + echo "Killing the server process $pid" + kill -15 $pid + fi +fi + + From 98e0b5c6a70bd5a7988aa05cd0689611e915ad4b Mon Sep 17 00:00:00 2001 From: Simon Meng Date: Mon, 23 Apr 2018 07:47:26 -0700 Subject: [PATCH 066/186] close file system after loading scripts --- .../operator/helpers/ConfigMapHelper.java | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java index 2b82158db57..612ad12680a 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java @@ -155,7 +155,7 @@ protected V1ConfigMap computeDomainConfigMap() { return cm; } - private Map loadScripts() { + private synchronized Map loadScripts() { URI uri = null; try { uri = getClass().getResource(SCRIPT_LOCATION).toURI(); @@ -164,15 +164,14 @@ private Map loadScripts() { throw new RuntimeException(e); } - try { - FileSystem fileSystem = FileSystems.newFileSystem(uri, Collections.emptyMap()); + try (FileSystem fileSystem = FileSystems.newFileSystem(uri, Collections.emptyMap())) { Stream walk = Files.walk(fileSystem.getPath(SCRIPT_LOCATION), 1); Map data = new HashMap<>(); for (Iterator it = walk.iterator(); it.hasNext();) { Path script = it.next(); String scriptName = script.toString(); if (!SCRIPT_LOCATION.equals(scriptName)) { - data.put(script.getFileName().toString(), readScript(getClass().getResourceAsStream(scriptName))); + data.put(script.getFileName().toString(), readScript(scriptName)); } } LOGGER.info(MessageKeys.SCRIPT_LOADED, domainNamespace); @@ -183,14 +182,18 @@ private Map loadScripts() { } } - private String readScript(InputStream inputStream) throws IOException { - ByteArrayOutputStream result = new ByteArrayOutputStream(); - byte[] buffer = new byte[1024]; - int length; - while ((length = inputStream.read(buffer)) != -1) { - result.write(buffer, 0, length); + private String readScript(String scriptName) throws IOException { + try ( + InputStream inputStream = getClass().getResourceAsStream(scriptName); + ByteArrayOutputStream result = new ByteArrayOutputStream() + ) { + byte[] buffer = new byte[1024]; + int length; + while ((length = inputStream.read(buffer)) != -1) { + result.write(buffer, 0, length); + } + return result.toString(); } - return result.toString(); } } From 7e242cc6c4d82c91bfea98480fd9ad00530a9b44 Mon Sep 17 00:00:00 2001 From: Ryan Eberhard Date: Wed, 25 Apr 2018 12:24:43 -0400 Subject: [PATCH 067/186] Update to use streaming API and support Jars --- .../operator/helpers/ConfigMapHelper.java | 58 +++++++++---------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java index 612ad12680a..a7e0d2fe95e 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java @@ -3,20 +3,20 @@ package oracle.kubernetes.operator.helpers; -import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.InputStream; import java.net.URI; import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; import java.nio.file.FileSystem; import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Collections; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import java.util.stream.Stream; import io.kubernetes.client.ApiException; @@ -25,7 +25,6 @@ import oracle.kubernetes.operator.KubernetesConstants; import oracle.kubernetes.operator.LabelConstants; import oracle.kubernetes.operator.ProcessingConstants; -import oracle.kubernetes.operator.WebLogicConstants; import oracle.kubernetes.operator.logging.LoggingFacade; import oracle.kubernetes.operator.logging.LoggingFactory; import oracle.kubernetes.operator.logging.MessageKeys; @@ -37,7 +36,8 @@ public class ConfigMapHelper { private static final LoggingFacade LOGGER = LoggingFactory.getLogger("Operator", "Operator"); - private static final String SCRIPT_LOCATION = "/scripts"; + private static final String SCRIPTS = "scripts"; + private static final String SCRIPT_LOCATION = "/" + SCRIPTS; private ConfigMapHelper() {} @@ -163,38 +163,38 @@ private synchronized Map loadScripts() { LOGGER.warning(MessageKeys.EXCEPTION, e); throw new RuntimeException(e); } - - try (FileSystem fileSystem = FileSystems.newFileSystem(uri, Collections.emptyMap())) { - Stream walk = Files.walk(fileSystem.getPath(SCRIPT_LOCATION), 1); - Map data = new HashMap<>(); - for (Iterator it = walk.iterator(); it.hasNext();) { - Path script = it.next(); - String scriptName = script.toString(); - if (!SCRIPT_LOCATION.equals(scriptName)) { - data.put(script.getFileName().toString(), readScript(scriptName)); + + try { + if ("jar".equals(uri.getScheme())) { + try (FileSystem fileSystem = FileSystems.newFileSystem(uri, Collections.emptyMap())) { + return walkScriptsPath(fileSystem.getPath(SCRIPTS)); } + } else { + return walkScriptsPath(Paths.get(uri)); } - LOGGER.info(MessageKeys.SCRIPT_LOADED, domainNamespace); - return data; } catch (IOException e) { LOGGER.warning(MessageKeys.EXCEPTION, e); throw new RuntimeException(e); } } - - private String readScript(String scriptName) throws IOException { - try ( - InputStream inputStream = getClass().getResourceAsStream(scriptName); - ByteArrayOutputStream result = new ByteArrayOutputStream() - ) { - byte[] buffer = new byte[1024]; - int length; - while ((length = inputStream.read(buffer)) != -1) { - result.write(buffer, 0, length); - } - return result.toString(); + + private Map walkScriptsPath(Path scriptsDir) throws IOException { + try (Stream walk = Files.walk(scriptsDir, 1)) { + Map data = walk.filter(i -> !Files.isDirectory(i)).collect(Collectors.toMap( + i -> i.getFileName().toString(), + i -> new String(read(i), StandardCharsets.UTF_8))); + LOGGER.info(MessageKeys.SCRIPT_LOADED, domainNamespace); + return data; } } + + private byte[] read(Path path) { + try { + return Files.readAllBytes(path); + } catch (IOException io) { + LOGGER.warning(MessageKeys.EXCEPTION, io); + } + return null; + } } - } From db69b93b7f7c890452616872c591379380eee7f1 Mon Sep 17 00:00:00 2001 From: Dongbo Xiao Date: Wed, 25 Apr 2018 09:51:27 -0700 Subject: [PATCH 068/186] Minor modification to apache k8s service name Signed-off-by: Dongbo Xiao --- kubernetes/internal/create-weblogic-domain.sh | 3 +-- .../weblogic-domain-apache-template.yaml | 18 +----------------- 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/kubernetes/internal/create-weblogic-domain.sh b/kubernetes/internal/create-weblogic-domain.sh index 5634619c81e..7001ad75c0f 100755 --- a/kubernetes/internal/create-weblogic-domain.sh +++ b/kubernetes/internal/create-weblogic-domain.sh @@ -709,7 +709,6 @@ function setupTraefikLoadBalancer { function setupApacheLoadBalancer { apacheName="${domainUID}-apache-webtier" - apacheServiceName="${domainUID}-external-apache-webtier-service" echo Deploying apache kubectl apply -f ${apacheOutput} @@ -727,7 +726,7 @@ function setupApacheLoadBalancer { fi echo Checking apache service - TSVC=`kubectl get services -n ${namespace} | grep ${apacheServiceName} | wc | awk ' { print $1; } '` + TSVC=`kubectl get services -n ${namespace} | grep ${apacheName} | wc | awk ' { print $1; } '` if [ "$TSVC" != "1" ]; then fail "The service ${apacheServiceName} was not created" fi diff --git a/kubernetes/internal/weblogic-domain-apache-template.yaml b/kubernetes/internal/weblogic-domain-apache-template.yaml index f6f821f732a..1d4b0689d59 100755 --- a/kubernetes/internal/weblogic-domain-apache-template.yaml +++ b/kubernetes/internal/weblogic-domain-apache-template.yaml @@ -75,7 +75,7 @@ spec: apiVersion: v1 kind: Service metadata: - name: %DOMAIN_UID%-external-apache-webtier-service + name: %DOMAIN_UID%-apache-webtier namespace: %NAMESPACE% labels: weblogic.domainUID: %DOMAIN_UID% @@ -91,19 +91,3 @@ spec: nodePort: %LOAD_BALANCER_WEB_PORT% name: rest-https -# --- -# apiVersion: v1 -# kind: Service -# metadata: -# name: %DOMAIN_UID%-internal-apache-webtier-service -# namespace: %NAMESPACE% -# labels: -# weblogic.domainUID: %DOMAIN_UID% -# weblogic.domainName: %DOMAIN_NAME% -# spec: -# type: ClusterIP -# selector: -# app: %DOMAIN_UID%-apache-webtier -# ports: -# - port: %LOAD_BALANCER_WEB_PORT_INTERNAL% -# name: rest-httpe700 From 7ee9cb300db1748910c05530575da90e576160c6 Mon Sep 17 00:00:00 2001 From: Dongbo Xiao Date: Wed, 25 Apr 2018 10:48:39 -0700 Subject: [PATCH 069/186] Add unit test Signed-off-by: Dongbo Xiao --- .../operator/create/CreateDomainInputs.java | 11 +++++++++++ .../operator/create/CreateDomainInputsFileTest.java | 2 ++ .../create/CreateDomainInputsValidationTest.java | 11 +++++++++++ 3 files changed, 24 insertions(+) diff --git a/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputs.java b/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputs.java index 33b0ed6ab59..1b5927fd393 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputs.java +++ b/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputs.java @@ -29,6 +29,7 @@ public class CreateDomainInputs { public static final String LOAD_BALANCER_NONE = "NONE"; public static final String LOAD_BALANCER_TRAEFIK = "TRAEFIK"; + public static final String LOAD_BALANCER_APACHE = "APACHE"; public static final String STORAGE_TYPE_HOST_PATH = "HOST_PATH"; public static final String STORAGE_TYPE_NFS = "NFS"; public static final String STORAGE_RECLAIM_POLICY_RETAIN = "Retain"; @@ -72,6 +73,8 @@ public class CreateDomainInputs { private String loadBalancer = ""; private String loadBalancerWebPort = ""; private String loadBalancerDashboardPort = ""; + private String loadBalancerVolumePath = ""; + private String loadBalancerAppPrepath = ""; private String javaOptions = ""; public static CreateDomainInputs newInputs() throws Exception { @@ -487,6 +490,14 @@ public void setLoadBalancerDashboardPort(String loadBalancerDashboardPort) { this.loadBalancerDashboardPort = convertNullToEmptyString(loadBalancerDashboardPort); } + public String getLoadBalancerVolumePath() { + return loadBalancerVolumePath; + } + + public void setLoadBalancerVolumePath(String loadBalancerVolumePath) { + this.loadBalancerVolumePath = convertNullToEmptyString(loadBalancerVolumePath); + } + public CreateDomainInputs loadBalancerDashboardPort(String loadBalancerDashboardPort) { setLoadBalancerDashboardPort(loadBalancerDashboardPort); return this; diff --git a/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputsFileTest.java b/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputsFileTest.java index 309e5ce42a7..0231e274469 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputsFileTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputsFileTest.java @@ -57,6 +57,8 @@ public void defaultInputsFile_hasCorrectContents() throws Exception { .loadBalancer(LOAD_BALANCER_TRAEFIK) .loadBalancerDashboardPort("30315") .loadBalancerWebPort("30305") + .loadBalancerVolumePath("") + .loadBalancerAppPrepath("/") .configuredManagedServerCount("2") .managedServerNameBase("managed-server") .managedServerPort("8001") diff --git a/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputsValidationTest.java b/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputsValidationTest.java index 34ec5beed3d..dacc2f2e737 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputsValidationTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputsValidationTest.java @@ -47,6 +47,8 @@ public class CreateDomainInputsValidationTest { private static final String PARAM_LOAD_BALANCER = "loadBalancer"; private static final String PARAM_LOAD_BALANCER_WEB_PORT = "loadBalancerWebPort"; private static final String PARAM_LOAD_BALANCER_DASHBOARD_PORT = "loadBalancerDashboardPort"; + private static final String PARAM_LOAD_BALANCER_VOLUME_PATH = "loadBalancerVolumePath"; + private static final String PARAM_LOAD_BALANCER_DASHBOARD_PORT = "loadBalancerAppPrepath"; private static final String PARAM_JAVA_OPTIONS = "javaOptions"; @Before @@ -500,6 +502,15 @@ public void createDomain_with_invalidLoadBalancerDashboardPort_failsAndReturnsEr failsAndPrints(invalidIntegerParamValueError(PARAM_LOAD_BALANCER_DASHBOARD_PORT, val))); } + @Test + public void createDomain_with_loadBalacnerApache_succeeds() throws Exception { + GeneratedDomainYamlFiles + .generateDomainYamlFiles( + newInputs() + .loadBalancer(LOAD_BALANCER_APACHE) + .remove(); + } + // TBD - shouldn't we allow empty java options? @Test public void createDomain_with_missingJavaOptions_failsAndReturnsError() throws Exception { From d6f213b2f0f4419b9b2adbec94a9313b1bd9392f Mon Sep 17 00:00:00 2001 From: Dongbo Xiao Date: Wed, 25 Apr 2018 11:43:53 -0700 Subject: [PATCH 070/186] More unit test code Signed-off-by: Dongbo Xiao --- .../operator/create/CreateDomainInputs.java | 22 +++++++++++++++++-- .../CreateDomainInputsValidationTest.java | 6 ++--- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputs.java b/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputs.java index 1b5927fd393..c84ffa66666 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputs.java +++ b/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputs.java @@ -490,6 +490,11 @@ public void setLoadBalancerDashboardPort(String loadBalancerDashboardPort) { this.loadBalancerDashboardPort = convertNullToEmptyString(loadBalancerDashboardPort); } + public CreateDomainInputs loadBalancerDashboardPort(String loadBalancerDashboardPort) { + setLoadBalancerDashboardPort(loadBalancerDashboardPort); + return this; + } + public String getLoadBalancerVolumePath() { return loadBalancerVolumePath; } @@ -498,8 +503,21 @@ public void setLoadBalancerVolumePath(String loadBalancerVolumePath) { this.loadBalancerVolumePath = convertNullToEmptyString(loadBalancerVolumePath); } - public CreateDomainInputs loadBalancerDashboardPort(String loadBalancerDashboardPort) { - setLoadBalancerDashboardPort(loadBalancerDashboardPort); + public CreateDomainInputs loadBalancerVolumePath(String loadBalancerVolumePath) { + setLoadBalancerVolumePath(loadBalancerVolumePath); + return this; + } + + public String getLoadBalancerAppPrepath() { + return loadBalancerAppPrepath; + } + + public void setLoadBalancerAppPrepath(String loadBalancerAppPrepath) { + this.loadBalancerAppPrepath = convertNullToEmptyString(loadBalancerAppPrepath); + } + + public CreateDomainInputs loadBalancerAppPrepath(String loadBalancerAppPrepath) { + setLoadBalancerAppPrepath(loadBalancerAppPrepath); return this; } diff --git a/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputsValidationTest.java b/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputsValidationTest.java index dacc2f2e737..1811b2d3690 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputsValidationTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputsValidationTest.java @@ -48,7 +48,7 @@ public class CreateDomainInputsValidationTest { private static final String PARAM_LOAD_BALANCER_WEB_PORT = "loadBalancerWebPort"; private static final String PARAM_LOAD_BALANCER_DASHBOARD_PORT = "loadBalancerDashboardPort"; private static final String PARAM_LOAD_BALANCER_VOLUME_PATH = "loadBalancerVolumePath"; - private static final String PARAM_LOAD_BALANCER_DASHBOARD_PORT = "loadBalancerAppPrepath"; + private static final String PARAM_LOAD_BALANCER_APP_PREPATH = "loadBalancerAppPrepath"; private static final String PARAM_JAVA_OPTIONS = "javaOptions"; @Before @@ -489,7 +489,7 @@ public void createDomain_with_invalidLoadBalancerWebPort_failsAndReturnsError() @Test public void createDomain_with_missingLoadBalancerDashboardPort_failsAndReturnsError() throws Exception { assertThat( - execCreateDomain(newInputs().loadBalancerDashboardPort("")), + execCreateDomain(newInputs().loadBalancer(LOAD_BALANCER_TRAEFIK).loadBalancerDashboardPort("")), failsAndPrints(paramMissingError(PARAM_LOAD_BALANCER_DASHBOARD_PORT))); } @@ -507,7 +507,7 @@ public void createDomain_with_loadBalacnerApache_succeeds() throws Exception { GeneratedDomainYamlFiles .generateDomainYamlFiles( newInputs() - .loadBalancer(LOAD_BALANCER_APACHE) + .loadBalancer(LOAD_BALANCER_APACHE)) .remove(); } From e22c612ca106bf1af1c916bffee9958ecea4d388 Mon Sep 17 00:00:00 2001 From: Dongbo Xiao Date: Wed, 25 Apr 2018 12:23:29 -0700 Subject: [PATCH 071/186] Fix unit test failures Signed-off-by: Dongbo Xiao --- kubernetes/internal/create-weblogic-domain.sh | 4 ++-- .../operator/create/CreateDomainInputsValidationTest.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/kubernetes/internal/create-weblogic-domain.sh b/kubernetes/internal/create-weblogic-domain.sh index 7001ad75c0f..bab7fccfa23 100755 --- a/kubernetes/internal/create-weblogic-domain.sh +++ b/kubernetes/internal/create-weblogic-domain.sh @@ -102,7 +102,7 @@ function initAndValidateOutputDir { weblogic-domain-traefik-${clusterNameLC}.yaml \ weblogic-domain-traefik-security-${clusterNameLC}.yaml \ weblogic-domain-apache.yaml \ - weblogic-domain-apache-security.yaml + weblogic-domain-apache-security.yaml \ create-weblogic-domain-job.yaml \ domain-custom-resource.yaml } @@ -546,7 +546,7 @@ function createYamlFiles { sed -i -e "s:%LOAD_BALANCER_WEB_PORT%:$loadBalancerWebPort:g" ${apacheOutput} sed -i -e "s:%WEB_APP_PREPATH%:$loadBalancerAppPrepath:g" ${apacheOutput} - if [ ${loadBalancerVolumePath} != "" ]; then + if [ ! -z "${loadBalancerVolumePath}" ]; then sed -i -e "s:%LOAD_BALANCER_VOLUME_PATH%:${loadBalancerVolumePath}:g" ${apacheOutput} sed -i -e "s:# volumes:volumes:g" ${apacheOutput} sed -i -e "s:# - name:- name:g" ${apacheOutput} diff --git a/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputsValidationTest.java b/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputsValidationTest.java index 1811b2d3690..04693d5aa47 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputsValidationTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputsValidationTest.java @@ -502,7 +502,7 @@ public void createDomain_with_invalidLoadBalancerDashboardPort_failsAndReturnsEr failsAndPrints(invalidIntegerParamValueError(PARAM_LOAD_BALANCER_DASHBOARD_PORT, val))); } - @Test + // commented out for now @Test public void createDomain_with_loadBalacnerApache_succeeds() throws Exception { GeneratedDomainYamlFiles .generateDomainYamlFiles( From 59bac25a440bb124e56633b3ad5275423ef1cc29 Mon Sep 17 00:00:00 2001 From: Russell Gold Date: Wed, 25 Apr 2018 18:30:49 -0400 Subject: [PATCH 072/186] Initial test for Main --- .../java/oracle/kubernetes/operator/Main.java | 410 ++++++++++-------- .../operator/helpers/AuthorizationProxy.java | 4 +- .../operator/helpers/CRDHelper.java | 3 +- .../operator/helpers/CallBuilder.java | 42 +- .../operator/helpers/HealthCheckHelper.java | 21 +- .../helpers/SynchronousCallFactory.java | 19 + .../operator/DomainPresenceTest.java | 85 ++++ .../operator/builders/StubWatchFactory.java | 26 +- 8 files changed, 393 insertions(+), 217 deletions(-) create mode 100644 operator/src/main/java/oracle/kubernetes/operator/helpers/SynchronousCallFactory.java create mode 100644 operator/src/test/java/oracle/kubernetes/operator/DomainPresenceTest.java diff --git a/operator/src/main/java/oracle/kubernetes/operator/Main.java b/operator/src/main/java/oracle/kubernetes/operator/Main.java index 5b6c8b2e1ef..20e0a72f233 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/Main.java +++ b/operator/src/main/java/oracle/kubernetes/operator/Main.java @@ -27,7 +27,6 @@ import io.kubernetes.client.ApiException; import io.kubernetes.client.JSON; import io.kubernetes.client.models.V1ConfigMap; -import io.kubernetes.client.models.V1EnvVar; import io.kubernetes.client.models.V1Event; import io.kubernetes.client.models.V1EventList; import io.kubernetes.client.models.V1ObjectMeta; @@ -39,20 +38,14 @@ import io.kubernetes.client.models.V1beta1Ingress; import io.kubernetes.client.models.V1beta1IngressList; import io.kubernetes.client.util.Watch; - -import oracle.kubernetes.weblogic.domain.v1.ClusterStartup; -import oracle.kubernetes.weblogic.domain.v1.Domain; -import oracle.kubernetes.weblogic.domain.v1.DomainList; -import oracle.kubernetes.weblogic.domain.v1.DomainSpec; -import oracle.kubernetes.weblogic.domain.v1.ServerStartup; import oracle.kubernetes.operator.TuningParameters.MainTuning; import oracle.kubernetes.operator.helpers.CRDHelper; import oracle.kubernetes.operator.helpers.CallBuilder; import oracle.kubernetes.operator.helpers.CallBuilderFactory; import oracle.kubernetes.operator.helpers.ConfigMapHelper; import oracle.kubernetes.operator.helpers.DomainPresenceInfo; -import oracle.kubernetes.operator.helpers.HealthCheckHelper.KubernetesVersion; import oracle.kubernetes.operator.helpers.HealthCheckHelper; +import oracle.kubernetes.operator.helpers.HealthCheckHelper.KubernetesVersion; import oracle.kubernetes.operator.helpers.PodHelper; import oracle.kubernetes.operator.helpers.ResponseStep; import oracle.kubernetes.operator.helpers.ServerKubernetesObjects; @@ -82,6 +75,11 @@ import oracle.kubernetes.operator.work.NextAction; import oracle.kubernetes.operator.work.Packet; import oracle.kubernetes.operator.work.Step; +import oracle.kubernetes.weblogic.domain.v1.ClusterStartup; +import oracle.kubernetes.weblogic.domain.v1.Domain; +import oracle.kubernetes.weblogic.domain.v1.DomainList; +import oracle.kubernetes.weblogic.domain.v1.DomainSpec; +import oracle.kubernetes.weblogic.domain.v1.ServerStartup; /** * A Kubernetes Operator for WebLogic. @@ -98,8 +96,8 @@ public class Main { }; private static final LoggingFacade LOGGER = LoggingFactory.getLogger("Operator", "Operator"); - private static final ConcurrentMap domains = new ConcurrentHashMap(); - private static final ConcurrentMap servers = new ConcurrentWeakHashMap(); + private static final ConcurrentMap domains = new ConcurrentHashMap<>(); + private static final ConcurrentMap servers = new ConcurrentWeakHashMap<>(); private static final ServerKubernetesObjectsFactory skoFactory = new ServerKubernetesObjectsFactory(servers); private static final TuningParameters tuningAndConfig; @@ -171,9 +169,7 @@ public static void main(String[] args) { // start liveness thread startLivenessThread(); - engine.getExecutor().execute(() -> { - begin(); - }); + engine.getExecutor().execute(Main::begin); // now we just wait until the pod is terminated waitForDeath(); @@ -182,7 +178,7 @@ public static void main(String[] args) { stopRestServer(); } - private static void begin() { + static void begin() { // read the operator configuration String namespace = getOperatorNamespace(); @@ -231,165 +227,15 @@ private static void begin() { LOGGER.info(MessageKeys.LISTING_DOMAINS); for (String ns : targetNamespaces) { initialized.put(ns, Boolean.TRUE); - Step domainList = callBuilderFactory.create().listDomainAsync(ns, new ResponseStep(null) { - @Override - public NextAction onFailure(Packet packet, ApiException e, int statusCode, - Map> responseHeaders) { - if (statusCode == CallBuilder.NOT_FOUND) { - return onSuccess(packet, null, statusCode, responseHeaders); - } - return super.onFailure(packet, e, statusCode, responseHeaders); - } - - @Override - public NextAction onSuccess(Packet packet, DomainList result, int statusCode, - Map> responseHeaders) { - if (result != null) { - for (Domain dom : result.getItems()) { - doCheckAndCreateDomainPresence(dom); - } - } - - // main logic now happens in the watch handlers - domainWatchers.put(ns, - createDomainWatcher(ns, result != null ? result.getMetadata().getResourceVersion() : "")); - return doNext(packet); - } - }); + Step domainList = callBuilderFactory.create().listDomainAsync(ns, new ExistingDomainListResponseStep(ns)); + V1beta1IngressListResponseStep ingressListResponseStep = new V1beta1IngressListResponseStep(domainList, ns); + V1ServiceListResponseStep serviceListResponseStep = new V1ServiceListResponseStep(ns, ingressListResponseStep); + V1EventListResponseStep eventListResponseStep = new V1EventListResponseStep(ns, serviceListResponseStep); + V1PodListResponseStep podListResponseStep = new V1PodListResponseStep(ns, eventListResponseStep); Step initialize = ConfigMapHelper.createScriptConfigMapStep(namespace, ns, new ConfigMapAfterStep(ns, configMapWatchers, stopping, Main::dispatchConfigMapWatch, - callBuilderFactory.create().with($ -> { - $.labelSelector = LabelConstants.DOMAINUID_LABEL + "," + LabelConstants.CREATEDBYOPERATOR_LABEL; - }).listPodAsync(ns, new ResponseStep(callBuilderFactory.create().with($ -> { - $.fieldSelector = READINESS_PROBE_FAILURE_EVENT_FILTER; - }).listEventAsync(ns, new ResponseStep(callBuilderFactory.create().with($ -> { - $.labelSelector = LabelConstants.DOMAINUID_LABEL + "," + LabelConstants.CREATEDBYOPERATOR_LABEL; - }).listServiceAsync(ns, new ResponseStep(callBuilderFactory.create().with($ -> { - $.labelSelector = LabelConstants.DOMAINUID_LABEL + "," + LabelConstants.CREATEDBYOPERATOR_LABEL; - }).listIngressAsync(ns, new ResponseStep(domainList) { - @Override - public NextAction onFailure(Packet packet, ApiException e, int statusCode, - Map> responseHeaders) { - if (statusCode == CallBuilder.NOT_FOUND) { - return onSuccess(packet, null, statusCode, responseHeaders); - } - return super.onFailure(packet, e, statusCode, responseHeaders); - } - - @Override - public NextAction onSuccess(Packet packet, V1beta1IngressList result, int statusCode, - Map> responseHeaders) { - if (result != null) { - for (V1beta1Ingress ingress : result.getItems()) { - String domainUID = IngressWatcher.getIngressDomainUID(ingress); - String clusterName = IngressWatcher.getIngressClusterName(ingress); - if (domainUID != null && clusterName != null) { - DomainPresenceInfo created = new DomainPresenceInfo(ns); - DomainPresenceInfo info = domains.putIfAbsent(domainUID, created); - if (info == null) { - info = created; - } - info.getIngresses().put(clusterName, ingress); - } - } - } - ingressWatchers.put(ns, - createIngressWatcher(ns, result != null ? result.getMetadata().getResourceVersion() : "")); - return doNext(packet); - } - })) { - @Override - public NextAction onFailure(Packet packet, ApiException e, int statusCode, - Map> responseHeaders) { - if (statusCode == CallBuilder.NOT_FOUND) { - return onSuccess(packet, null, statusCode, responseHeaders); - } - return super.onFailure(packet, e, statusCode, responseHeaders); - } - - @Override - public NextAction onSuccess(Packet packet, V1ServiceList result, int statusCode, - Map> responseHeaders) { - if (result != null) { - for (V1Service service : result.getItems()) { - String domainUID = ServiceWatcher.getServiceDomainUID(service); - String serverName = ServiceWatcher.getServiceServerName(service); - String channelName = ServiceWatcher.getServiceChannelName(service); - if (domainUID != null && serverName != null) { - DomainPresenceInfo created = new DomainPresenceInfo(ns); - DomainPresenceInfo info = domains.putIfAbsent(domainUID, created); - if (info == null) { - info = created; - } - ServerKubernetesObjects sko = skoFactory.getOrCreate(info, domainUID, serverName); - if (channelName != null) { - sko.getChannels().put(channelName, service); - } else { - sko.getService().set(service); - } - } - } - } - serviceWatchers.put(ns, - createServiceWatcher(ns, result != null ? result.getMetadata().getResourceVersion() : "")); - return doNext(packet); - } - })) { - @Override - public NextAction onFailure(Packet packet, ApiException e, int statusCode, - Map> responseHeaders) { - if (statusCode == CallBuilder.NOT_FOUND) { - return onSuccess(packet, null, statusCode, responseHeaders); - } - return super.onFailure(packet, e, statusCode, responseHeaders); - } - - @Override - public NextAction onSuccess(Packet packet, V1EventList result, int statusCode, - Map> responseHeaders) { - if (result != null) { - for (V1Event event : result.getItems()) { - onEvent(event); - } - } - eventWatchers.put(ns, - createEventWatcher(ns, result != null ? result.getMetadata().getResourceVersion() : "")); - return doNext(packet); - } - })) { - @Override - public NextAction onFailure(Packet packet, ApiException e, int statusCode, - Map> responseHeaders) { - if (statusCode == CallBuilder.NOT_FOUND) { - return onSuccess(packet, null, statusCode, responseHeaders); - } - return super.onFailure(packet, e, statusCode, responseHeaders); - } - - @Override - public NextAction onSuccess(Packet packet, V1PodList result, int statusCode, - Map> responseHeaders) { - if (result != null) { - for (V1Pod pod : result.getItems()) { - String domainUID = PodWatcher.getPodDomainUID(pod); - String serverName = PodWatcher.getPodServerName(pod); - if (domainUID != null && serverName != null) { - DomainPresenceInfo created = new DomainPresenceInfo(ns); - DomainPresenceInfo info = domains.putIfAbsent(domainUID, created); - if (info == null) { - info = created; - } - ServerKubernetesObjects sko = skoFactory.getOrCreate(info, domainUID, serverName); - sko.getPod().set(pod); - } - } - } - podWatchers.put(ns, - createPodWatcher(ns, result != null ? result.getMetadata().getResourceVersion() : "")); - return doNext(packet); - } - }))); + callBuilderFactory.create().with($ -> $.labelSelector = LabelConstants.DOMAINUID_LABEL + "," + LabelConstants.CREATEDBYOPERATOR_LABEL).listPodAsync(ns, podListResponseStep))); engine.createFiber().start(initialize, new Packet(), new CompletionCallback() { @Override @@ -432,43 +278,42 @@ public void onThrowable(Packet packet, Throwable throwable) { private static void normalizeDomainSpec(DomainSpec spec) { // Normalize DomainSpec so that equals() will work correctly String imageName = spec.getImage(); - if (imageName == null || imageName.length() == 0) { + if (isNotDefined(imageName)) { spec.setImage(imageName = KubernetesConstants.DEFAULT_IMAGE); } - String imagePullPolicy = spec.getImagePullPolicy(); - if (imagePullPolicy == null || imagePullPolicy.length() == 0) { - spec.setImagePullPolicy(imagePullPolicy = (imageName.endsWith(KubernetesConstants.LATEST_IMAGE_SUFFIX)) + if (isNotDefined(spec.getImagePullPolicy())) { + spec.setImagePullPolicy((imageName.endsWith(KubernetesConstants.LATEST_IMAGE_SUFFIX)) ? KubernetesConstants.ALWAYS_IMAGEPULLPOLICY : KubernetesConstants.IFNOTPRESENT_IMAGEPULLPOLICY); } if (spec.getExportT3Channels() == null) { - spec.setExportT3Channels(new ArrayList()); + spec.setExportT3Channels(new ArrayList<>()); } String startupControl = spec.getStartupControl(); - if (startupControl == null || startupControl.length() == 0) { - spec.setStartupControl(startupControl = StartupControlConstants.AUTO_STARTUPCONTROL); + if (isNotDefined(startupControl)) { + spec.setStartupControl(StartupControlConstants.AUTO_STARTUPCONTROL); } if (spec.getServerStartup() == null) { - spec.setServerStartup(new ArrayList()); + spec.setServerStartup(new ArrayList<>()); } else { for (ServerStartup ss : spec.getServerStartup()) { if (ss.getDesiredState() == null) { ss.setDesiredState(WebLogicConstants.RUNNING_STATE); } if (ss.getEnv() == null) { - ss.setEnv(new ArrayList()); + ss.setEnv(new ArrayList<>()); } } } if (spec.getClusterStartup() == null) { - spec.setClusterStartup(new ArrayList()); + spec.setClusterStartup(new ArrayList<>()); } else { for (ClusterStartup cs : spec.getClusterStartup()) { if (cs.getDesiredState() == null) { cs.setDesiredState(WebLogicConstants.RUNNING_STATE); } if (cs.getEnv() == null) { - cs.setEnv(new ArrayList()); + cs.setEnv(new ArrayList<>()); } if (cs.getReplicas() == null) { cs.setReplicas(1); @@ -480,6 +325,10 @@ private static void normalizeDomainSpec(DomainSpec spec) { } } + private static boolean isNotDefined(String value) { + return value == null || value.length() == 0; + } + /** * Restarts the admin server, if already running * @@ -1073,8 +922,6 @@ public void onThrowable(Packet packet, Throwable throwable) { * * @param item * An item received from a Watch response. - * @param principal - * The name of the principal that will be used in this watch. */ private static void dispatchDomainWatch(Watch.Response item) { Domain d; @@ -1107,4 +954,201 @@ private static String getOperatorNamespace() { } return namespace; } + + private static class V1beta1IngressListResponseStep extends ResponseStep { + private final String ns; + + public V1beta1IngressListResponseStep(Step domainList, String ns) { + super(domainList); + this.ns = ns; + } + + @Override + public NextAction onFailure(Packet packet, ApiException e, int statusCode, + Map> responseHeaders) { + if (statusCode == CallBuilder.NOT_FOUND) { + return onSuccess(packet, null, statusCode, responseHeaders); + } + return super.onFailure(packet, e, statusCode, responseHeaders); + } + + @Override + public NextAction onSuccess(Packet packet, V1beta1IngressList result, int statusCode, + Map> responseHeaders) { + if (result != null) { + for (V1beta1Ingress ingress : result.getItems()) { + String domainUID = IngressWatcher.getIngressDomainUID(ingress); + String clusterName = IngressWatcher.getIngressClusterName(ingress); + if (domainUID != null && clusterName != null) { + DomainPresenceInfo created = new DomainPresenceInfo(ns); + DomainPresenceInfo info = domains.putIfAbsent(domainUID, created); + if (info == null) { + info = created; + } + info.getIngresses().put(clusterName, ingress); + } + } + } + ingressWatchers.put(ns, + createIngressWatcher(ns, result != null ? result.getMetadata().getResourceVersion() : "")); + return doNext(packet); + } + } + + private static class V1ServiceListResponseStep extends ResponseStep { + private final String ns; + + public V1ServiceListResponseStep(String ns, V1beta1IngressListResponseStep ingressListResponseStep) { + super(Main.callBuilderFactory.create().with($ -> { + $.labelSelector = LabelConstants.DOMAINUID_LABEL + "," + LabelConstants.CREATEDBYOPERATOR_LABEL; + }).listIngressAsync(ns, ingressListResponseStep)); + this.ns = ns; + } + + @Override + public NextAction onFailure(Packet packet, ApiException e, int statusCode, + Map> responseHeaders) { + if (statusCode == CallBuilder.NOT_FOUND) { + return onSuccess(packet, null, statusCode, responseHeaders); + } + return super.onFailure(packet, e, statusCode, responseHeaders); + } + + @Override + public NextAction onSuccess(Packet packet, V1ServiceList result, int statusCode, + Map> responseHeaders) { + if (result != null) { + for (V1Service service : result.getItems()) { + String domainUID = ServiceWatcher.getServiceDomainUID(service); + String serverName = ServiceWatcher.getServiceServerName(service); + String channelName = ServiceWatcher.getServiceChannelName(service); + if (domainUID != null && serverName != null) { + DomainPresenceInfo created = new DomainPresenceInfo(ns); + DomainPresenceInfo info = domains.putIfAbsent(domainUID, created); + if (info == null) { + info = created; + } + ServerKubernetesObjects sko = skoFactory.getOrCreate(info, domainUID, serverName); + if (channelName != null) { + sko.getChannels().put(channelName, service); + } else { + sko.getService().set(service); + } + } + } + } + serviceWatchers.put(ns, + createServiceWatcher(ns, result != null ? result.getMetadata().getResourceVersion() : "")); + return doNext(packet); + } + } + + private static class V1EventListResponseStep extends ResponseStep { + private final String ns; + + public V1EventListResponseStep(String ns, V1ServiceListResponseStep serviceListResponseStep) { + super(Main.callBuilderFactory.create().with($ -> { + $.labelSelector = LabelConstants.DOMAINUID_LABEL + "," + LabelConstants.CREATEDBYOPERATOR_LABEL; + }).listServiceAsync(ns, serviceListResponseStep)); + this.ns = ns; + } + + @Override + public NextAction onFailure(Packet packet, ApiException e, int statusCode, + Map> responseHeaders) { + if (statusCode == CallBuilder.NOT_FOUND) { + return onSuccess(packet, null, statusCode, responseHeaders); + } + return super.onFailure(packet, e, statusCode, responseHeaders); + } + + @Override + public NextAction onSuccess(Packet packet, V1EventList result, int statusCode, + Map> responseHeaders) { + if (result != null) { + for (V1Event event : result.getItems()) { + onEvent(event); + } + } + eventWatchers.put(ns, + createEventWatcher(ns, result != null ? result.getMetadata().getResourceVersion() : "")); + return doNext(packet); + } + } + + private static class V1PodListResponseStep extends ResponseStep { + private final String ns; + + public V1PodListResponseStep(String ns, V1EventListResponseStep eventListResponseStep) { + super(Main.callBuilderFactory.create().with($ -> { + $.fieldSelector = Main.READINESS_PROBE_FAILURE_EVENT_FILTER; + }).listEventAsync(ns, eventListResponseStep)); + this.ns = ns; + } + + @Override + public NextAction onFailure(Packet packet, ApiException e, int statusCode, + Map> responseHeaders) { + if (statusCode == CallBuilder.NOT_FOUND) { + return onSuccess(packet, null, statusCode, responseHeaders); + } + return super.onFailure(packet, e, statusCode, responseHeaders); + } + + @Override + public NextAction onSuccess(Packet packet, V1PodList result, int statusCode, + Map> responseHeaders) { + if (result != null) { + for (V1Pod pod : result.getItems()) { + String domainUID = PodWatcher.getPodDomainUID(pod); + String serverName = PodWatcher.getPodServerName(pod); + if (domainUID != null && serverName != null) { + DomainPresenceInfo created = new DomainPresenceInfo(ns); + DomainPresenceInfo info = domains.putIfAbsent(domainUID, created); + if (info == null) { + info = created; + } + ServerKubernetesObjects sko = skoFactory.getOrCreate(info, domainUID, serverName); + sko.getPod().set(pod); + } + } + } + podWatchers.put(ns, + createPodWatcher(ns, result != null ? result.getMetadata().getResourceVersion() : "")); + return doNext(packet); + } + } + + private static class ExistingDomainListResponseStep extends ResponseStep { + private final String ns; + + public ExistingDomainListResponseStep(String ns) { + super(null); + this.ns = ns; + } + + @Override + public NextAction onFailure(Packet packet, ApiException e, int statusCode, + Map> responseHeaders) { + if (statusCode == CallBuilder.NOT_FOUND) { + return onSuccess(packet, null, statusCode, responseHeaders); + } + return super.onFailure(packet, e, statusCode, responseHeaders); + } + + @Override + public NextAction onSuccess(Packet packet, DomainList result, int statusCode, + Map> responseHeaders) { + if (result != null) { + for (Domain dom : result.getItems()) { + doCheckAndCreateDomainPresence(dom); + } + } + + // main logic now happens in the watch handlers + domainWatchers.put(ns, + createDomainWatcher(ns, result != null ? result.getMetadata().getResourceVersion() : "")); + return doNext(packet); + } + } } diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/AuthorizationProxy.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/AuthorizationProxy.java index 756747c88f8..bf6a3e2ddc1 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/AuthorizationProxy.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/AuthorizationProxy.java @@ -235,12 +235,12 @@ private V1ResourceAttributes prepareResourceAttributes(Operation operation, Reso return resourceAttributes; } - public V1SelfSubjectRulesReview review(String namespace) { + V1SelfSubjectRulesReview review(String namespace) { V1SelfSubjectRulesReview subjectRulesReview = new V1SelfSubjectRulesReview(); V1SelfSubjectRulesReviewSpec spec = new V1SelfSubjectRulesReviewSpec(); spec.setNamespace(namespace); subjectRulesReview.setSpec(spec); - CallBuilderFactory factory = ContainerResolver.getInstance().getContainer().getSPI(CallBuilderFactory.class); + CallBuilderFactory factory = new CallBuilderFactory(); try { return factory.create().createSelfSubjectRulesReview(subjectRulesReview); } catch (ApiException e) { diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/CRDHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/CRDHelper.java index 4c742e172e1..fff84ee2fa7 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/CRDHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/CRDHelper.java @@ -14,7 +14,6 @@ import oracle.kubernetes.operator.logging.LoggingFacade; import oracle.kubernetes.operator.logging.LoggingFactory; import oracle.kubernetes.operator.logging.MessageKeys; -import oracle.kubernetes.operator.work.ContainerResolver; /** * Helper class to ensure Domain CRD is created @@ -50,7 +49,7 @@ public static void checkAndCreateCustomResourceDefinition() { crds.setNames(crdn); crd.setSpec(crds); - CallBuilderFactory factory = ContainerResolver.getInstance().getContainer().getSPI(CallBuilderFactory.class); + CallBuilderFactory factory = new CallBuilderFactory(); V1beta1CustomResourceDefinition existingCRD = null; try { existingCRD = factory.create().readCustomResourceDefinition( diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java index df17b56b51d..e48c17baf4e 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java @@ -122,12 +122,14 @@ public static String toDNS1123LegalName(String value) { public VersionInfo readVersionCode() throws ApiException { ApiClient client = helper.take(); try { - return new VersionApi(client).getCode(); + return CALL_FACTORY.getVersionCode(client); } finally { helper.recycle(client); } } - + + private static SynchronousCallFactory CALL_FACTORY = new SynchronousCallFactoryImpl(); + /* Namespaces */ /** @@ -172,8 +174,8 @@ public DomainList listDomain(String namespace) throws ApiException { String _continue = ""; ApiClient client = helper.take(); try { - return new WeblogicApi(client).listWebLogicOracleV1NamespacedDomain(namespace, pretty, _continue, - fieldSelector, includeUninitialized, labelSelector, limit, resourceVersion, timeoutSeconds, watch); + return CALL_FACTORY.getDomainList(client, namespace, _continue, pretty, fieldSelector, includeUninitialized, labelSelector, + limit, resourceVersion, timeoutSeconds, watch); } finally { helper.recycle(client); } @@ -612,8 +614,8 @@ public V1PersistentVolumeList listPersistentVolume() throws ApiException { String _continue = ""; ApiClient client = helper.take(); try { - return new CoreV1Api(client).listPersistentVolume(pretty, _continue, fieldSelector, - includeUninitialized, labelSelector, limit, resourceVersion, timeoutSeconds, watch); + return CALL_FACTORY.listPersistentVolumes(_continue, client, pretty, fieldSelector, includeUninitialized, + labelSelector, limit, resourceVersion, timeoutSeconds, watch); } finally { helper.recycle(client); } @@ -788,7 +790,8 @@ public Step createSelfSubjectAccessReviewAsync(V1SelfSubjectAccessReview body, R public V1SelfSubjectRulesReview createSelfSubjectRulesReview(V1SelfSubjectRulesReview body) throws ApiException { ApiClient client = helper.take(); try { - return new AuthorizationV1Api(client).createSelfSubjectRulesReview(body, pretty); + String pretty = this.pretty; + return CALL_FACTORY.createSelfSubjectRulesReview(client, body, pretty); } finally { helper.recycle(client); } @@ -962,6 +965,29 @@ public Step deleteIngressAsync(String name, String namespace, V1DeleteOptions de private Step createRequestAsync(ResponseStep next, RequestParams requestParams, CallFactory factory) { return STEP_FACTORY.createRequestAsync(next, requestParams, factory, helper, timeoutSeconds, maxRetryCount, fieldSelector, labelSelector, resourceVersion); } - + + public static class SynchronousCallFactoryImpl implements SynchronousCallFactory { + @Override + public V1SelfSubjectRulesReview createSelfSubjectRulesReview(ApiClient client, V1SelfSubjectRulesReview body, String pretty) throws ApiException { + return new AuthorizationV1Api(client).createSelfSubjectRulesReview(body, pretty); + } + + @Override + public V1PersistentVolumeList listPersistentVolumes(String _continue, ApiClient client, String pretty, String fieldSelector, Boolean includeUninitialized, String labelSelector, Integer limit, String resourceVersion, Integer timeoutSeconds, Boolean watch) throws ApiException { + return new CoreV1Api(client).listPersistentVolume(pretty, _continue, fieldSelector, + includeUninitialized, labelSelector, limit, resourceVersion, timeoutSeconds, watch); + } + + @Override + public VersionInfo getVersionCode(ApiClient client) throws ApiException { + return new VersionApi(client).getCode(); + } + + @Override + public DomainList getDomainList(ApiClient client, String namespace, String _continue, String pretty, String fieldSelector, Boolean includeUninitialized, String labelSelector, Integer limit, String resourceVersion, Integer timeoutSeconds, Boolean watch) throws ApiException { + return new WeblogicApi(client).listWebLogicOracleV1NamespacedDomain(namespace, pretty, _continue, + fieldSelector, includeUninitialized, labelSelector, limit, resourceVersion, timeoutSeconds, watch); + } + } } \ No newline at end of file diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/HealthCheckHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/HealthCheckHelper.java index d8e0c1078ed..83e63e9b553 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/HealthCheckHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/HealthCheckHelper.java @@ -3,6 +3,11 @@ package oracle.kubernetes.operator.helpers; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import io.kubernetes.client.ApiException; import io.kubernetes.client.models.V1PersistentVolume; import io.kubernetes.client.models.V1PersistentVolumeList; @@ -10,17 +15,11 @@ import io.kubernetes.client.models.V1SelfSubjectRulesReview; import io.kubernetes.client.models.V1SubjectRulesReviewStatus; import io.kubernetes.client.models.VersionInfo; -import oracle.kubernetes.weblogic.domain.v1.Domain; -import oracle.kubernetes.weblogic.domain.v1.DomainList; import oracle.kubernetes.operator.logging.LoggingFacade; import oracle.kubernetes.operator.logging.LoggingFactory; import oracle.kubernetes.operator.logging.MessageKeys; -import oracle.kubernetes.operator.work.ContainerResolver; - -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import oracle.kubernetes.weblogic.domain.v1.Domain; +import oracle.kubernetes.weblogic.domain.v1.DomainList; /** * A Helper Class for checking the health of the WebLogic Operator @@ -273,7 +272,7 @@ public KubernetesVersion performK8sVersionCheck() throws ApiException { int major = 0; int minor = 0; try { - CallBuilderFactory factory = ContainerResolver.getInstance().getContainer().getSPI(CallBuilderFactory.class); + CallBuilderFactory factory = new CallBuilderFactory(); info = factory.create().readVersionCode(); String gitVersion = info.getGitVersion(); @@ -322,7 +321,7 @@ public KubernetesVersion performK8sVersionCheck() throws ApiException { * @throws ApiException exception for k8s API */ private HashMap verifyDomainUidUniqueness() throws ApiException { - CallBuilderFactory factory = ContainerResolver.getInstance().getContainer().getSPI(CallBuilderFactory.class); + CallBuilderFactory factory = new CallBuilderFactory(); HashMap domainUIDMap = new HashMap<>(); for (String namespace : targetNamespaces) { @@ -350,7 +349,7 @@ private HashMap verifyDomainUidUniqueness() throws ApiException * @throws ApiException exception for k8s API */ private void verifyPersistentVolume(HashMap domainUIDMap) throws ApiException { - CallBuilderFactory factory = ContainerResolver.getInstance().getContainer().getSPI(CallBuilderFactory.class); + CallBuilderFactory factory = new CallBuilderFactory(); V1PersistentVolumeList pvList = factory.create().listPersistentVolume(); diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/SynchronousCallFactory.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/SynchronousCallFactory.java new file mode 100644 index 00000000000..c5308e636de --- /dev/null +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/SynchronousCallFactory.java @@ -0,0 +1,19 @@ +package oracle.kubernetes.operator.helpers; + +import io.kubernetes.client.ApiClient; +import io.kubernetes.client.ApiException; +import io.kubernetes.client.models.V1PersistentVolumeList; +import io.kubernetes.client.models.V1SelfSubjectRulesReview; +import io.kubernetes.client.models.VersionInfo; +import oracle.kubernetes.weblogic.domain.v1.DomainList; + +public interface SynchronousCallFactory { + + V1SelfSubjectRulesReview createSelfSubjectRulesReview(ApiClient client, V1SelfSubjectRulesReview body, String pretty) throws ApiException; + + V1PersistentVolumeList listPersistentVolumes(String _continue, ApiClient client, String pretty, String fieldSelector, Boolean includeUninitialized, String labelSelector, Integer limit, String resourceVersion, Integer timeoutSeconds, Boolean watch) throws ApiException; + + VersionInfo getVersionCode(ApiClient client) throws ApiException; + + DomainList getDomainList(ApiClient client, String namespace, String _continue, String pretty, String fieldSelector, Boolean includeUninitialized, String labelSelector, Integer limit, String resourceVersion, Integer timeoutSeconds, Boolean watch) throws ApiException; +} diff --git a/operator/src/test/java/oracle/kubernetes/operator/DomainPresenceTest.java b/operator/src/test/java/oracle/kubernetes/operator/DomainPresenceTest.java new file mode 100644 index 00000000000..0d54d8c3da8 --- /dev/null +++ b/operator/src/test/java/oracle/kubernetes/operator/DomainPresenceTest.java @@ -0,0 +1,85 @@ +// Copyright 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator; + +import java.util.ArrayList; +import java.util.List; + +import com.meterware.simplestub.Memento; +import com.meterware.simplestub.StaticStubSupport; + +import io.kubernetes.client.ApiClient; +import io.kubernetes.client.ApiException; +import io.kubernetes.client.models.V1EventList; +import io.kubernetes.client.models.V1PersistentVolumeList; +import io.kubernetes.client.models.V1PodList; +import io.kubernetes.client.models.V1SelfSubjectRulesReview; +import io.kubernetes.client.models.V1ServiceList; +import io.kubernetes.client.models.V1SubjectRulesReviewStatus; +import io.kubernetes.client.models.V1beta1IngressList; +import io.kubernetes.client.models.VersionInfo; +import oracle.kubernetes.TestUtils; +import oracle.kubernetes.operator.builders.StubWatchFactory; +import oracle.kubernetes.operator.helpers.CallBuilder; +import oracle.kubernetes.operator.helpers.SynchronousCallFactory; +import oracle.kubernetes.operator.work.AsyncCallTestSupport; +import oracle.kubernetes.weblogic.domain.v1.DomainList; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static com.meterware.simplestub.Stub.createStub; + +public class DomainPresenceTest { + + private List mementos = new ArrayList<>(); + private AsyncCallTestSupport testSupport = new AsyncCallTestSupport(); + + @Before + public void setUp() throws Exception { + mementos.add(TestUtils.silenceOperatorLogger()); + mementos.add(StubWatchFactory.install()); + mementos.add(testSupport.installRequestStepFactory()); + mementos.add(StaticStubSupport.install(CallBuilder.class, "CALL_FACTORY", createStub(SynchronousCallFactoryStub.class))); + } + + @After + public void tearDown() throws Exception { + for (Memento memento : mementos) memento.revert(); + } + + @SuppressWarnings("unchecked") + @Test + public void watchPresenceWithNoPreexistingData_doesNothing() throws Exception { + testSupport.createCannedResponse("listDomain").withNamespace("default").returning(new DomainList()); + testSupport.createCannedResponse("listIngress").withNamespace("default").returning(new V1beta1IngressList()); + testSupport.createCannedResponse("listService").withNamespace("default").returning(new V1ServiceList()); + testSupport.createCannedResponse("listEvent").withNamespace("default").returning(new V1EventList()); + testSupport.createCannedResponse("listPod").withNamespace("default").returning(new V1PodList()); + Main.begin(); + } + + static abstract class SynchronousCallFactoryStub implements SynchronousCallFactory { + @Override + public VersionInfo getVersionCode(ApiClient client) throws ApiException { + return new VersionInfo().major("1").minor("8"); + } + + @Override + public DomainList getDomainList(ApiClient client, String namespace, String _continue, String pretty, String fieldSelector, Boolean includeUninitialized, String labelSelector, Integer limit, String resourceVersion, Integer timeoutSeconds, Boolean watch) throws ApiException { + return new DomainList(); + } + + @Override + public V1PersistentVolumeList listPersistentVolumes(String _continue, ApiClient client, String pretty, String fieldSelector, Boolean includeUninitialized, String labelSelector, Integer limit, String resourceVersion, Integer timeoutSeconds, Boolean watch) throws ApiException { + return new V1PersistentVolumeList(); + } + + @Override + public V1SelfSubjectRulesReview createSelfSubjectRulesReview(ApiClient client, V1SelfSubjectRulesReview body, String pretty) throws ApiException { + return new V1SelfSubjectRulesReview().status(new V1SubjectRulesReviewStatus()); + } + } +} \ No newline at end of file diff --git a/operator/src/test/java/oracle/kubernetes/operator/builders/StubWatchFactory.java b/operator/src/test/java/oracle/kubernetes/operator/builders/StubWatchFactory.java index 8c5e44c7a2a..555167648ca 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/builders/StubWatchFactory.java +++ b/operator/src/test/java/oracle/kubernetes/operator/builders/StubWatchFactory.java @@ -3,6 +3,15 @@ package oracle.kubernetes.operator.builders; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.function.BiFunction; + import com.meterware.simplestub.Memento; import com.meterware.simplestub.StaticStubSupport; import com.squareup.okhttp.Call; @@ -13,15 +22,6 @@ import io.kubernetes.client.util.Watch.Response; import oracle.kubernetes.operator.helpers.Pool; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.function.BiFunction; - /** * A test-time replacement for the factory that creates Watch objects, allowing * tests to specify directly the events they want returned from the Watch. @@ -69,8 +69,8 @@ public static List> getRecordedParameters() { public WatchI createWatch(Pool pool, CallParams callParams, Class responseBodyType, BiFunction function) throws ApiException { getRecordedParameters().add(recordedParams(callParams)); - if (exceptionOnNext == null) - return (WatchI) new WatchStub((List)calls.remove(0)); + if (exceptionOnNext == null && hasDefinedCallResponses()) + return new WatchStub((List)calls.remove(0)); else try { return new ExceptionThrowingWatchStub(exceptionOnNext); } finally { @@ -78,6 +78,10 @@ public WatchI createWatch(Pool pool, CallParams callParams, Cl } } + private boolean hasDefinedCallResponses() { + return calls != null && !calls.isEmpty(); + } + private Map recordedParams(CallParams callParams) { Map result = new HashMap<>(); if (callParams.getResourceVersion() != null) From 2ed09f8d291d7cfb59d52eeff639c8f3b1eafe8f Mon Sep 17 00:00:00 2001 From: Russell Gold Date: Wed, 25 Apr 2018 19:57:45 -0400 Subject: [PATCH 073/186] Use call factory for custom resource definition calls --- .../kubernetes/operator/helpers/CallBuilder.java | 14 ++++++++++++-- .../operator/helpers/SynchronousCallFactory.java | 5 +++++ .../operator/builders/StubWatchFactory.java | 6 +----- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java index e48c17baf4e..df2b6458c54 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java @@ -250,7 +250,7 @@ public Step replaceDomainAsync(String name, String namespace, Domain body, Respo public V1beta1CustomResourceDefinition readCustomResourceDefinition(String name) throws ApiException { ApiClient client = helper.take(); try { - return new ApiextensionsV1beta1Api(client).readCustomResourceDefinition(name, pretty, exact, export); + return CALL_FACTORY.readCustomResourceDefinition(client, name, pretty, exact, export); } finally { helper.recycle(client); } @@ -266,7 +266,7 @@ public V1beta1CustomResourceDefinition createCustomResourceDefinition(V1beta1Cus throws ApiException { ApiClient client = helper.take(); try { - return new ApiextensionsV1beta1Api(client).createCustomResourceDefinition(body, pretty); + return CALL_FACTORY.createCustomResourceDefinition(client, body, pretty); } finally { helper.recycle(client); } @@ -968,6 +968,16 @@ private Step createRequestAsync(ResponseStep next, RequestParams requestP public static class SynchronousCallFactoryImpl implements SynchronousCallFactory { + @Override + public V1beta1CustomResourceDefinition readCustomResourceDefinition(ApiClient client, String name, String pretty, Boolean exact, Boolean export) throws ApiException { + return new ApiextensionsV1beta1Api(client).readCustomResourceDefinition(name, pretty, exact, export); + } + + @Override + public V1beta1CustomResourceDefinition createCustomResourceDefinition(ApiClient client, V1beta1CustomResourceDefinition body, String pretty) throws ApiException { + return new ApiextensionsV1beta1Api(client).createCustomResourceDefinition(body, pretty); + } + @Override public V1SelfSubjectRulesReview createSelfSubjectRulesReview(ApiClient client, V1SelfSubjectRulesReview body, String pretty) throws ApiException { return new AuthorizationV1Api(client).createSelfSubjectRulesReview(body, pretty); diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/SynchronousCallFactory.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/SynchronousCallFactory.java index c5308e636de..d81b331b0f8 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/SynchronousCallFactory.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/SynchronousCallFactory.java @@ -4,11 +4,16 @@ import io.kubernetes.client.ApiException; import io.kubernetes.client.models.V1PersistentVolumeList; import io.kubernetes.client.models.V1SelfSubjectRulesReview; +import io.kubernetes.client.models.V1beta1CustomResourceDefinition; import io.kubernetes.client.models.VersionInfo; import oracle.kubernetes.weblogic.domain.v1.DomainList; public interface SynchronousCallFactory { + V1beta1CustomResourceDefinition readCustomResourceDefinition(ApiClient client, String name, String pretty, Boolean exact, Boolean export) throws ApiException; + + V1beta1CustomResourceDefinition createCustomResourceDefinition(ApiClient client, V1beta1CustomResourceDefinition body, String pretty) throws ApiException; + V1SelfSubjectRulesReview createSelfSubjectRulesReview(ApiClient client, V1SelfSubjectRulesReview body, String pretty) throws ApiException; V1PersistentVolumeList listPersistentVolumes(String _continue, ApiClient client, String pretty, String fieldSelector, Boolean includeUninitialized, String labelSelector, Integer limit, String resourceVersion, Integer timeoutSeconds, Boolean watch) throws ApiException; diff --git a/operator/src/test/java/oracle/kubernetes/operator/builders/StubWatchFactory.java b/operator/src/test/java/oracle/kubernetes/operator/builders/StubWatchFactory.java index 555167648ca..79332341450 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/builders/StubWatchFactory.java +++ b/operator/src/test/java/oracle/kubernetes/operator/builders/StubWatchFactory.java @@ -69,7 +69,7 @@ public static List> getRecordedParameters() { public WatchI createWatch(Pool pool, CallParams callParams, Class responseBodyType, BiFunction function) throws ApiException { getRecordedParameters().add(recordedParams(callParams)); - if (exceptionOnNext == null && hasDefinedCallResponses()) + if (exceptionOnNext == null) return new WatchStub((List)calls.remove(0)); else try { return new ExceptionThrowingWatchStub(exceptionOnNext); @@ -78,10 +78,6 @@ public WatchI createWatch(Pool pool, CallParams callParams, Cl } } - private boolean hasDefinedCallResponses() { - return calls != null && !calls.isEmpty(); - } - private Map recordedParams(CallParams callParams) { Map result = new HashMap<>(); if (callParams.getResourceVersion() != null) From 2da7ebe2d0839beac4ca8c205f2476920d6260fa Mon Sep 17 00:00:00 2001 From: Dongbo Xiao Date: Wed, 25 Apr 2018 19:24:15 -0700 Subject: [PATCH 074/186] Minor fixes Signed-off-by: Dongbo Xiao --- kubernetes/internal/create-weblogic-domain.sh | 3 --- src/integration-tests/bash/run.sh | 5 ++++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/kubernetes/internal/create-weblogic-domain.sh b/kubernetes/internal/create-weblogic-domain.sh index bab7fccfa23..64b107b0732 100755 --- a/kubernetes/internal/create-weblogic-domain.sh +++ b/kubernetes/internal/create-weblogic-domain.sh @@ -557,9 +557,6 @@ function createYamlFiles { sed -i -e "s:# mountPath: mountPath:g" ${apacheOutput} fi - mkdir -p ${loadBalancerVolumePath} - cp ${scriptDir}/custom_mod_wl_apache.conf ${loadBalancerVolumePath}/ - # Apache security file cp ${apacheSecurityInput} ${apacheSecurityOutput} echo Generating ${apacheSecurityOutput} diff --git a/src/integration-tests/bash/run.sh b/src/integration-tests/bash/run.sh index 7453d371ebc..f21da2b9a4d 100755 --- a/src/integration-tests/bash/run.sh +++ b/src/integration-tests/bash/run.sh @@ -113,6 +113,9 @@ # process) or some uuid potentially generated by uuidgen # command or some-such.. # +# LB_TYPE Load balaner type. Should be TRAEFIL, APACHE or NONE. +# Default is TRAEFIK. +# # The following additional overrides are currently only used when # WERCKER=true: # @@ -762,7 +765,7 @@ function run_create_domain_job { local MS_PORT="`dom_get $1 MS_PORT`" local LOAD_BALANCER_WEB_PORT="`dom_get $1 LOAD_BALANCER_WEB_PORT`" local LOAD_BALANCER_DASHBOARD_PORT="`dom_get $1 LOAD_BALANCER_DASHBOARD_PORT`" - local LOAD_BALANCER_VOLUME_PATH="" + # local LOAD_BALANCER_VOLUME_PATH="/scratch/DockerVolume/ApacheVolume" local TMP_DIR="`dom_get $1 TMP_DIR`" local WLS_JAVA_OPTIONS="$JVM_ARGS" From e67e2d36d161b59e2adcad684b10df0e61abaafd Mon Sep 17 00:00:00 2001 From: Dongbo Xiao Date: Wed, 25 Apr 2018 19:49:53 -0700 Subject: [PATCH 075/186] Remove uncompleted unit test Signed-off-by: Dongbo Xiao --- .../create/CreateDomainInputsValidationTest.java | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputsValidationTest.java b/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputsValidationTest.java index 04693d5aa47..34ec5beed3d 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputsValidationTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputsValidationTest.java @@ -47,8 +47,6 @@ public class CreateDomainInputsValidationTest { private static final String PARAM_LOAD_BALANCER = "loadBalancer"; private static final String PARAM_LOAD_BALANCER_WEB_PORT = "loadBalancerWebPort"; private static final String PARAM_LOAD_BALANCER_DASHBOARD_PORT = "loadBalancerDashboardPort"; - private static final String PARAM_LOAD_BALANCER_VOLUME_PATH = "loadBalancerVolumePath"; - private static final String PARAM_LOAD_BALANCER_APP_PREPATH = "loadBalancerAppPrepath"; private static final String PARAM_JAVA_OPTIONS = "javaOptions"; @Before @@ -489,7 +487,7 @@ public void createDomain_with_invalidLoadBalancerWebPort_failsAndReturnsError() @Test public void createDomain_with_missingLoadBalancerDashboardPort_failsAndReturnsError() throws Exception { assertThat( - execCreateDomain(newInputs().loadBalancer(LOAD_BALANCER_TRAEFIK).loadBalancerDashboardPort("")), + execCreateDomain(newInputs().loadBalancerDashboardPort("")), failsAndPrints(paramMissingError(PARAM_LOAD_BALANCER_DASHBOARD_PORT))); } @@ -502,15 +500,6 @@ public void createDomain_with_invalidLoadBalancerDashboardPort_failsAndReturnsEr failsAndPrints(invalidIntegerParamValueError(PARAM_LOAD_BALANCER_DASHBOARD_PORT, val))); } - // commented out for now @Test - public void createDomain_with_loadBalacnerApache_succeeds() throws Exception { - GeneratedDomainYamlFiles - .generateDomainYamlFiles( - newInputs() - .loadBalancer(LOAD_BALANCER_APACHE)) - .remove(); - } - // TBD - shouldn't we allow empty java options? @Test public void createDomain_with_missingJavaOptions_failsAndReturnsError() throws Exception { From be5017e86cc1380ad1cca75fb4c662a17a44452b Mon Sep 17 00:00:00 2001 From: Lily He Date: Thu, 26 Apr 2018 00:11:20 -0700 Subject: [PATCH 076/186] restore genericDelete in cleanup.sh --- src/integration-tests/bash/cleanup.sh | 221 +++++++++++++++++++++----- 1 file changed, 183 insertions(+), 38 deletions(-) diff --git a/src/integration-tests/bash/cleanup.sh b/src/integration-tests/bash/cleanup.sh index 36d4533eeff..9863893487c 100755 --- a/src/integration-tests/bash/cleanup.sh +++ b/src/integration-tests/bash/cleanup.sh @@ -27,10 +27,14 @@ # # The test runs in 4 phases: # -# Phase 1: Delete domain resources with label 'weblogic.domainUID'. +# Phase 1: Delete test kubernetes artifacts with labels. # -# Phase 2: Delete wls operator with lable 'weblogic.operatorName' and -# delete voyager controller. +# Phase 2: Wait 15 seconds to see if stage 1 succeeded, and +# if not, repeatedly search for all test related kubectl +# artifacts and try delete them directly for up to 60 more +# seconds. This phase has no dependency on the +# previous test run's yaml files. It makes no +# attempt to delete artifacts in a particular order. # # Phase 3: Use a kubernetes job to delete the PV directories # on the kubernetes cluster. @@ -57,6 +61,7 @@ function fail { #!/bin/bash # +# Usage: # getResWithLabel outfilename # function getResWithLabel { @@ -75,15 +80,17 @@ function getResWithLabel { } # +# Usage: # deleteResWithLabel outputfile # -function deleteResWithLabel { +function deleteWithOneLabel { echo @@ Delete resources with label $LABEL_SELECTOR. # clean the output file first if [ -e $1 ]; then rm $1 fi + echo @@ Deleting resources with label $LABEL_SELECTOR. getResWithLabel $1 # delete namespaced types cat $1 | awk '{ print $4 }' | grep -v "^$" | sort -u | while read line; do @@ -98,19 +105,23 @@ function deleteResWithLabel { echo "@@ Waiting for pods to stop running." local total=0 - for count in {1..100}; do - pods=($(kubectl get pods --all-namespaces -l weblogic.domainUID -o jsonpath='{range .items[*]}{.metadata.name} {end}')) + local mstart=`date +%s` + local mnow=mstart + local maxwaitsecs=60 + while [ $((mnow - mstart)) -lt $maxwaitsecs ]; do + pods=($(kubectl get pods --all-namespaces -l $LABEL_SELECTOR -o jsonpath='{range .items[*]}{.metadata.name} {end}')) total=${#pods[*]} if [ $total -eq 0 ] ; then break else - echo "@@ There are still $total running pods with label $LABEL_SELECTOR." + echo "@@ There are $total running pods with label $LABEL_SELECTOR." fi sleep 3 + mnow=`date +%s` done if [ $total -gt 0 ]; then - echo "Warning: after waiting 300 seconds, there are still $total running pods with label $LABEL_SELECTOR." + echo "Warning: after waiting $maxwaitsecs seconds, there are still $total running pods with label $LABEL_SELECTOR." fi } @@ -122,6 +133,7 @@ function deleteVoyagerController { } # +# Usage: # deleteNamespaces outputfile # function deleteNamespaces { @@ -132,46 +144,179 @@ function deleteNamespaces { done } -echo @@ Starting cleanup. -script="${BASH_SOURCE[0]}" -scriptDir="$( cd "$(dirname "${script}")" > /dev/null 2>&1 ; pwd -P)" -echo "@@ RESULT_ROOT=$RESULT_ROOT TMP_DIR=$TMP_DIR RESULT_DIR=$RESULT_DIR PROJECT_ROOT=$PROJECT_ROOT" +function deleteWithLabels { + NAMESPACED_TYPES="pod,job,deploy,rs,service,pvc,ingress,cm,serviceaccount,role,rolebinding,secret" -mkdir -p $TMP_DIR || fail No permision to create directory $TMP_DIR + HANDLE_VOYAGER="false" + VOYAGER_ING_NAME="ingresses.voyager.appscode.com" + if [ `kubectl get crd $VOYAGER_ING_NAME --ignore-not-found | grep $VOYAGER_ING_NAME | wc -l` = 1 ]; then + NAMESPACED_TYPES="$VOYAGER_ING_NAME,$NAMESPACED_TYPES" + HANDLE_VOYAGER="true" + fi -NAMESPACED_TYPES="pod,job,deploy,rs,service,pvc,ingress,cm,serviceaccount,role,rolebinding,secret" + DOMAIN_CRD="domains.weblogic.oracle" + if [ `kubectl get crd $DOMAIN_CRD --ignore-not-found | grep $DOMAIN_CRD | wc -l` = 1 ]; then + NAMESPACED_TYPES="$DOMAIN_CRD,$NAMESPACED_TYPES" + fi -HANDLE_VOYAGER="false" -VOYAGER_ING_NAME="ingresses.voyager.appscode.com" -if [ `kubectl get crd $VOYAGER_ING_NAME --ignore-not-found | grep $VOYAGER_ING_NAME | wc -l` = 1 ]; then - NAMESPACED_TYPES="$VOYAGER_ING_NAME,$NAMESPACED_TYPES" - HANDLE_VOYAGER="true" -fi + NOT_NAMESPACED_TYPES="pv,crd,clusterroles,clusterrolebindings" -DOMAIN_CRD="domains.weblogic.oracle" -if [ `kubectl get crd $DOMAIN_CRD --ignore-not-found | grep $DOMAIN_CRD | wc -l` = 1 ]; then - NAMESPACED_TYPES="$DOMAIN_CRD,$NAMESPACED_TYPES" -fi + tempfile="/tmp/$(basename $0).tmp.$$" # == /tmp/[script-file-name].tmp.[pid] -NOT_NAMESPACED_TYPES="pv,crd,clusterroles,clusterrolebindings" + echo @@ Deleting domain resources. + LABEL_SELECTOR="weblogic.domainUID" + deleteWithOneLabel "$tempfile-0" -tempfile="/tmp/$(basename $0).tmp.$$" # == /tmp/[script-file-name].tmp.[pid] + echo @@ Deleting wls operator resources. + LABEL_SELECTOR="weblogic.operatorName" + deleteWithOneLabel "$tempfile-1" -echo @@ Deleting domain resources. -LABEL_SELECTOR="weblogic.domainUID" -deleteResWithLabel "$tempfile-0" -deleteNamespaces "$tempfile-0" + deleteNamespaces "$tempfile-0" + deleteNamespaces "$tempfile-1" -echo @@ Deleting wls operator resources. -LABEL_SELECTOR="weblogic.operatorName" -deleteResWithLabel "$tempfile-1" -deleteNamespaces "$tempfile-1" + echo @@ Deleting voyager controller. + if [ "$HANDLE_VOYAGER" = "true" ]; then + deleteVoyagerController + fi +} -echo @@ Deleting voyager controller. -if [ "$HANDLE_VOYAGER" = "true" ]; then - deleteVoyagerController -fi +# function genericDelete +# +# This function is a 'generic kubernetes delete' that takes three arguments: +# +# arg1: Comma separated list of types of kubernetes namespaced types to search/delete. +# example: "all,cm,pvc,ns,roles,rolebindings,secrets" +# +# arg2: Comma separated list of types of kubernetes non-namespaced types to search/delete. +# example: "crd,pv,clusterroles,clusterrolebindings" +# +# arg3: '|' (pipe) separated list of keywords. +# Artifacts with a label or name that contains one +# or more of the keywords are delete candidates. +# example: "logstash|kibana|elastisearch|weblogic|elk|domain" +# +# It runs in two stages: +# In the first, wait to see if artifacts delete on their own. +# In the second, try to delete any leftovers. +# +function genericDelete { + + for iteration in first second; do + # In the first iteration, we wait to see if artifacts delete. + # in the second iteration, we try to delete any leftovers. + + if [ "$iteration" = "first" ]; then + local maxwaitsecs=15 + else + local maxwaitsecs=60 + fi + + echo "@@ Waiting up to $maxwaitsecs seconds for ${1:?} and ${2:?} artifacts that contain string ${3:?} to delete." + + local artcount_no + local artcount_yes + local artcount_total + local resfile_no + local resfile_yes + + local mstart=`date +%s` + + while : ; do + resfile_no="$TMP_DIR/kinv_filtered_nonamespace.out.tmp" + resfile_yes="$TMP_DIR/kinv_filtered_yesnamespace.out.tmp" + + # leftover namespaced artifacts + kubectl get $1 \ + -o=jsonpath='{range .items[*]}{.metadata.namespace}{" "}{.kind}{"/"}{.metadata.name}{"\n"}{end}' \ + --all-namespaces=true 2>&1 \ + | egrep -e "($3)" | sort > $resfile_yes 2>&1 + artcount_yes="`cat $resfile_yes | wc -l`" + + # leftover non-namespaced artifacts + kubectl get $2 \ + -o=jsonpath='{range .items[*]}{.kind}{"/"}{.metadata.name}{"\n"}{end}' \ + --all-namespaces=true 2>&1 \ + | egrep -e "($3)" | sort > $resfile_no 2>&1 + artcount_no="`cat $resfile_no | wc -l`" + + artcount_total=$((artcount_yes + artcount_no)) + + mnow=`date +%s` + + if [ $((artcount_total)) -eq 0 ]; then + echo "@@ No artifacts found." + return 0 + fi + + if [ "$iteration" = "first" ]; then + # in the first iteration we just wait to see if artifacts go away on there own + + echo "@@ Waiting for $artcount_total artifacts to delete. Wait time $((mnow - mstart)) seconds (max=$maxwaitsecs). Waiting for:" + + cat $resfile_yes | awk '{ print "n=" $1 " " $2 }' + cat $resfile_no | awk '{ print $1 }' + + else + # in the second thirty seconds we try to delete remaining artifacts + + echo "@@ Trying to delete ${artcount_total} leftover artifacts, including ${artcount_yes} namespaced artifacts and ${artcount_no} non-namespaced artifacts, wait time $((mnow - mstart)) seconds (max=$maxwaitsecs)." + + if [ ${artcount_yes} -gt 0 ]; then + cat "$resfile_yes" | while read line; do + local args="`echo \"$line\" | awk '{ print "-n " $1 " delete " $2 " --ignore-not-found" }'`" + echo "kubectl $args" + kubectl $args + done + fi + + if [ ${artcount_no} -gt 0 ]; then + cat "$resfile_no" | while read line; do + echo "kubectl delete $line --ignore-not-found" + kubectl delete $line --ignore-not-found + done + fi + + fi + + if [ $((mnow - mstart)) -gt $((maxwaitsecs)) ]; then + if [ "$iteration" = "first" ]; then + echo "@@ Warning: ${maxwaitsecs} seconds reached. Will try deleting unexpected resources via kubectl delete." + else + echo "@@ Error: ${maxwaitsecs} seconds reached and possibly ${artcount_total} artifacts remaining. Giving up." + fi + break + fi + + sleep 5 + done + done + return 1 +} + +echo @@ Starting cleanup. +script="${BASH_SOURCE[0]}" +scriptDir="$( cd "$(dirname "${script}")" > /dev/null 2>&1 ; pwd -P)" + +echo "@@ RESULT_ROOT=$RESULT_ROOT TMP_DIR=$TMP_DIR RESULT_DIR=$RESULT_DIR PROJECT_ROOT=$PROJECT_ROOT" + +mkdir -p $TMP_DIR || fail No permision to create directory $TMP_DIR + +# first, try to delete with labels since the conversion is that all created resources need to +# have the proper label(s) +echo @@ Starting deleteWithLabels +deleteWithLabels + +# second, try a generic delete in case there are some leftover resources, this runs in two phases: +# phase 1: wait to see if artifacts dissappear naturally due to the above orderlyDelete +# phase 2: kubectl delete left over artifacts +# arguments +# arg1 - namespaced kubernetes artifacts +# arg2 - non-namespaced artifacts +# arg3 - keywords in deletable artificats +echo @@ Starting genericDelete +genericDelete "all,cm,pvc,roles,rolebindings,serviceaccount,secrets" "crd,pv,ns,clusterroles,clusterrolebindings" "logstash|kibana|elastisearch|weblogic|elk|domain|traefik|voyager" +SUCCESS="$?" # Delete pv directories using a job (/scratch maps to PV_ROOT on the k8s cluster machines). From 6f493e785402111e4289a53c80801a092e5d94ef Mon Sep 17 00:00:00 2001 From: Lily He Date: Thu, 26 Apr 2018 02:06:29 -0700 Subject: [PATCH 077/186] minor change in create-weblogic-domain.sh --- kubernetes/internal/create-weblogic-domain.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/kubernetes/internal/create-weblogic-domain.sh b/kubernetes/internal/create-weblogic-domain.sh index bc76c99c3c2..56cc5c0d704 100755 --- a/kubernetes/internal/create-weblogic-domain.sh +++ b/kubernetes/internal/create-weblogic-domain.sh @@ -648,18 +648,18 @@ function setupVoyagerLoadBalancer { # deploy Voyager Ingress resource kubectl apply -f ${voyagerOutput} - echo Checking Voyager deploy + echo Checking Voyager Ingress resource local maxwaitsecs=100 local mstart=`date +%s` while : ; do local mnow=`date +%s` - local vdep=`kubectl get deploy -n ${namespace} | grep ${domainUID}-voyager | wc | awk ' { print $1; } '` + local vdep=`kubectl get ingresses.voyager.appscode.com -n ${namespace} | grep ${domainUID}-voyager | wc | awk ' { print $1; } '` if [ "$vdep" = "1" ]; then - echo 'The deployment ${domainUID}-voyager is created successful.' + echo 'The Voyager Ingress resource ${domainUID}-voyager is created successfully.' break fi if [ $((mnow - mstart)) -gt $((maxwaitsecs)) ]; then - fail "The deployment ${domainUID}-voyager was not created." + fail "The Voyager Ingress resource ${domainUID}-voyager was not created." fi sleep 5 done @@ -671,7 +671,7 @@ function setupVoyagerLoadBalancer { local mnow=`date +%s` local vscv=`kubectl get service ${domainUID}-voyager-stats -n ${namespace} | grep ${domainUID}-voyager-stats | wc | awk ' { print $1; } '` if [ "$vscv" = "1" ]; then - echo 'The service ${domainUID}-voyager-stats is created successful.' + echo 'The service ${domainUID}-voyager-stats is created successfully.' break fi if [ $((mnow - mstart)) -gt $((maxwaitsecs)) ]; then From 8dd597af9808796251cd418b42c9032090d83e66 Mon Sep 17 00:00:00 2001 From: Dongbo Xiao Date: Thu, 26 Apr 2018 03:03:07 -0700 Subject: [PATCH 078/186] Minor fix in cleanup.sh Signed-off-by: Dongbo Xiao --- src/integration-tests/bash/cleanup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/integration-tests/bash/cleanup.sh b/src/integration-tests/bash/cleanup.sh index 52efd5109df..06decb296a6 100755 --- a/src/integration-tests/bash/cleanup.sh +++ b/src/integration-tests/bash/cleanup.sh @@ -235,7 +235,7 @@ function orderlyDelete { kubectl -n $curns delete clusterrolebinding ${curdomain}-cluster-1-traefik --ignore-not-found=true kubectl -n $curns delete deploy ${curdomain}-apache-webtier --ignore-not-found=true - kubectl -n $curns delete service ${curdomain}-external-apache-webtier-service --ignore-not-found=true + kubectl -n $curns delete service ${curdomain}-apache-webtier --ignore-not-found=true kubectl -n $curns delete serviceaccount ${curdomain}-apache-webtier --ignore-not-found=true kubectl -n $curns delete clusterrole ${curdomain}-apache-webtier --ignore-not-found=true kubectl -n $curns delete clusterrolebinding ${curdomain}-apache-webtier --ignore-not-found=true From c15c3f4a49cecb54b062e85b5fc04bf09e62a4ec Mon Sep 17 00:00:00 2001 From: Rosemary Marano Date: Thu, 26 Apr 2018 11:26:39 -0400 Subject: [PATCH 079/186] first pass edits for May release --- README.md | 41 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 0edf61eb4dc..e5dd209267f 100644 --- a/README.md +++ b/README.md @@ -4,17 +4,17 @@ Built with [Wercker](http://www.wercker.com) [![wercker status](https://app.wercker.com/status/68ce42623fce7fb2e52d304de8ea7530/m/develop "wercker status")](https://app.wercker.com/project/byKey/68ce42623fce7fb2e52d304de8ea7530) -Many organizations are exploring, testing, or actively moving application workloads into a cloud environment, either in house or using an external cloud provider. Kubernetes has emerged as a leading cloud platform and is seeing widespread adoption. But a new computing model does not necessarily mean new applications or workloads; many of the existing application workloads running in environments designed and built over many years, before the ‘cloud era’, are still mission critical today. As such, there is a lot of interest in moving such workloads into a cloud environment, like Kubernetes, without forcing application rewrites, retesting and additional process and cost. There is also a desire to not just run the application in the new environment, but to run it ‘well’ – to adopt some of the idioms of the new environment and to realize some of the benefits of that new environment. +Many organizations are exploring, testing, or actively moving application workloads into a cloud environment, either in house or using an external cloud provider. Kubernetes has emerged as a leading cloud platform and is seeing widespread adoption. But a new computing model does not necessarily mean new applications or workloads; many of the existing application workloads running in environments designed and built over many years, before the ‘cloud era’, are still mission critical today. As such, there is a lot of interest in moving such workloads into a cloud environment, like Kubernetes, without forcing application rewrites, retesting, and additional process and cost. There is also a desire to not just run the application in the new environment, but to run it ‘well’ – to adopt some of the idioms of the new environment and to realize some of the benefits of that new environment. -Oracle has been working with the WebLogic community to find ways to make it as easy as possible for organizations using WebLogic Server to run important workloads, to move those workloads into the cloud. One aspect of that effort is the creation of the Oracle WebLogic Server Kubernetes Operator. The Technology Preview release of the Operator provides a number of features to assist with the management of WebLogic domains in a Kubernetes environment, including: +Oracle has been working with the WebLogic community to find ways to make it as easy as possible for organizations using WebLogic Server to run important workloads, to move those workloads into the cloud. One aspect of that effort is the creation of the Oracle WebLogic Server Kubernetes Operator. This release of the Operator provides a number of features to assist with the management of WebLogic domains in a Kubernetes environment, including: -* A mechanism to create a WebLogic domain on a Kubernetes persistent volume -* A mechanism to define a WebLogic domain as a Kubernetes resource (using a Kubernetes custom resource definition) -* The ability to automatically start servers based on declarative startup parameters and desired states -* The ability to automatically expose the WebLogic Server Administration Console outside the Kubernetes cluster (if desired) -* The ability to automatically expose T3 channels outside the Kubernetes domain (if desired) -* The ability to automatically expose HTTP paths on a WebLogic domain outside the Kubernetes domain with load balancing, and to update the load balancer when Managed Servers in the WebLogic domain are started or stopped -* The ability to scale a WebLogic domain by starting and stopping Managed Servers on demand, or by integrating with a REST API to initiate scaling based on WLDF, Prometheus/Grafana or other rules +* A mechanism to create a WebLogic domain on a Kubernetes persistent volume. +* A mechanism to define a WebLogic domain as a Kubernetes resource (using a Kubernetes custom resource definition). +* The ability to automatically start servers based on declarative startup parameters and desired states. +* The ability to automatically expose the WebLogic Server Administration Console outside the Kubernetes cluster (if desired). +* The ability to automatically expose T3 channels outside the Kubernetes domain (if desired). +* The ability to automatically expose HTTP paths on a WebLogic domain outside the Kubernetes domain with load balancing, and to update the load balancer when Managed Servers in the WebLogic domain are started or stopped. +* The ability to scale a WebLogic domain by starting and stopping Managed Servers on demand, or by integrating with a REST API to initiate scaling based on WLDF, Prometheus/Grafana, or other rules. * The ability to publish Operator and WebLogic Server logs into ElasticSearch and interact with them in Kibana. As part of Oracle’s ongoing commitment to open source in general, and to Kubernetes and the Cloud Native Computing Foundation specifically, Oracle has open sourced the Operator and is committed to enhancing it with additional features. Oracle welcomes feedback, issues, pull requests, and feature requests from the WebLogic community. @@ -41,7 +41,7 @@ In this documentation, several important terms are used and are intended to have Before using the operator, it is highly recommended that you read the [design philosophy](site/design.md) to develop an understanding of the operator's design, and the [architectural overview](site/architecture.md) to understand its architecture, including how WebLogic domains are deployed in Kubernetes using the operator. It is also worth reading the details of the [Kubernetes RBAC definitions](site/rbac.md) required by the operator. # Exposing applications outside the Kubernetes cluster -The operator can configure services to expose WebLogic applications and features outside of the Kubernetes cluster. Care should be taken when exposing anything externally to ensure that the appropriate security considerations are taken into account. There is no significant difference between a WebLogic domain running in a Kubernetes cluster and a domain running in a traditional data center in this regard. The same kinds of considerations should be taken into account, for example: +The operator can configure services to expose WebLogic applications and features outside of the Kubernetes cluster. Care should be taken when exposing anything externally to ensure that the appropriate security considerations are taken into account. In this regard, there is no significant difference between a WebLogic domain running in a Kubernetes cluster and a domain running in a traditional data center. The same kinds of considerations should be taken into account, for example: * Only expose those protocols and ports that need to be exposed. * Use secure protocols (HTTPS, T3S, and such). @@ -68,12 +68,11 @@ The Oracle WebLogic Server Kubernetes Operator has the following requirements: # Restrictions -The following features are not certified or supported in the Technology Preview release at the time of writing: +The following features are not certified or supported in this release: * Whole Server Migration * Consensus Leasing * Node Manager (although it is used internally for the liveness probe and to start WebLogic Server instances) -* Dynamic domains (the current certification only covers configured clusters; certification of dynamic clusters is planned at a future date) * Multicast * If using a `hostPath` persistent volume, then it must have read/write/many permissions for all container/pods in the WebLogic Server deployment * Multitenancy @@ -97,9 +96,9 @@ If you would rather see the developers demonstrating the operator rather than re * [Installing the operator](https://youtu.be/B5UmY2xAJnk) includes the installation and also shows using the operator's REST API. * [Creating a WebLogic domain with the operator](https://youtu.be/Ey7o8ldKv9Y) shows the creation of two WebLogic domains including accessing the Administration Console and looking at the various resources created in Kubernetes - services, Ingresses, pods, load balancers, and such. -* [Deploying a web application, scaling a WebLogic cluster with the operator and verifying load balancing](https://youtu.be/hx4OPhNFNDM) +* [Deploying a web application, scaling a WebLogic cluster with the operator and verifying load balancing](https://youtu.be/hx4OPhNFNDM). * [Using WLST against a domain running in Kubernetes](https://youtu.be/eY-KXEk8rI4) shows how to create a data source for an Oracle database that is also running in Kubernetes. -* [Scaling a WebLogic cluster with WLDF](https://youtu.be/Q8iZi2e9HvU) +* [Scaling a WebLogic cluster with WLDF](https://youtu.be/Q8iZi2e9HvU). * Watch this space, more to come! Like what you see? Read on for all the nitty-gritty details... @@ -125,7 +124,7 @@ All of the [installation steps are explained in detail here](site/installation.m ## Using the operator's REST services -The operator provides a REST API that can be used to obtain information about the configuration and to initiate scaling actions. Please refer to [Using the operator's REST services](site/rest.md) for details about how to use the REST APIs. +The operator provides a REST API that you can use to obtain information about the configuration and to initiate scaling actions. Please refer to [Using the operator's REST services](site/rest.md) for details about how to use the REST APIs. ## Creating a WebLogic domain with the operator @@ -158,14 +157,14 @@ Please refer to [Shutting down a domain](site/shutdown-domain.md) for informatio ## Load balancing with the Traefik Ingress controller -The initial Technology Preview release of the operator supports only the Traefik load balancer/Ingress controller. Support for other load balancers is planned in the future. -Please refer to [Load balancing with Traefik](site/traefik.md) for information about current capabilities. +This release of the operator supports only the Traefik load balancer/Ingress controller. Support for other load balancers is planned in the future. +Please refer to [Load balancing with Traefik](site/traefik.md) for information about the current capabilities. [comment]: # (Exporting operator logs to ELK. The operator provides an option to export its log files to the ELK stack. Please refer to [ELK integration]site/elk.md for information about this capability.) ## Removing a domain -To permanently remove a domain from a Kubernetes cluster, first shut down the domain using the instructions provided above in the section titled “Shutting down a domain”, then remove the persistent volume claim and the persistent volume using these commands: +To permanently remove a domain from a Kubernetes cluster, first shut down the domain using the instructions provided in [Shutting down a domain](site/shutdown-domain.md), then remove the persistent volume claim and the persistent volume using these commands: ``` kubectl delete pvc PVC-NAME -n NAMESPACE @@ -193,7 +192,7 @@ To remove more than one operator, repeat these steps for each operator namespace # Recent changes -See [Recent changes](site/recent-changes.md) for recent changes to the operator, including any backwards incompatible changes. +See [Recent changes](site/recent-changes.md) for recent changes to the operator, including any backward incompatible changes. # Developer guide @@ -214,7 +213,7 @@ If you have any questions about a possible submission, feel free to open an issu Pull requests can be made under The Oracle Contributor Agreement (OCA), which is available at [https://www.oracle.com/technetwork/community/oca-486395.html](https://www.oracle.com/technetwork/community/oca-486395.html). -For pull requests to be accepted, the bottom of the commit message must have the following line using the contributor’s name and e-mail address as it appears in the OCA Signatories list. +For pull requests to be accepted, the bottom of the commit message must have the following line, using the contributor’s name and e-mail address as it appears in the OCA Signatories list. ``` Signed-off-by: Your Name @@ -231,7 +230,7 @@ Only pull requests from committers that can be verified as having signed the OCA ## Pull request process * Fork the repository. -* Create a branch in your fork to implement the changes. We recommend using the issue number as part of your branch name, e.g. `1234-fixes`. +* Create a branch in your fork to implement the changes. We recommend using the issue number as part of your branch name, for example, `1234-fixes`. * Ensure that any documentation is updated with the changes that are required by your fix. * Ensure that any samples are updated if the base image has been changed. * Submit the pull request. Do not leave the pull request blank. Explain exactly what your changes are meant to do and provide simple steps on how to validate your changes. Ensure that you reference the issue you created as well. We will assign the pull request to 2-3 people for review before it is merged. From d296c2fe915e4c1e12bf9800284da6564972e244 Mon Sep 17 00:00:00 2001 From: Dongbo Xiao Date: Thu, 26 Apr 2018 08:43:46 -0700 Subject: [PATCH 080/186] Deploy RBAC yaml for apache Signed-off-by: Dongbo Xiao --- kubernetes/internal/create-weblogic-domain.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/kubernetes/internal/create-weblogic-domain.sh b/kubernetes/internal/create-weblogic-domain.sh index 64b107b0732..e2638d55009 100755 --- a/kubernetes/internal/create-weblogic-domain.sh +++ b/kubernetes/internal/create-weblogic-domain.sh @@ -707,6 +707,21 @@ function setupApacheLoadBalancer { apacheName="${domainUID}-apache-webtier" + echo Setting up apache security + kubectl apply -f ${apacheSecurityOutput} + + echo Checking the cluster role ${apacheName} was created + CLUSTERROLE=`kubectl get clusterroles | grep ${apacheName} | wc | awk ' { print $1; } '` + if [ "$CLUSTERROLE" != "1" ]; then + fail "The cluster role ${apacheName} was not created" + fi + + echo Checking the cluster role binding ${apacheName} was created + CLUSTERROLEBINDING=`kubectl get clusterrolebindings | grep ${apacheName} | wc | awk ' { print $1; } '` + if [ "$CLUSTERROLEBINDING" != "1" ]; then + fail "The cluster role binding ${apacheName} was not created" + fi + echo Deploying apache kubectl apply -f ${apacheOutput} From 28abfe43761f4d413cd1c56e32b85da87090d9f2 Mon Sep 17 00:00:00 2001 From: Dongbo Xiao Date: Thu, 26 Apr 2018 09:38:24 -0700 Subject: [PATCH 081/186] Add more comments for the two new inputs properties Signed-off-by: Dongbo Xiao --- kubernetes/create-weblogic-domain-inputs.yaml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/kubernetes/create-weblogic-domain-inputs.yaml b/kubernetes/create-weblogic-domain-inputs.yaml index 5459af9c6ec..d29e09e3c27 100644 --- a/kubernetes/create-weblogic-domain-inputs.yaml +++ b/kubernetes/create-weblogic-domain-inputs.yaml @@ -89,14 +89,17 @@ exposeAdminNodePort: false # Name of the domain namespace namespace: default -# Load balancer to deploy. Supported values are:TRAEFIK, APACHE, NONE +# Load balancer to deploy. Supported values are: TRAEFIK, APACHE, NONE loadBalancer: TRAEFIK -# Load balancer app prepath +# Load balancer app prepath used for APACHE load balancer +# This defines the /location in the built-in Apache plugin configuration module for WebLogic loadBalancerAppPrepath: / -# Docker volume path for APACHE -# By default, the VolumePath is empty, which will cause the volume mount be disabled +# Docker volume path for APACHE. By default, it is empty, which causes the volume mount be +# disabled and, thereforem the built-in Apache plugin config be used. +# Use this to provide your own Apache plugin configuration as needed; simply define this +# path and put your own custom_mod_wl_apache.conf file under this path. loadBalancerVolumePath: # Load balancer web port From 20668842ba9b8b00164666e9c25d460fc14dc0b4 Mon Sep 17 00:00:00 2001 From: Ryan Eberhard Date: Thu, 26 Apr 2018 12:41:47 -0400 Subject: [PATCH 082/186] Update delete-weblogic-domain-resources.sh --- kubernetes/delete-weblogic-domain-resources.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/delete-weblogic-domain-resources.sh b/kubernetes/delete-weblogic-domain-resources.sh index d92782bd768..1cab748c732 100755 --- a/kubernetes/delete-weblogic-domain-resources.sh +++ b/kubernetes/delete-weblogic-domain-resources.sh @@ -204,7 +204,7 @@ function deleteDomains { # for each namespace with leftover resources, try delete them cat $tempfile | awk '{ print $4 }' | grep -v "^$" | sort -u | while read line; do if [ "$test_mode" = "true" ]; then - echo kubectl -n $line delete $NAMESPACED_TYPES -l "$LABEL_SELECTOR" + echo kubectl -n $line delete $NAMESPACED_TYPES -l "$LABEL_SELECTOR" else kubectl -n $line delete $NAMESPACED_TYPES -l "$LABEL_SELECTOR" fi From fc225c812cf191a5a531f196b7cf74636952ffa1 Mon Sep 17 00:00:00 2001 From: Ryan Eberhard Date: Thu, 26 Apr 2018 12:52:42 -0400 Subject: [PATCH 083/186] Update cleanup.sh --- src/integration-tests/bash/cleanup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/integration-tests/bash/cleanup.sh b/src/integration-tests/bash/cleanup.sh index 6e974c66e17..27534744fdb 100755 --- a/src/integration-tests/bash/cleanup.sh +++ b/src/integration-tests/bash/cleanup.sh @@ -457,7 +457,7 @@ deleteWithLabels # arg3 - keywords in deletable artificats echo @@ Starting genericDelete -genericDelete "all,cm,pvc,roles,rolebindings,serviceaccount,secrets" "crd,pv,ns,clusterroles,clusterrolebindings" "logstash|kibana|elastisearch|weblogic|elk|domain|traefik|voyater|apache-webtier" +genericDelete "all,cm,pvc,roles,rolebindings,serviceaccount,secrets" "crd,pv,ns,clusterroles,clusterrolebindings" "logstash|kibana|elastisearch|weblogic|elk|domain|traefik|voyager|apache-webtier" SUCCESS="$?" # Delete pv directories using a job (/scratch maps to PV_ROOT on the k8s cluster machines). From e0e96369277181ea579e28680255b81e42777fdc Mon Sep 17 00:00:00 2001 From: Ryan Eberhard Date: Thu, 26 Apr 2018 12:54:15 -0400 Subject: [PATCH 084/186] Update run.sh --- src/integration-tests/bash/run.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/integration-tests/bash/run.sh b/src/integration-tests/bash/run.sh index 1320b017798..c6dd3fbcee9 100755 --- a/src/integration-tests/bash/run.sh +++ b/src/integration-tests/bash/run.sh @@ -78,7 +78,7 @@ # See "Directory Configuration and Structure" below for # defaults and a detailed description of test directories. # -# LB_TYPE Load balancer type. Can be 'TRAEFIK', 'VOYAGER'. +# LB_TYPE Load balancer type. Can be 'TRAEFIK', 'VOYAGER', or 'APACHE'. # Default is 'TRAEFIK'. # # VERBOSE Set to 'true' to echo verbose output to stdout. @@ -116,9 +116,6 @@ # process) or some uuid potentially generated by uuidgen # command or some-such.. # -# LB_TYPE Load balaner type. Should be TRAEFIL, APACHE or NONE. -# Default is TRAEFIK. -# # The following additional overrides are currently only used when # WERCKER=true: # From 05e8429fb3aaaefeb5fb4d6f440eb2b787d50081 Mon Sep 17 00:00:00 2001 From: Ryan Eberhard Date: Thu, 26 Apr 2018 13:08:30 -0400 Subject: [PATCH 085/186] Resolve merge conflicts --- kubernetes/internal/create-weblogic-domain.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kubernetes/internal/create-weblogic-domain.sh b/kubernetes/internal/create-weblogic-domain.sh index 31daeaa04e0..4e551e1fdad 100755 --- a/kubernetes/internal/create-weblogic-domain.sh +++ b/kubernetes/internal/create-weblogic-domain.sh @@ -354,7 +354,8 @@ function initialize { apacheInput="${scriptDir}/weblogic-domain-apache-template.yaml" if [ ! -f ${apacheInput} ]; then validationError "The template file ${apacheInput} for generating the apache-webtier deployment was not found" - + fi + voyagerInput="${scriptDir}/voyager-ingress-template.yaml" if [ ! -f ${voyagerInput} ]; then validationError "The template file ${voyagerInput} for generating the Voyager Ingress was not found" From 82ce098ca303353a1d0b95dd0561ad1a741b7577 Mon Sep 17 00:00:00 2001 From: Ryan Eberhard Date: Thu, 26 Apr 2018 13:22:22 -0400 Subject: [PATCH 086/186] Handle wercker undefined variable --- wercker.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/wercker.yml b/wercker.yml index e3c5c9c3ccc..abffa437164 100644 --- a/wercker.yml +++ b/wercker.yml @@ -89,8 +89,7 @@ integration-test: function finish { exit_code=$? - varname=${WERCKER_DEPLOYTARGET_NAME} - export ${!varname}-result="$exit_code" + export ${WERCKER_DEPLOYTARGET_NAME:-integration-test}-result="$exit_code" cleanup_and_store exit 0 From 4efac2b1b994c1064d05578a86dc9bc2c4784eea Mon Sep 17 00:00:00 2001 From: Ryan Eberhard Date: Thu, 26 Apr 2018 13:35:14 -0400 Subject: [PATCH 087/186] Try using common export variable name --- wercker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wercker.yml b/wercker.yml index abffa437164..3b90e08607f 100644 --- a/wercker.yml +++ b/wercker.yml @@ -89,7 +89,7 @@ integration-test: function finish { exit_code=$? - export ${WERCKER_DEPLOYTARGET_NAME:-integration-test}-result="$exit_code" + export integration-test-result="$exit_code" cleanup_and_store exit 0 From 691d5ec68d530b5e06b27780aa767b6b695ba3f8 Mon Sep 17 00:00:00 2001 From: Ryan Eberhard Date: Thu, 26 Apr 2018 13:42:23 -0400 Subject: [PATCH 088/186] Update identifier --- wercker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wercker.yml b/wercker.yml index 3b90e08607f..7ce2e6451ac 100644 --- a/wercker.yml +++ b/wercker.yml @@ -89,7 +89,7 @@ integration-test: function finish { exit_code=$? - export integration-test-result="$exit_code" + export INTEGRATION_TEST_RESULT="$exit_code" cleanup_and_store exit 0 From bd00ee809dbf97c2246e17c5a225b456f4df7a83 Mon Sep 17 00:00:00 2001 From: Ryan Eberhard Date: Thu, 26 Apr 2018 14:02:10 -0400 Subject: [PATCH 089/186] Move to kubernetes client 1.0.0 --- operator/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/operator/pom.xml b/operator/pom.xml index 646842f6468..4e92dbcc92e 100644 --- a/operator/pom.xml +++ b/operator/pom.xml @@ -244,7 +244,7 @@ io.kubernetes client-java - 1.0.0-beta4 + 1.0.0 javax From 4d0eed057acaafd7f09aad8c788b5dfe36652e0c Mon Sep 17 00:00:00 2001 From: Rosemary Marano Date: Thu, 26 Apr 2018 14:53:11 -0400 Subject: [PATCH 090/186] first pass edit on architecture doc --- site/architecture.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/site/architecture.md b/site/architecture.md index 5f57966ed65..b9ff2261eee 100644 --- a/site/architecture.md +++ b/site/architecture.md @@ -19,7 +19,7 @@ The Kubernetes cluster has several namespaces. Components may be deployed into * The operator is deployed into its own namespace. If the ELK integration option is configured, then a logstash pod will also be deployed in the operator’s namespace. * WebLogic domains will be deployed into various namespaces. There can be more than one domain in a namespace if desired. There is no limit on the number of domains or namespaces that an operator can manage. Note that there can be more than one operator in a Kubernetes cluster, but each operator is configured with a list of the specific namespaces that it is responsible for. The operator will not take any action on any domain that is not in one of the namespaces the operator is configured to manage. -* If the ELK integration option is configured, ElasticSearch and Kibana will be deployed in the `default` namespace. +* If the ELK integration option is configured, Elasticsearch and Kibana will be deployed in the `default` namespace. * If a load balancer is configured, it will be deployed in the `kube-system` namespace. ## Domain architecture @@ -32,14 +32,14 @@ This diagram shows the following details: * A persistent volume is created using one of the available providers. The chosen provider must support “Read Write Many” access mode. A persistent volume claim is created to claim space in that persistent volume. Both the persistent volume and the persistent volume claim are labeled with `weblogic.domainUID` and these labels allow the operator to find the correct volume for a particular domain. There must be a different persistent volume for each domain. The shared state on the persistent volume include the “domain” directory, the “applications” directory, a directory for storing logs and a directory for any file-based persistence stores. -* A pod is created for the WebLogic Administration Server. This pod is labeled with `weblogic.domainUID`, `weblogic.serverName` and `weblogic.domainName`. One container runs in this pod. WebLogic Node Manager and Administration Server processes are run inside this container. The Node Manager process is used as an internal implementation detail for the liveness probe, for patching and to provide monitoring and control capabilities to the Administration Console. It is not intended to be used for other purposes, and it may be removed in some future release. +* A pod is created for the WebLogic Administration Server. This pod is labeled with `weblogic.domainUID`, `weblogic.serverName` and `weblogic.domainName`. One container runs in this pod. WebLogic Node Manager and Administration Server processes are run inside this container. The Node Manager process is used as an internal implementation detail for the liveness probe, for patching, and to provide monitoring and control capabilities to the Administration Console. It is not intended to be used for other purposes, and it may be removed in some future release. * A `ClusterIP` type service is created for the Administration Server pod. This service provides a stable, well-known network (DNS) name for the Administration Server. This name is derived from the `domainUID` and the Administration Server name, and it is known before starting up any pod. The Administration Server `ListenAddress` is set to this well-known name. `ClusterIP` type services are only visible inside the Kubernetes cluster. They are used to provide the well-known names that all of the servers in a domain use to communicate with each other. This service is labeled with `weblogic.domainUID` and `weblogic.domainName`. * A `NodePort` type service is created for the Administration Server pod. This service provides HTTP access to the Administration Server to clients that are outside the Kubernetes cluster. This service is intended to be used to access the WebLogic Server Administration Console only. This service is labeled with `weblogic.domainUID` and `weblogic.domainName`. * If requested when configuring the domain, a second `NodePort` type service is created for the Administration Server pod. This second service is used to expose a WebLogic channel for the T3 protocol. This service provides T3 access to the Administration Server to clients that are outside the Kubernetes cluster. This service is intended to be used for WLST connections to the Administration Server. This service is labeled with `weblogic.domainUID` and `weblogic.domainName`. -* A pod is created for each WebLogic Managed Server. These pods are labeled with `weblogic.domainUID`, `weblogic.serverName` and `weblogic.domainName`. One container runs in each pod. WebLogic Node Manager and Managed Server processes are run inside each of these containers. The Node Manager process is used as an internal implementation detail for the liveness probe. It is not intended to be used for other purposes, and it may be removed in some future release. +* A pod is created for each WebLogic Managed Server. These pods are labeled with `weblogic.domainUID`, `weblogic.serverName`, and `weblogic.domainName`. One container runs in each pod. WebLogic Node Manager and Managed Server processes are run inside each of these containers. The Node Manager process is used as an internal implementation detail for the liveness probe. It is not intended to be used for other purposes, and it may be removed in some future release. * A `NodePort` type service is created for each Managed Server pod that contains a Managed Server that is not part of a WebLogic cluster. These services provide HTTP access to the Managed Servers to clients that are outside the Kubernetes cluster. These services are intended to be used to access applications running on the Managed Servers. These services are labeled with `weblogic.domainUID` and `weblogic.domainName`. -* An Ingress is created for each WebLogic cluster. This Ingress provides load balanced HTTP access to all Managed Servers in that WebLogic cluster. The operator updates the Ingress every time a Managed Server in the WebLogic cluster becomes “ready” or ceases to be able to service requests, such that the Ingress always points to just those Managed Servers that are able to handle user requests. The Ingress is labeled with `weblogic.domainUID`, `weblogic.clusterName`, and `weblogic.domainName`. The Ingress is also annotated with a class which is used to match Ingresses to the correct instances of the load balancer. In the technology preview release, there is one instance of the load balancer running for each WebLogic cluster, and the load balancers are configured with the root URL path (“/”). More flexible load balancer configuration is planned for a future release. -* If the ELK integration was requested when configuring the operator, there will also be another pod that runs logstash in a container. This pod will publish the logs from all WebLogic Server instances in the domain into ElasticSearch. There is one logstash per domain, but only one ElasticSearch and one Kibana for the entire Kubernetes cluster. +* An Ingress is created for each WebLogic cluster. This Ingress provides load balanced HTTP access to all Managed Servers in that WebLogic cluster. The operator updates the Ingress every time a Managed Server in the WebLogic cluster becomes “ready” or ceases to be able to service requests, such that the Ingress always points to just those Managed Servers that are able to handle user requests. The Ingress is labeled with `weblogic.domainUID`, `weblogic.clusterName`, and `weblogic.domainName`. The Ingress is also annotated with a class which is used to match Ingresses to the correct instances of the load balancer. In this release, there is one instance of the load balancer running for each WebLogic cluster, and the load balancers are configured with the root URL path (“/”). More flexible load balancer configuration is planned for a future release. +* If the ELK integration was requested when configuring the operator, there will also be another pod that runs logstash in a container. This pod will publish the logs from all WebLogic Server instances in the domain into Elasticsearch. There is one logstash per domain, but only one Elasticsearch and one Kibana for the entire Kubernetes cluster. The diagram below shows the components inside the containers running WebLogic Server instances: @@ -49,26 +49,26 @@ All containers running WebLogic Server use the same Docker image, `store/oracle/ Within the container, the following aspects are configured by the operator: -* The `ENTRYPOINT` is configured to a script that starts up a Node Manager process, and then uses WLST to ask that Node Manager start the server. Node Manager is used to start servers so that the socket connection to the server will be available to obtain server status even when the server is unresponsive. This is used by the liveness probe. +* The `ENTRYPOINT` is configured to a script that starts up a Node Manager process, and then uses WLST to request that Node Manager start the server. Node Manager is used to start servers so that the socket connection to the server will be available to obtain server status even when the server is unresponsive. This is used by the liveness probe. * The liveness probe is configured to check that the server is alive by querying the Node Manager process. The liveness probe is configured to check liveness every 15 seconds, and to timeout after five seconds. If a pod fails the liveness probe, Kubernetes will restart that container, and possibly the pod. * The readiness probe is configured to use the WebLogic Server ReadyApp. The readiness probe is used to determine if the server is ready to accept user requests. The readiness is used to determine when a server should be included in a load balancer Ingress, when a restarted server is fully started in the case of a rolling restart, and for various other purposes. * A shutdown hook is configured that will execute a script that performs a graceful shutdown of the server. This ensures that servers have an opportunity to shut down cleanly before they are killed. ## Domain state stored outside Docker images -The operator expects (and requires) that all state be stored outside of the Docker images that are used to run the domain. This means either in a persistent file system, or in a database. The WebLogic configuration, i.e. the domain directory, the applications directory, file-based persistent stores, etc., are stored on a persistent volume. A database could also be used to host persistent stores. All of the containers that are participating in the WebLogic domain use the exact same image, and take on their personality; i.e., which server they execute, at startup time. Each container mounts the same shared storage and has access to the state information that it needs to fulfill its role in the domain. +The operator expects (and requires) that all state be stored outside of the Docker images that are used to run the domain. This means either in a persistent file system, or in a database. The WebLogic configuration, that is, the domain directory, the applications directory, file-based persistent stores, and such, are stored on a persistent volume. You could also use a database to host persistent stores. All of the containers that are participating in the WebLogic domain use the exact same image, and take on their personality; that is, which server they execute, at startup time. Each container mounts the same shared storage and has access to the state information that it needs to fulfill its role in the domain. -It is worth providing some background on why this approach was adopted, in addition to the fact that this separation is consistent with other existing operators (for other products) and the Kubernetes “cattle, not pets” philosophy when it comes to containers. +It is worth providing some background information on why this approach was adopted, in addition to the fact that this separation is consistent with other existing operators (for other products) and the Kubernetes “cattle, not pets” philosophy when it comes to containers. The external state approach allows the operator to treat the Docker images as essentially immutable, read-only, binary images. This means that the image needs to be pulled only once, and that many domains can share the same image. This helps to minimize the amount of bandwidth and storage needed for WebLogic Server Docker images. This approach also eliminates the need to manage any state created in a running container, because all of the state that needs to be preserved is written into either the persistent volume or a database back end. The containers and pods are completely throwaway and can be replaced with new containers and pods as necessary. This makes handling failures and rolling restarts much simpler because there is no need to preserve any state inside a running container. -When users wish to apply a binary patch to WebLogic Server, it is necessary only to create a single new, patched Docker image. Any domains that are running may be updated to this new patched image with a rolling restart if desired, because there is no state in the containers. +When users wish to apply a binary patch to WebLogic Server, it is necessary to create only a single new, patched Docker image. If desired, any domains that are running may be updated to this new patched image with a rolling restart, because there is no state in the containers. -It is envisaged that in some future release of the operator, it will be desirable to be able to “move” or “copy” domains in order to support scenarios like Kubernetes federation, high availability and disaster recovery. Separating the state from the running containers is seen as a way to greatly simplify this feature, and to minimize the amount of data that would need to be moved over the network, because the configuration is generally much smaller than the size of WebLogic Server Docker images. +It is envisaged that in some future release of the operator, it will be desirable to be able to “move” or “copy” domains in order to support scenarios like Kubernetes federation, high availability, and disaster recovery. Separating the state from the running containers is seen as a way to greatly simplify this feature, and to minimize the amount of data that would need to be moved over the network, because the configuration is generally much smaller than the size of WebLogic Server Docker images. The team developing the operator felt that these considerations provided adequate justification for adopting the external state approach. ## Network name predictability -The operator uses services to provide stable, well-known DNS names for each server. These names are known in advance of starting up a pod to run a server, and are used in the `ListenAddress` fields in the WebLogic Server configuration to ensure servers will always be able to find each other. This also eliminates the need for pod names or the actual WebLogic Server instance names to be the same as the DNS addresses. +The operator uses services to provide stable, well-known DNS names for each server. These names are known in advance of starting up a pod to run a server, and are used in the `ListenAddress` fields in the WebLogic Server configuration to ensure that servers will always be able to find each other. This also eliminates the need for pod names or the actual WebLogic Server instance names to be the same as the DNS addresses. From d75bf415d18a70efbf53873101b37ee81e10fc6c Mon Sep 17 00:00:00 2001 From: Ryan Eberhard Date: Thu, 26 Apr 2018 15:18:11 -0400 Subject: [PATCH 091/186] Simplify pre-integation test cleanup --- wercker.yml | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/wercker.yml b/wercker.yml index 7ce2e6451ac..035c6900bb8 100644 --- a/wercker.yml +++ b/wercker.yml @@ -178,20 +178,30 @@ integration-test: kubectl delete secret docker-store --ignore-not-found=true kubectl create secret docker-registry docker-store --docker-server=index.docker.io/v1/ --docker-username=$DOCKER_USERNAME --docker-password=$DOCKER_PASSWORD --docker-email=$DOCKER_EMAIL - kubectl delete ns test1 --ignore-not-found=true - kubectl create ns test1 + local nss = `kubectl get ns -o name` + if [ `kubectl get ns -o name | grep namespaces/test1 | wc -l` = 0 ]; then + kubectl create ns test1 + fi + if [ `kubectl get ns -o name | grep namespaces/test2 | wc -l` = 0 ]; then + kubectl create ns test2 + fi + if [ `kubectl get ns -o name | grep namespaces/weblogic-operator-1 | wc -l` = 0 ]; then + kubectl create ns weblogic-operator-1 + fi + if [ `kubectl get ns -o name | grep namespaces/weblogic-operator-2 | wc -l` = 0 ]; then + kubectl create ns weblogic-operator-2 + fi + + kubectl delete secret docker-store -n test1 --ignore-not-found=true kubectl create secret docker-registry docker-store -n test1 --docker-server=index.docker.io/v1/ --docker-username=$DOCKER_USERNAME --docker-password=$DOCKER_PASSWORD --docker-email=$DOCKER_EMAIL - kubectl delete ns test2 --ignore-not-found=true - kubectl create ns test2 + kubectl delete secret docker-store -n test2 --ignore-not-found=true kubectl create secret docker-registry docker-store -n test2 --docker-server=index.docker.io/v1/ --docker-username=$DOCKER_USERNAME --docker-password=$DOCKER_PASSWORD --docker-email=$DOCKER_EMAIL - kubectl delete ns weblogic-operator-1 --ignore-not-found=true - kubectl create ns weblogic-operator-1 + kubectl delete secret quay-io -n weblogic-operator-1 --ignore-not-found=true kubectl create secret docker-registry quay-io -n weblogic-operator-1 --docker-server=quay.io --docker-username=$QUAY_USERNAME --docker-password=$QUAY_PASSWORD --docker-email=$QUAY_EMAIL - kubectl delete ns weblogic-operator-2 --ignore-not-found=true - kubectl create ns weblogic-operator-2 + kubectl delete secret quay-io -n weblogic-operator-2 --ignore-not-found=true kubectl create secret docker-registry quay-io -n weblogic-operator-2 --docker-server=quay.io --docker-username=$QUAY_USERNAME --docker-password=$QUAY_PASSWORD --docker-email=$QUAY_EMAIL export IMAGE_NAME_OPERATOR="quay.io/markxnelson/weblogic-kubernetes-operator" From d4536b1753991fa09b70c6420bc518a6e636b180 Mon Sep 17 00:00:00 2001 From: Ryan Eberhard Date: Thu, 26 Apr 2018 15:25:06 -0400 Subject: [PATCH 092/186] Typo --- wercker.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/wercker.yml b/wercker.yml index 035c6900bb8..fe14cdb56c9 100644 --- a/wercker.yml +++ b/wercker.yml @@ -178,7 +178,6 @@ integration-test: kubectl delete secret docker-store --ignore-not-found=true kubectl create secret docker-registry docker-store --docker-server=index.docker.io/v1/ --docker-username=$DOCKER_USERNAME --docker-password=$DOCKER_PASSWORD --docker-email=$DOCKER_EMAIL - local nss = `kubectl get ns -o name` if [ `kubectl get ns -o name | grep namespaces/test1 | wc -l` = 0 ]; then kubectl create ns test1 fi From f64cedb996520297b6b1241f586d03006aaf8f92 Mon Sep 17 00:00:00 2001 From: Ryan Eberhard Date: Thu, 26 Apr 2018 15:47:00 -0400 Subject: [PATCH 093/186] Still trying to safely cleanup before test --- wercker.yml | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/wercker.yml b/wercker.yml index fe14cdb56c9..4f9c49baf1d 100644 --- a/wercker.yml +++ b/wercker.yml @@ -178,16 +178,32 @@ integration-test: kubectl delete secret docker-store --ignore-not-found=true kubectl create secret docker-registry docker-store --docker-server=index.docker.io/v1/ --docker-username=$DOCKER_USERNAME --docker-password=$DOCKER_PASSWORD --docker-email=$DOCKER_EMAIL - if [ `kubectl get ns -o name | grep namespaces/test1 | wc -l` = 0 ]; then + if [ `kubectl get ns -o name | grep namespaces/test1 | wc -l` = 1 ]; then + kubectl delete ns test1 --ignore-not-found=true + while [`kubectl get ns -o name | grep namespaces/test1 | wc -l` = 1 ]; do + sleep 5 + done kubectl create ns test1 fi - if [ `kubectl get ns -o name | grep namespaces/test2 | wc -l` = 0 ]; then + if [ `kubectl get ns -o name | grep namespaces/test2 | wc -l` = 1 ]; then + kubectl delete ns test2 --ignore-not-found=true + while [`kubectl get ns -o name | grep namespaces/test2 | wc -l` = 1 ]; do + sleep 5 + done kubectl create ns test2 fi - if [ `kubectl get ns -o name | grep namespaces/weblogic-operator-1 | wc -l` = 0 ]; then + if [ `kubectl get ns -o name | grep namespaces/weblogic-operator-1 | wc -l` = 1 ]; then + kubectl delete ns weblogic-operator-1 --ignore-not-found=true + while [`kubectl get ns -o name | grep namespaces/weblogic-operator-1 | wc -l` = 1 ]; do + sleep 5 + done kubectl create ns weblogic-operator-1 fi - if [ `kubectl get ns -o name | grep namespaces/weblogic-operator-2 | wc -l` = 0 ]; then + if [ `kubectl get ns -o name | grep namespaces/weblogic-operator-2 | wc -l` = 1 ]; then + kubectl delete ns weblogic-operator-2 --ignore-not-found=true + while [`kubectl get ns -o name | grep namespaces/weblogic-operator-2 | wc -l` = 1 ]; do + sleep 5 + done kubectl create ns weblogic-operator-2 fi From fee990f3bb1914582f546bd3dda4b48ad5968e34 Mon Sep 17 00:00:00 2001 From: Ryan Eberhard Date: Thu, 26 Apr 2018 15:56:54 -0400 Subject: [PATCH 094/186] Friends dont let friends use scripts --- wercker.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/wercker.yml b/wercker.yml index 4f9c49baf1d..b73ea772403 100644 --- a/wercker.yml +++ b/wercker.yml @@ -183,29 +183,29 @@ integration-test: while [`kubectl get ns -o name | grep namespaces/test1 | wc -l` = 1 ]; do sleep 5 done - kubectl create ns test1 fi + kubectl create ns test1 if [ `kubectl get ns -o name | grep namespaces/test2 | wc -l` = 1 ]; then kubectl delete ns test2 --ignore-not-found=true while [`kubectl get ns -o name | grep namespaces/test2 | wc -l` = 1 ]; do sleep 5 done - kubectl create ns test2 fi + kubectl create ns test2 if [ `kubectl get ns -o name | grep namespaces/weblogic-operator-1 | wc -l` = 1 ]; then kubectl delete ns weblogic-operator-1 --ignore-not-found=true while [`kubectl get ns -o name | grep namespaces/weblogic-operator-1 | wc -l` = 1 ]; do sleep 5 done - kubectl create ns weblogic-operator-1 fi + kubectl create ns weblogic-operator-1 if [ `kubectl get ns -o name | grep namespaces/weblogic-operator-2 | wc -l` = 1 ]; then kubectl delete ns weblogic-operator-2 --ignore-not-found=true while [`kubectl get ns -o name | grep namespaces/weblogic-operator-2 | wc -l` = 1 ]; do sleep 5 done - kubectl create ns weblogic-operator-2 fi + kubectl create ns weblogic-operator-2 kubectl delete secret docker-store -n test1 --ignore-not-found=true kubectl create secret docker-registry docker-store -n test1 --docker-server=index.docker.io/v1/ --docker-username=$DOCKER_USERNAME --docker-password=$DOCKER_PASSWORD --docker-email=$DOCKER_EMAIL From e1901a7cb4c589fe7c68d52f43c60084ce93f4b5 Mon Sep 17 00:00:00 2001 From: Rosemary Marano Date: Thu, 26 Apr 2018 16:09:51 -0400 Subject: [PATCH 095/186] first pass edit on creating domains doc --- site/creating-domain.md | 91 +++++++++++++++++++++-------------------- 1 file changed, 46 insertions(+), 45 deletions(-) diff --git a/site/creating-domain.md b/site/creating-domain.md index e6e48ad705f..9f29ee25703 100644 --- a/site/creating-domain.md +++ b/site/creating-domain.md @@ -8,13 +8,13 @@ Note that there is a short video demonstration of the domain creation process av When running a WebLogic domain in Kubernetes, there are some additional considerations that must be taken into account to ensure correct functioning: -* Multicast is not well supported in Kubernetes at the time of writing. Some networking providers have some support for multicast, but it is generally considered of “beta” quality. Oracle recommends configuring WebLogic clusters to use unicast. +* Multicast is not currently well supported in Kubernetes. Some networking providers have some support for multicast, but it is generally considered of “beta” quality. Oracle recommends configuring WebLogic clusters to use unicast. * The `ListenAddress` for all servers must be set to the correct DNS name; it should not be set to `0.0.0.0` or left blank. This is required for cluster discovery of servers to work correctly. * If there is a desire to expose a T3 channel outside of the Kubernetes cluster -- for example, to allow WLST or RMI connections from clients outside Kubernetes -- then the recommended approach is to define a dedicated channel (per server) and to expose that channel using a `NodePort` service. It is required that the channel’s internal and external ports be set to the same value as the chosen `NodePort`; for example, they could all be `32000`. If all three are not the same, WebLogic Server will reject T3 connection requests. ## Creating a domain namespace -Oracle recommends creating namespaces to host WebLogic domains. It is not required to maintain a one-to-one relationship between WebLogic domains and Kubernetes namespaces, but this may be done if desired. More than one WebLogic domain may be run in a single namespace if desired. +Oracle recommends creating namespaces to host WebLogic domains. It is not required to maintain a one-to-one relationship between WebLogic domains and Kubernetes namespaces, but this may be done if desired. More than one WebLogic domain may be run in a single namespace, if desired. Any namespaces that were listed in the `targetNamespaces` parameter in the operator parameters file when deploying the operator would have been created by the script. @@ -45,11 +45,11 @@ In this command, replace the uppercase items with the appropriate values. The `S ## Go to the Docker Store and accept the license agreement for the WebLogic Server image -If you have never used the WebLogic Server image before, you will need to go to the [Docker Store web interface](https://store.docker.com/images/oracle-weblogic-server-12c) and accept the license agreement before Docker Store will allow you to pull this image. This is a one-time requirement, you do not have to repeat it for each machine you want to use the image on. +If you have never used the WebLogic Server image before, you will need to go to the [Docker Store web interface](https://store.docker.com/images/oracle-weblogic-server-12c) and accept the license agreement before Docker Store will allow you to pull this image. This is a one-time requirement, you do not have to repeat it for each machine you want to use the image on. ## Pull the WebLogic Server image -You can let Kubernetes pull the Docker image for you the first time you try to create a pod that uses the image, but we have found that you can generally avoid various common issues like putting the secret in the wrong namespace or getting the credentials wrong by just manually pulling the image by running these commands *on the Kubernetes nodes*: +You can let Kubernetes pull the Docker image for you the first time you try to create a pod that uses the image, but we have found that you can generally avoid various common issues, like putting the secret in the wrong namespace or getting the credentials wrong, by just manually pulling the image by running these commands *on the Kubernetes nodes*: ``` docker login @@ -73,9 +73,9 @@ The `SECRET_NAME` value specified here must be in the format `DOMAINUID-weblogic ## Important considerations for persistent volumes -WebLogic domains that are managed by the operator are required to store their configuration and state on a persistent volume. This volume is mounted read/write on all containers that are running a server in the domain. There are many different persistent volume providers, but they do not all support multiple read/write mounts. It is required that the chosen provider supports multiple read/write mounts. Details of available providers are available at [https://kubernetes.io/docs/concepts/storage/persistent-volumes](https://kubernetes.io/docs/concepts/storage/persistent-volumes). Careful attention should be given to the table in the section titled “Access Modes”. +WebLogic domains that are managed by the operator are required to store their configuration and state on a persistent volume. This volume is mounted read/write on all containers that are running a server in the domain. There are many different persistent volume providers, but they do not all support multiple read/write mounts. It is required that the chosen provider supports multiple read/write mounts. Details of available providers are available at [https://kubernetes.io/docs/concepts/storage/persistent-volumes](https://kubernetes.io/docs/concepts/storage/persistent-volumes). Careful attention should be given to the information in the table in the section titled, “Access Modes”. -In a single-node Kubernetes cluster, such as may be used for testing or proof of concept activities, `HOST_PATH` provides the simplest configuration. In a multinode Kubernetes cluster a `HOST_PATH` that is located on shared storage mounted by all nodes in the Kubernetes cluster is the simplest configuration. If nodes do not have shared storage, then NFS is probably the most widely available option. There are other options listed in the referenced table. +In a single-node Kubernetes cluster, such as may be used for testing or proof of concept activities, `HOST_PATH` provides the simplest configuration. In a multinode Kubernetes cluster, a `HOST_PATH` that is located on shared storage mounted by all nodes in the Kubernetes cluster is the simplest configuration. If nodes do not have shared storage, then NFS is probably the most widely available option. There are other options listed in the referenced table. ## Creating a persistent volume for the domain @@ -85,50 +85,50 @@ The persistent volume for the domain must be created using the appropriate tools mkdir –m 777 –p /path/to/domain1PersistentVolume ``` -For other providers, consult the documentation for the provider for instructions on how to create a persistent volume. +For other providers, consult the provider documentation for instructions on how to create a persistent volume. ## Customizing the domain parameters file The domain is created with the provided installation script (`create-weblogic-domain.sh`). The input to this script is the file `create-weblogic-domain-inputs.yaml`, which must be copied and updated to reflect the target environment. -The following parameters must be provided in the input file: +The following parameters must be provided in the input file. -### CONFIGURATION PARAMETERS FOR THE CREATE DOMAIN JOB +### Configuration parameters for the create domain job | Parameter | Definition | Default | | --- | --- | --- | -| adminPort | Port number for the Administration Server inside the Kubernetes cluster. | 7001 | -| adminNodePort | Port number of the Administration Server outside the Kubernetes cluster. | 30701 | -| adminServerName | The name of the Administration Server. | admin-server | -| clusterName | The name of WebLogic Cluster instance to generate for the domain. | cluster-1 | -| configuredManagedServerCount | Number of Managed Server instances to generate for the domain. | 2 | -| domainName | Name of the WebLogic domain to create. | base_domain | -| domainUID | Unique ID that will be used to identify this particular domain. This ID must be unique across all domains in a Kubernetes cluster. | no default | -| exposeAdminNodePort | Boolean indicating if the the Administration Server is exposed outside of the Kubernetes cluster. | false | -| exposeAdminT3Channel | Boolean indicating if the T3 admin channel is exposed outside the Kubernetes cluster. | false | -| initialManagedServerReplicas | The number of managed servers to initially start for the domain | 2 | -| javaOptions | Java options for starting the Admin and Managed servers. | -Dweblogic.StdoutDebugEnabled=false | -| loadBalancer | The kind of load balancer to create. Legal values are 'NONE' and 'TRAEFIK'. | TRAEFIK | -| loadBalancerDashboardPort | The node port for the load balancer to accept dashboard traffic. | 30315 | -| loadBalancerWebPort | The node port for the load balancer to accept user traffic. | 30305 | -| managedServerNameBase | Base string used to generate Managed Server names. | managed-server | -| managedServerPort | Port number for each Managed Server. | 8001 | -| namespace | The Kubernetes namespace to create the domain in. | default | -| productionModeEnabled | Boolean indicating if production mode is enabled for the domain. | true | -| startupControl | Determines which WebLogic servers will be started up. Legal values are 'NONE', 'ALL', 'ADMIN', 'SPECIFIED', or 'AUTO' | AUTO | -| t3ChannelPort | Port for the T3Channel of the NetworkAccessPoint. | 30012 | -| t3PublicAddress | Public address for the t3 channel. | kubernetes | -| weblogicCredentialsSecretName | Name of the Kubernetes secret for the Administration Server's username and password. | domain1-weblogic-credentials | -| weblogicDomainStoragePath | Physical path of the storage for the domain. | no default | -| weblogicDomainStorageReclaimPolicy | Kubernetes persistent volume reclaim policy for the domain persistent storage | Retain | -| weblogicDomainStorageSize | Total storage allocated for the domain. | 10Gi | -| weblogicDomainStorageType | Type of storage for the domain. Legal values are 'NFS' and 'HOST_PATH". | HOST_PATH | -| weblogicDomainStorageNFSServer| The name of IP address of the NFS server for the domain's storage. | no default | -| weblogicImagePullSecretName | Name of the Kubernetes secret for the Docker Store, used to pull the WebLogic Server image. | docker-store-secret | +| `adminPort` | Port number for the Administration Server inside the Kubernetes cluster. | `7001` | +| `adminNodePort` | Port number of the Administration Server outside the Kubernetes cluster. | `30701` | +| `adminServerName` | Name of the Administration Server. | `admin-server` | +| `clusterName` | Name of the WebLogic cluster instance to generate for the domain. | `cluster-1` | +| `configuredManagedServerCount` | Number of Managed Server instances to generate for the domain. | `2` | +| `domainName` | Name of the WebLogic domain to create. | `base_domain` | +| `domainUID` | Unique ID that will be used to identify this particular domain. This ID must be unique across all domains in a Kubernetes cluster. | no default | +| `exposeAdminNodePort` | Boolean indicating if the Administration Server is exposed outside of the Kubernetes cluster. | `false` | +| `exposeAdminT3Channel` | Boolean indicating if the T3 administrative channel is exposed outside the Kubernetes cluster. | `false` | +| `initialManagedServerReplicas` | Number of Managed Servers to initially start for the domain. | `2` | +| `javaOptions` | Java options for starting the Administration and Managed Servers. | `-Dweblogic.StdoutDebugEnabled=false` | +| `loadBalancer` | Type of load balancer to create. Legal values are `NONE` and `TRAEFIK`. | `TRAEFIK` | +| `loadBalancerDashboardPort` | Node port for the load balancer to accept dashboard traffic. | `30315` | +| `loadBalancerWebPort` | Node port for the load balancer to accept user traffic. | `30305` | +| `managedServerNameBase` | Base string used to generate Managed Server names. | `managed-server` | +| `managedServerPort` | Port number for each Managed Server. | `8001` | +| `namespace` | Kubernetes namespace to create the domain in. | `default` | +| `productionModeEnabled` | Boolean indicating if production mode is enabled for the domain. | `true` | +| `startupControl` | Determines which WebLogic servers will be started up. Legal values are `NONE`, `ALL`, `ADMIN`, `SPECIFIED`, or `AUTO` | `AUTO` | +| `t3ChannelPort` | Port for the T3 channel of the NetworkAccessPoint. | `30012` | +| `t3PublicAddress` | Public address for the T3 channel. | `kubernetes` | +| `weblogicCredentialsSecretName` | Name of the Kubernetes secret for the Administration Server's username and password. | `domain1-weblogic-credentials` | +| `weblogicDomainStoragePath` | Physical path of the storage for the domain. | no default | +| `weblogicDomainStorageReclaimPolicy` | Kubernetes persistent volume reclaim policy for the domain persistent storage | `Retain` | +| `weblogicDomainStorageSize` | Total storage allocated for the domain. | `10Gi` | +| `weblogicDomainStorageType` | Type of storage for the domain. Legal values are `NFS` and `HOST_PATH`. | `HOST_PATH` | +| `weblogicDomainStorageNFSServer`| Name of the IP address of the NFS server for the domain's storage. | no default | +| `weblogicImagePullSecretName` | Name of the Kubernetes secret for the Docker Store, used to pull the WebLogic Server image. | `docker-store-secret` | ## Limitations of the create domain script -This technology preview release has some limitations in the create domain script that users should be aware of. +This release has some limitations in the create domain script that users should be aware of. * The script creates the specified number of Managed Servers and places them all in one cluster. * The script always creates one cluster. @@ -139,11 +139,12 @@ Oracle intends to remove these limitations in a future release. At this point, you've created a customized copy of the inputs file. -Next, choose and create a directory that generated operator related files will be stored in, e.g. /path/to/weblogic-operator-output-directory. +Next, choose and create a directory in which generated operator related files will be stored, for example, `/path/to/weblogic-operator-output-directory`. Finally, run the create script, pointing it at your inputs file and output directory: -./create-weblogic-domain.sh \ +``` + ./create-weblogic-domain.sh \ –i create-domain-job-inputs.yaml \ -o /path/to/weblogic-operator-output-directory ``` @@ -152,19 +153,19 @@ Finally, run the create script, pointing it at your inputs file and output direc The script will perform the following steps: -* Create a directory for the generated Kubernetes YAML files for this domain. The pathname is /path/to/weblogic-operator-output-directory/weblogic-domains/. +* Create a directory for the generated Kubernetes YAML files for this domain. The pathname is `/path/to/weblogic-operator-output-directory/weblogic-domains/`. * Create Kubernetes YAML files based on the provided inputs. * Create a persistent volume for the shared state. * Create a persistent volume claim for that volume. * Create a Kubernetes job that will start up a utility WebLogic Server container and run WLST to create the domain on the shared storage. * Wait for the job to finish and then create a domain custom resource for the new domain. -The default domain created the script has the following characteristics: +The default domain created by the script has the following characteristics: * An Administration Server named `admin-server` listening on port `7001`. * A single cluster named `cluster-1` containing the specified number of Managed Servers. * A Managed Server named `managed-server1` listening on port `8001` (and so on up to the requested number of Managed Servers). -* Log files are located in `/shared/logs`. +* Log files that are located in `/shared/logs`. * No applications deployed. * No data sources. * No JMS resources. @@ -313,7 +314,7 @@ Status: Events: ``` -In the "Status" section of the output, the available servers and clusters are listed. Note that if this command is issued very soon after the script finishes, there may be no servers available yet, or perhaps only the Administration Server but no Managed Servers. The operator will start up the Administration Server first and wait for it to become ready before starting Managed Servers. +In the `Status` section of the output, the available servers and clusters are listed. Note that if this command is issued very soon after the script finishes, there may be no servers available yet, or perhaps only the Administration Server but no Managed Servers. The operator will start up the Administration Server first and wait for it to become ready before starting Managed Servers. ### Verify pods @@ -381,6 +382,6 @@ Events: ## Configuring the domain readiness -Kubernetes has a concept of “readiness” that is used to determine when external requests should be routed to a particular pod. The domain creation job provided with the operator configures the readiness probe to use the WebLogic Server ReadyApp, which provides a mechanism to check when the server is actually ready to process work, as opposed to simply being in the RUNNING state. Often applications have some work to do after the server is RUNNING but before they are ready to process new requests. +Kubernetes has a concept of “readiness” that is used to determine when external requests should be routed to a particular pod. The domain creation job provided with the operator configures the readiness probe to use the WebLogic Server ReadyApp, which provides a mechanism to check when the server is actually ready to process work, as opposed to simply being in the `RUNNING` state. Often applications have some work to do after the server is `RUNNING` but before they are ready to process new requests. ReadyApp provides an API that allows an application to register itself, so that its state will be taken into consideration when determining if the server is “ready”, and an API that allows the application to inform ReadyApp when it considers itself to be ready. From a376a0227d59241da1fe7e16da693e20f4901aac Mon Sep 17 00:00:00 2001 From: Ryan Eberhard Date: Thu, 26 Apr 2018 16:17:12 -0400 Subject: [PATCH 096/186] Fingers crossed --- wercker.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/wercker.yml b/wercker.yml index b73ea772403..3a08b7f58ff 100644 --- a/wercker.yml +++ b/wercker.yml @@ -178,30 +178,30 @@ integration-test: kubectl delete secret docker-store --ignore-not-found=true kubectl create secret docker-registry docker-store --docker-server=index.docker.io/v1/ --docker-username=$DOCKER_USERNAME --docker-password=$DOCKER_PASSWORD --docker-email=$DOCKER_EMAIL - if [ `kubectl get ns -o name | grep namespaces/test1 | wc -l` = 1 ]; then + if [ `kubectl get ns test1 | grep Error | wc -l` = 0 ]; then kubectl delete ns test1 --ignore-not-found=true - while [`kubectl get ns -o name | grep namespaces/test1 | wc -l` = 1 ]; do + while [`kubectl get ns test1 -o jsonpath='{.status.phase}' | grep Terminating | wc -l` = 1 ]; do sleep 5 done fi kubectl create ns test1 - if [ `kubectl get ns -o name | grep namespaces/test2 | wc -l` = 1 ]; then + if [ `kubectl get ns test2 | grep Error | wc -l` = 0 ]; then kubectl delete ns test2 --ignore-not-found=true - while [`kubectl get ns -o name | grep namespaces/test2 | wc -l` = 1 ]; do + while [`kubectl get ns test2 -o jsonpath='{.status.phase}' | grep Terminating | wc -l` = 1 ]; do sleep 5 done fi kubectl create ns test2 - if [ `kubectl get ns -o name | grep namespaces/weblogic-operator-1 | wc -l` = 1 ]; then + if [ `kubectl get ns weblogic-operator-1 | grep Error | wc -l` = 0 ]; then kubectl delete ns weblogic-operator-1 --ignore-not-found=true - while [`kubectl get ns -o name | grep namespaces/weblogic-operator-1 | wc -l` = 1 ]; do + while [`kubectl get ns weblogic-operator-1 -o jsonpath='{.status.phase}' | grep Terminating | wc -l` = 1 ]; do sleep 5 done fi kubectl create ns weblogic-operator-1 - if [ `kubectl get ns -o name | grep namespaces/weblogic-operator-2 | wc -l` = 1 ]; then + if [ `kubectl get ns weblogic-operator-2 | grep Error | wc -l` = 0 ]; then kubectl delete ns weblogic-operator-2 --ignore-not-found=true - while [`kubectl get ns -o name | grep namespaces/weblogic-operator-2 | wc -l` = 1 ]; do + while [`kubectl get ns weblogic-operator-2 -o jsonpath='{.status.phase}' | grep Terminating | wc -l` = 1 ]; do sleep 5 done fi From 2fe4120b7084dd5757005b6e8b78a7a8584637a6 Mon Sep 17 00:00:00 2001 From: Rosemary Marano Date: Thu, 26 Apr 2018 16:23:59 -0400 Subject: [PATCH 097/186] incorporated Monica's review comments --- site/architecture.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/site/architecture.md b/site/architecture.md index b9ff2261eee..5d5f004a2bb 100644 --- a/site/architecture.md +++ b/site/architecture.md @@ -7,7 +7,7 @@ The operator consists of the following two main parts: The operator is packaged in a Docker image `container-registry.oracle.com/middleware/weblogic-operator:latest`. This image can be deployed to a Kubernetes cluster. It is recommended that the operator be deployed in its own namespace. Only one operator is permitted in a namespace; however, multiple operators may be deployed in a Kubernetes cluster provided they are each in their own namespace and the list of namespaces they manage do not overlap. -Scripts are provided to deploy the operator to a Kubernetes cluster. These scripts also provide options to install and configure a load balancer and ELK integration. +Scripts are provided to deploy the operator to a Kubernetes cluster. These scripts also provide options to install and configure a load balancer and Elastic Stack integration. The operator registers a Kubernetes custom resource definition called `domain.weblogic.oracle` (shortname `domain`, plural `domains`). @@ -17,9 +17,9 @@ The diagram below shows the general layout of high-level components, including o The Kubernetes cluster has several namespaces. Components may be deployed into namespaces as follows: -* The operator is deployed into its own namespace. If the ELK integration option is configured, then a logstash pod will also be deployed in the operator’s namespace. +* The operator is deployed into its own namespace. If the Elastic Stack integration option is configured, then a logstash pod will also be deployed in the operator’s namespace. * WebLogic domains will be deployed into various namespaces. There can be more than one domain in a namespace if desired. There is no limit on the number of domains or namespaces that an operator can manage. Note that there can be more than one operator in a Kubernetes cluster, but each operator is configured with a list of the specific namespaces that it is responsible for. The operator will not take any action on any domain that is not in one of the namespaces the operator is configured to manage. -* If the ELK integration option is configured, Elasticsearch and Kibana will be deployed in the `default` namespace. +* If the Elastic Stack integration option is configured, Elasticsearch and Kibana will be deployed in the `default` namespace. * If a load balancer is configured, it will be deployed in the `kube-system` namespace. ## Domain architecture @@ -39,7 +39,7 @@ This diagram shows the following details: * A pod is created for each WebLogic Managed Server. These pods are labeled with `weblogic.domainUID`, `weblogic.serverName`, and `weblogic.domainName`. One container runs in each pod. WebLogic Node Manager and Managed Server processes are run inside each of these containers. The Node Manager process is used as an internal implementation detail for the liveness probe. It is not intended to be used for other purposes, and it may be removed in some future release. * A `NodePort` type service is created for each Managed Server pod that contains a Managed Server that is not part of a WebLogic cluster. These services provide HTTP access to the Managed Servers to clients that are outside the Kubernetes cluster. These services are intended to be used to access applications running on the Managed Servers. These services are labeled with `weblogic.domainUID` and `weblogic.domainName`. * An Ingress is created for each WebLogic cluster. This Ingress provides load balanced HTTP access to all Managed Servers in that WebLogic cluster. The operator updates the Ingress every time a Managed Server in the WebLogic cluster becomes “ready” or ceases to be able to service requests, such that the Ingress always points to just those Managed Servers that are able to handle user requests. The Ingress is labeled with `weblogic.domainUID`, `weblogic.clusterName`, and `weblogic.domainName`. The Ingress is also annotated with a class which is used to match Ingresses to the correct instances of the load balancer. In this release, there is one instance of the load balancer running for each WebLogic cluster, and the load balancers are configured with the root URL path (“/”). More flexible load balancer configuration is planned for a future release. -* If the ELK integration was requested when configuring the operator, there will also be another pod that runs logstash in a container. This pod will publish the logs from all WebLogic Server instances in the domain into Elasticsearch. There is one logstash per domain, but only one Elasticsearch and one Kibana for the entire Kubernetes cluster. +* If the Elastic Stack integration was requested when configuring the operator, there will also be another pod that runs logstash in a container. This pod will publish the logs from all WebLogic Server instances in the domain into Elasticsearch. There is one logstash per domain, but only one Elasticsearch and one Kibana for the entire Kubernetes cluster. The diagram below shows the components inside the containers running WebLogic Server instances: From 912b3fd1d2c0b1e6eb0db3c195d9469e7c3c2ad6 Mon Sep 17 00:00:00 2001 From: Ryan Eberhard Date: Thu, 26 Apr 2018 16:34:10 -0400 Subject: [PATCH 098/186] Try bash --- wercker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/wercker.yml b/wercker.yml index 3a08b7f58ff..4d58eb82098 100644 --- a/wercker.yml +++ b/wercker.yml @@ -74,6 +74,7 @@ integration-test: - script: name: Run integration tests code: | + #!/bin/bash function cleanup_and_store { # release lease in case run.sh failed to release it # (the following command only releases the release after confirming this pipeline still owns it) From 78ea79e61c68c02e1f08cc86966c0712859cd62e Mon Sep 17 00:00:00 2001 From: Anthony Lai Date: Thu, 26 Apr 2018 13:34:38 -0700 Subject: [PATCH 099/186] add unit tests for apache load balance work --- .../CreateDomainGeneratedFilesBaseTest.java | 187 ++++++++++++++++-- ...ratedFilesOptionalFeaturesEnabledTest.java | 66 ++++++- .../CreateDomainInputsValidationTest.java | 19 ++ .../operator/create/DomainFiles.java | 8 + .../create/GeneratedDomainYamlFiles.java | 19 +- .../create/ParsedApacheSecurityYaml.java | 37 ++++ .../operator/create/ParsedApacheYaml.java | 45 +++++ 7 files changed, 358 insertions(+), 23 deletions(-) create mode 100644 operator/src/test/java/oracle/kubernetes/operator/create/ParsedApacheSecurityYaml.java create mode 100644 operator/src/test/java/oracle/kubernetes/operator/create/ParsedApacheYaml.java diff --git a/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainGeneratedFilesBaseTest.java b/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainGeneratedFilesBaseTest.java index 353beb104b0..a0a5ee10744 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainGeneratedFilesBaseTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainGeneratedFilesBaseTest.java @@ -6,15 +6,8 @@ import static java.util.Arrays.asList; import io.kubernetes.client.custom.Quantity; -import io.kubernetes.client.models.ExtensionsV1beta1Deployment; -import io.kubernetes.client.models.V1beta1ClusterRole; -import io.kubernetes.client.models.V1beta1ClusterRoleBinding; -import io.kubernetes.client.models.V1ConfigMap; -import io.kubernetes.client.models.V1Job; -import io.kubernetes.client.models.V1PersistentVolume; -import io.kubernetes.client.models.V1PersistentVolumeClaim; -import io.kubernetes.client.models.V1Service; -import io.kubernetes.client.models.V1ServiceAccount; +import io.kubernetes.client.models.*; + import static oracle.kubernetes.operator.LabelConstants.*; import static oracle.kubernetes.operator.create.CreateDomainInputs.readInputsYamlFile; import static oracle.kubernetes.operator.create.KubernetesArtifactUtils.*; @@ -66,6 +59,14 @@ protected ParsedTraefikYaml getTraefikYaml() { return getGeneratedFiles().getTraefikYaml(); } + protected ParsedApacheSecurityYaml getApacheSecurityYaml() { + return getGeneratedFiles().getApacheSecurityYaml(); + } + + protected ParsedApacheYaml getApacheYaml() { + return getGeneratedFiles().getApacheYaml(); + } + protected ParsedWeblogicDomainPersistentVolumeClaimYaml getWeblogicDomainPersistentVolumeClaimYaml() { return getGeneratedFiles().getWeblogicDomainPersistentVolumeClaimYaml(); } @@ -108,14 +109,14 @@ public void domainCustomResourceYaml_hasCorrectNumberOfObjects() throws Exceptio } @Test - public void traefikSecurityYaml_hasCorrectNumberOfObjects() throws Exception { + public void loadBalancerSecurityYaml_hasCorrectNumberOfObjects() throws Exception { assertThat( getTraefikSecurityYaml().getObjectCount(), is(getTraefikSecurityYaml().getExpectedObjectCount())); } @Test - public void traefikYaml_hasCorrectNumberOfObjects() throws Exception { + public void loadBalancerYaml_hasCorrectNumberOfObjects() throws Exception { assertThat( getTraefikYaml().getObjectCount(), is(getTraefikYaml().getExpectedObjectCount())); @@ -392,12 +393,28 @@ protected Domain getExpectedDomain() { } @Test - public void generatesCorrect_traefikServiceAccount() throws Exception { + public void generatesCorrect_loadBalancerServiceAccount() throws Exception { assertThat( getActualTraefikServiceAccount(), yamlEqualTo(getExpectedTraefikServiceAccount())); } + protected V1ServiceAccount getActualApacheServiceAccount() { + return getApacheYaml().getApacheServiceAccount(); + } + + protected V1ServiceAccount getExpectedApacheServiceAccount() { + return + newServiceAccount() + .metadata(newObjectMeta() + .name(getApacheName()) + .namespace(getInputs().getNamespace()) + .putLabelsItem(DOMAINUID_LABEL, getInputs().getDomainUID()) + .putLabelsItem(DOMAINNAME_LABEL, getInputs().getDomainName()) + .putLabelsItem(APP_LABEL, getApacheAppName())); + } + + protected V1ServiceAccount getActualTraefikServiceAccount() { return getTraefikYaml().getTraefikServiceAccount(); } @@ -414,12 +431,78 @@ protected V1ServiceAccount getExpectedTraefikServiceAccount() { } @Test - public void generatesCorrect_traefikDeployment() throws Exception { + public void generatesCorrect_loadBalancerDeployment() throws Exception { assertThat( getActualTraefikDeployment(), yamlEqualTo(getExpectedTraefikDeployment())); } + protected ExtensionsV1beta1Deployment getActualApacheDeployment() { + return getApacheYaml().getApacheDeployment(); + } + + protected ExtensionsV1beta1Deployment getExpectedApacheDeployment() { + return + newDeployment() + .apiVersion(API_VERSION_EXTENSIONS_V1BETA1) // TBD - why is apache using this older version? + .metadata(newObjectMeta() + .name(getApacheName()) + .namespace(getInputs().getNamespace()) + .putLabelsItem(DOMAINUID_LABEL, getInputs().getDomainUID()) + .putLabelsItem(DOMAINNAME_LABEL, getInputs().getDomainName()) + .putLabelsItem(APP_LABEL, getApacheAppName())) + .spec(newDeploymentSpec() + .replicas(1) + .selector(newLabelSelector() + .putMatchLabelsItem(DOMAINUID_LABEL, getInputs().getDomainUID()) + .putMatchLabelsItem(DOMAINNAME_LABEL, getInputs().getDomainName()) + .putMatchLabelsItem(APP_LABEL, getApacheAppName())) + .template(newPodTemplateSpec() + .metadata(newObjectMeta() + .putLabelsItem(DOMAINUID_LABEL, getInputs().getDomainUID()) + .putLabelsItem(DOMAINNAME_LABEL, getInputs().getDomainName()) + .putLabelsItem(APP_LABEL, getApacheAppName())) + .spec(newPodSpec() + .serviceAccountName(getApacheName()) + .terminationGracePeriodSeconds(60L) + .addContainersItem(newContainer() + .name(getApacheName()) + .image("12213-apache:latest") + .imagePullPolicy("Never") + .addEnvItem(newEnvVar() + .name("WEBLOGIC_CLUSTER") + .value(getInputs().getDomainUID() + "-cluster-" + getClusterNameLC() + ":" + getInputs().getManagedServerPort())) + .addEnvItem(newEnvVar() + .name("LOCATION") + .value(getInputs().getLoadBalancerAppPrepath())) + .addEnvItem(newEnvVar() + .name("WEBLOGIC_HOST") + .value(getInputs().getDomainUID() + "-" + getInputs().getAdminServerName())) + .addEnvItem(newEnvVar() + .name("WEBLOGIC_PORT") + .value(getInputs().getAdminPort())) + .readinessProbe(newProbe() + .tcpSocket(newTCPSocketAction() + .port(newIntOrString(80))) + .failureThreshold(1) + .initialDelaySeconds(10) + .periodSeconds(10) + .successThreshold(1) + .timeoutSeconds(2)) + .livenessProbe(newProbe() + .tcpSocket(newTCPSocketAction() + .port(newIntOrString(80))) + .failureThreshold(3) + .initialDelaySeconds(10) + .periodSeconds(10) + .successThreshold(1) + .timeoutSeconds(2)) + ) + ) + ) + ); + } + protected ExtensionsV1beta1Deployment getActualTraefikDeployment() { return getTraefikYaml().getTraefikDeployment(); } @@ -490,7 +573,7 @@ protected ExtensionsV1beta1Deployment getExpectedTraefikDeployment() { } @Test - public void generatesCorrect_traefikConfigMap() throws Exception { + public void generatesCorrect_loadBalancerConfigMap() throws Exception { // The config map contains a 'traefik.toml' property that has a lot of text // that we don't want to duplicate in the test. However, part of the text // is computed from the inputs, so we want to validate that part of the info. @@ -529,12 +612,35 @@ protected void assertThatActualTraefikTomlIsCorrect(String actualTraefikToml) { } @Test - public void generatesCorrect_traefikService() throws Exception { + public void generatesCorrect_loadBalancerService() throws Exception { assertThat( getActualTraefikService(), yamlEqualTo(getExpectedTraefikService())); } + protected V1Service getActualApacheService() { + return getApacheYaml().getApacheService(); + } + + protected V1Service getExpectedApacheService() { + return + newService() + .metadata(newObjectMeta() + .name(getApacheName()) + .namespace(getInputs().getNamespace()) + .putLabelsItem(DOMAINUID_LABEL, getInputs().getDomainUID()) + .putLabelsItem(DOMAINNAME_LABEL, getInputs().getDomainName())) + .spec(newServiceSpec() + .type("NodePort") + .putSelectorItem(DOMAINUID_LABEL, getInputs().getDomainUID()) + .putSelectorItem(DOMAINNAME_LABEL, getInputs().getDomainName()) + .putSelectorItem(APP_LABEL, getApacheAppName()) + .addPortsItem(newServicePort() + .name("rest-https") + .port(80) + .nodePort(Integer.parseInt(getInputs().getLoadBalancerWebPort())))); + } + protected V1Service getActualTraefikService() { return getTraefikYaml().getTraefikService(); } @@ -560,7 +666,7 @@ protected V1Service getExpectedTraefikService() { } @Test - public void generatesCorrect_traefikDashboardService() throws Exception { + public void generatesCorrect_loadBalancerDashboardService() throws Exception { assertThat( getActualTraefikDashboardService(), yamlEqualTo(getExpectedTraefikDashboardService())); @@ -591,12 +697,33 @@ protected V1Service getExpectedTraefikDashboardService() { } @Test - public void generatesCorrect_traefikClusterRole() throws Exception { + public void generatesCorrect_loadBalancerClusterRole() throws Exception { assertThat( getActualTraefikClusterRole(), yamlEqualTo(getExpectedTraefikClusterRole())); } + protected V1beta1ClusterRole getActualApacheClusterRole() { + return getApacheSecurityYaml().getApacheClusterRole(); + } + + protected V1beta1ClusterRole getExpectedApacheClusterRole() { + return + newClusterRole() + .metadata(newObjectMeta() + .name(getApacheName()) + .putLabelsItem(DOMAINUID_LABEL, getInputs().getDomainUID()) + .putLabelsItem(DOMAINNAME_LABEL, getInputs().getDomainName())) + .addRulesItem(newPolicyRule() + .addApiGroupsItem("") + .resources(asList("pods", "services", "endpoints", "secrets")) + .verbs(asList("get", "list", "watch"))) + .addRulesItem(newPolicyRule() + .addApiGroupsItem("extensions") + .addResourcesItem("ingresses") + .verbs(asList("get", "list", "watch"))); + } + protected V1beta1ClusterRole getActualTraefikClusterRole() { return getTraefikSecurityYaml().getTraefikClusterRole(); } @@ -620,12 +747,32 @@ protected V1beta1ClusterRole getExpectedTraefikClusterRole() { } @Test - public void generatesCorrect_traefikDashboardClusterRoleBinding() throws Exception { + public void generatesCorrect_loadBalancerClusterRoleBinding() throws Exception { assertThat( getActualTraefikDashboardClusterRoleBinding(), yamlEqualTo(getExpectedTraefikDashboardClusterRoleBinding())); } + protected V1beta1ClusterRoleBinding getActualApacheClusterRoleBinding() { + return getApacheSecurityYaml().getApacheClusterRoleBinding(); + } + + protected V1beta1ClusterRoleBinding getExpectedApacheClusterRoleBinding() { + return + newClusterRoleBinding() + .metadata(newObjectMeta() + .name(getApacheName()) + .putLabelsItem(DOMAINUID_LABEL, getInputs().getDomainUID()) + .putLabelsItem(DOMAINNAME_LABEL, getInputs().getDomainName())) + .addSubjectsItem(newSubject() + .kind("ServiceAccount") + .name(getApacheName()) + .namespace(getInputs().getNamespace())) + .roleRef(newRoleRef() + .name(getApacheName()) + .apiGroup("rbac.authorization.k8s.io")); + } + protected V1beta1ClusterRoleBinding getActualTraefikDashboardClusterRoleBinding() { return getTraefikSecurityYaml().getTraefikDashboardClusterRoleBinding(); } @@ -709,4 +856,8 @@ protected String getClusterLCScope() { protected String getClusterNameLC() { return getInputs().getClusterName().toLowerCase(); } + + protected String getApacheName() { return getInputs().getDomainUID() + "-" + getApacheAppName(); } + + protected String getApacheAppName() { return "apache-webtier";} } diff --git a/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainGeneratedFilesOptionalFeaturesEnabledTest.java b/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainGeneratedFilesOptionalFeaturesEnabledTest.java index dd939a2b33e..43a91cd1850 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainGeneratedFilesOptionalFeaturesEnabledTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainGeneratedFilesOptionalFeaturesEnabledTest.java @@ -7,8 +7,14 @@ import io.kubernetes.client.models.V1PersistentVolume; import static oracle.kubernetes.operator.create.CreateDomainInputs.*; import static oracle.kubernetes.operator.create.KubernetesArtifactUtils.*; +import static oracle.kubernetes.operator.create.YamlUtils.yamlEqualTo; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertNull; + import oracle.kubernetes.weblogic.domain.v1.Domain; import org.junit.BeforeClass; +import org.junit.Test; /** * Tests that the all artifacts in the yaml files that create-weblogic-domain.sh @@ -26,7 +32,8 @@ public static void setup() throws Exception { .exposeAdminT3Channel("true") .clusterType("CONFIGURED") .weblogicImagePullSecretName("test-weblogic-image-pull-secret-name") - .loadBalancer(LOAD_BALANCER_TRAEFIK) + .loadBalancer(LOAD_BALANCER_APACHE) + .loadBalancerAppPrepath("/loadBalancerAppPrePath") .weblogicDomainStorageType(STORAGE_TYPE_NFS) .productionModeEnabled("true") ); @@ -58,4 +65,61 @@ protected V1PersistentVolume getExpectedWeblogicDomainPersistentVolume() { .path(getInputs().getWeblogicDomainStoragePath())); return expected; } + + @Override + public void generatesCorrect_loadBalancerDeployment() throws Exception { + assertThat( + getActualApacheDeployment(), + yamlEqualTo(getExpectedApacheDeployment())); + } + + @Override + public void generatesCorrect_loadBalancerServiceAccount() throws Exception { + assertThat( + getActualApacheServiceAccount(), + yamlEqualTo(getExpectedApacheServiceAccount())); } + + @Override + public void generatesCorrect_loadBalancerConfigMap() throws Exception { + // No config map in generated yaml for LOAD_BALANCER_APACHE + } + + @Override + public void generatesCorrect_loadBalancerService() throws Exception { + assertThat( + getActualApacheService(), + yamlEqualTo(getExpectedApacheService())); + } + + @Override + public void generatesCorrect_loadBalancerDashboardService() throws Exception { + // No dashboard service in generated yaml for LOAD_BALANCER_APACHE + } + + @Override + public void generatesCorrect_loadBalancerClusterRole() throws Exception { + assertThat( + getActualApacheClusterRole(), + yamlEqualTo(getExpectedApacheClusterRole())); } + + @Override + public void generatesCorrect_loadBalancerClusterRoleBinding() throws Exception { + assertThat( + getActualApacheClusterRoleBinding(), + yamlEqualTo(getExpectedApacheClusterRoleBinding())); + } + + @Override + public void loadBalancerSecurityYaml_hasCorrectNumberOfObjects() throws Exception { + assertThat( + getApacheSecurityYaml().getObjectCount(), + is(getApacheSecurityYaml().getExpectedObjectCount())); + } + + @Override + public void loadBalancerYaml_hasCorrectNumberOfObjects() throws Exception { + assertThat( + getApacheYaml().getObjectCount(), + is(getApacheYaml().getExpectedObjectCount())); + } } diff --git a/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputsValidationTest.java b/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputsValidationTest.java index 34ec5beed3d..e7c203c98b8 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputsValidationTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputsValidationTest.java @@ -451,6 +451,21 @@ public void createDomain_with_upperCaseNamespace_failsAndReturnsError() throws E failsAndPrints(paramNotLowercaseError(PARAM_NAMESPACE, val))); } + @Test + public void createDomain_with_loadBalanceTypeTraefik_succeeds() throws Exception { + createDomain_with_validLoadBalancer_succeeds(LOAD_BALANCER_TRAEFIK); + } + + @Test + public void createDomain_with_loadBalanceTypeNone_succeeds() throws Exception { + createDomain_with_validLoadBalancer_succeeds(LOAD_BALANCER_NONE); + } + + @Test + public void createDomain_with_loadBalanceTypeApache_succeeds() throws Exception { + createDomain_with_validLoadBalancer_succeeds(LOAD_BALANCER_APACHE); + } + @Test public void createDomain_with_missingLoadBalancer_failsAndReturnsError() throws Exception { assertThat( @@ -516,6 +531,10 @@ private void createDomain_with_validClusterType_succeeds(String clusterType) thr createDomain_with_validInputs_succeeds(newInputs().clusterType(clusterType)); } + private void createDomain_with_validLoadBalancer_succeeds(String loadBalancerType) throws Exception { + createDomain_with_validInputs_succeeds(newInputs().loadBalancer(loadBalancerType)); + } + private void createDomain_with_validInputs_succeeds(CreateDomainInputs inputs) throws Exception { // throws an error if the inputs are not valid, succeeds otherwise: GeneratedDomainYamlFiles.generateDomainYamlFiles(inputs).remove(); diff --git a/operator/src/test/java/oracle/kubernetes/operator/create/DomainFiles.java b/operator/src/test/java/oracle/kubernetes/operator/create/DomainFiles.java index cc778e84603..30b52a51797 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/create/DomainFiles.java +++ b/operator/src/test/java/oracle/kubernetes/operator/create/DomainFiles.java @@ -41,6 +41,14 @@ public Path getDomainCustomResourceYamlPath() { return getWeblogicDomainPath().resolve(DOMAIN_CUSTOM_RESOURCE_YAML); } + public Path getApacheYamlPath() { + return getWeblogicDomainPath().resolve("weblogic-domain-apache.yaml"); + } + + public Path getApacheSecurityYamlPath() { + return getWeblogicDomainPath().resolve("weblogic-domain-apache-security.yaml"); + } + public Path getTraefikYamlPath() { return getWeblogicDomainPath().resolve("weblogic-domain-traefik-" + inputs.getClusterName().toLowerCase() + ".yaml"); } diff --git a/operator/src/test/java/oracle/kubernetes/operator/create/GeneratedDomainYamlFiles.java b/operator/src/test/java/oracle/kubernetes/operator/create/GeneratedDomainYamlFiles.java index 35a55aaa588..a27dc9bfce4 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/create/GeneratedDomainYamlFiles.java +++ b/operator/src/test/java/oracle/kubernetes/operator/create/GeneratedDomainYamlFiles.java @@ -23,6 +23,8 @@ public class GeneratedDomainYamlFiles { private ParsedDomainCustomResourceYaml domainCustomResourceYaml; private ParsedTraefikYaml traefikYaml; private ParsedTraefikSecurityYaml traefikSecurityYaml; + private ParsedApacheYaml apacheYaml; + private ParsedApacheSecurityYaml apacheSecurityYaml; private ParsedWeblogicDomainPersistentVolumeYaml weblogicDomainPersistentVolumeYaml; private ParsedWeblogicDomainPersistentVolumeClaimYaml weblogicDomainPersistentVolumeClaimYaml; @@ -40,10 +42,17 @@ private GeneratedDomainYamlFiles(CreateDomainInputs inputs) throws Exception { new ParsedCreateWeblogicDomainJobYaml(domainFiles.getCreateWeblogicDomainJobYamlPath(), inputs); domainCustomResourceYaml = new ParsedDomainCustomResourceYaml(domainFiles.getDomainCustomResourceYamlPath(), inputs); - traefikYaml = - new ParsedTraefikYaml(domainFiles.getTraefikYamlPath(), inputs); - traefikSecurityYaml = - new ParsedTraefikSecurityYaml(domainFiles.getTraefikSecurityYamlPath(), inputs); + if (CreateDomainInputs.LOAD_BALANCER_TRAEFIK.equals(inputs.getLoadBalancer())) { + traefikYaml = + new ParsedTraefikYaml(domainFiles.getTraefikYamlPath(), inputs); + traefikSecurityYaml = + new ParsedTraefikSecurityYaml(domainFiles.getTraefikSecurityYamlPath(), inputs); + } else if (CreateDomainInputs.LOAD_BALANCER_APACHE.equals(inputs.getLoadBalancer())) { + apacheYaml = + new ParsedApacheYaml(domainFiles.getApacheYamlPath(), inputs); + apacheSecurityYaml = + new ParsedApacheSecurityYaml(domainFiles.getApacheSecurityYamlPath(), inputs); + } weblogicDomainPersistentVolumeYaml = new ParsedWeblogicDomainPersistentVolumeYaml(domainFiles.getWeblogicDomainPersistentVolumeYamlPath(), inputs); weblogicDomainPersistentVolumeClaimYaml = @@ -62,6 +71,8 @@ private GeneratedDomainYamlFiles(CreateDomainInputs inputs) throws Exception { public ParsedDomainCustomResourceYaml getDomainCustomResourceYaml() { return domainCustomResourceYaml; } public ParsedTraefikYaml getTraefikYaml() { return traefikYaml; } public ParsedTraefikSecurityYaml getTraefikSecurityYaml() { return traefikSecurityYaml; } + public ParsedApacheYaml getApacheYaml() { return apacheYaml; } + public ParsedApacheSecurityYaml getApacheSecurityYaml() { return apacheSecurityYaml; } public ParsedWeblogicDomainPersistentVolumeYaml getWeblogicDomainPersistentVolumeYaml() { return weblogicDomainPersistentVolumeYaml; } public ParsedWeblogicDomainPersistentVolumeClaimYaml getWeblogicDomainPersistentVolumeClaimYaml() { return weblogicDomainPersistentVolumeClaimYaml; } diff --git a/operator/src/test/java/oracle/kubernetes/operator/create/ParsedApacheSecurityYaml.java b/operator/src/test/java/oracle/kubernetes/operator/create/ParsedApacheSecurityYaml.java new file mode 100644 index 00000000000..7045ebb6a8c --- /dev/null +++ b/operator/src/test/java/oracle/kubernetes/operator/create/ParsedApacheSecurityYaml.java @@ -0,0 +1,37 @@ +// Copyright 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.create; + +import java.nio.file.Path; + +import io.kubernetes.client.models.V1beta1ClusterRole; +import io.kubernetes.client.models.V1beta1ClusterRoleBinding; + +/** + * Parses a generated weblogic-domain-apache-security.yaml file into a set of typed k8s java objects + */ +public class ParsedApacheSecurityYaml extends ParsedKubernetesYaml { + + private CreateDomainInputs inputs; + + public ParsedApacheSecurityYaml(Path yamlPath, CreateDomainInputs inputs) throws Exception { + super(yamlPath); + this.inputs = inputs; + } + + public V1beta1ClusterRole getApacheClusterRole() { + return getClusterRoles().find(getApacheName()); + } + + public V1beta1ClusterRoleBinding getApacheClusterRoleBinding() { + return getClusterRoleBindings().find(getApacheName()); + } + + public int getExpectedObjectCount() { + return 2; + } + + private String getApacheName() { return inputs.getDomainUID() + "-apache-webtier"; } +} + diff --git a/operator/src/test/java/oracle/kubernetes/operator/create/ParsedApacheYaml.java b/operator/src/test/java/oracle/kubernetes/operator/create/ParsedApacheYaml.java new file mode 100644 index 00000000000..526f1112ce7 --- /dev/null +++ b/operator/src/test/java/oracle/kubernetes/operator/create/ParsedApacheYaml.java @@ -0,0 +1,45 @@ +// Copyright 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.create; + +import java.nio.file.Path; + +import io.kubernetes.client.models.ExtensionsV1beta1Deployment; +import io.kubernetes.client.models.V1ConfigMap; +import io.kubernetes.client.models.V1Service; +import io.kubernetes.client.models.V1ServiceAccount; +import io.kubernetes.client.models.V1beta2Deployment; + +/** + * Parses a generated weblogic-domain-apache.yaml file into a set of typed k8s java objects + */ +public class ParsedApacheYaml extends ParsedKubernetesYaml { + + private CreateDomainInputs inputs; + + public ParsedApacheYaml(Path yamlPath, CreateDomainInputs inputs) throws Exception { + super(yamlPath); + this.inputs = inputs; + } + + public V1ServiceAccount getApacheServiceAccount() { + return getServiceAccounts().find(getApacheName()); + } + + public ExtensionsV1beta1Deployment getApacheDeployment() { + return getDeployments().find(getApacheName()); + + } + + public V1Service getApacheService() { + return getServices().find(getApacheName()); + } + + public int getExpectedObjectCount() { + return 3; + } + + private String getApacheName() { return inputs.getDomainUID() + "-apache-webtier"; } +} + From f9296f1815fb749d67e02751b18fc8d1a28f8898 Mon Sep 17 00:00:00 2001 From: Rosemary Marano Date: Thu, 26 Apr 2018 16:52:44 -0400 Subject: [PATCH 100/186] incorporated Monica's review comments --- README.md | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index e5dd209267f..68d4e196972 100644 --- a/README.md +++ b/README.md @@ -8,14 +8,17 @@ Many organizations are exploring, testing, or actively moving application worklo Oracle has been working with the WebLogic community to find ways to make it as easy as possible for organizations using WebLogic Server to run important workloads, to move those workloads into the cloud. One aspect of that effort is the creation of the Oracle WebLogic Server Kubernetes Operator. This release of the Operator provides a number of features to assist with the management of WebLogic domains in a Kubernetes environment, including: -* A mechanism to create a WebLogic domain on a Kubernetes persistent volume. +* A mechanism to create a WebLogic domain on a Kubernetes persistent volume.This persistent volume can reside in NFS. * A mechanism to define a WebLogic domain as a Kubernetes resource (using a Kubernetes custom resource definition). * The ability to automatically start servers based on declarative startup parameters and desired states. +* The ability to manage a WebLogic configured or dynamic cluster. * The ability to automatically expose the WebLogic Server Administration Console outside the Kubernetes cluster (if desired). * The ability to automatically expose T3 channels outside the Kubernetes domain (if desired). * The ability to automatically expose HTTP paths on a WebLogic domain outside the Kubernetes domain with load balancing, and to update the load balancer when Managed Servers in the WebLogic domain are started or stopped. * The ability to scale a WebLogic domain by starting and stopping Managed Servers on demand, or by integrating with a REST API to initiate scaling based on WLDF, Prometheus/Grafana, or other rules. -* The ability to publish Operator and WebLogic Server logs into ElasticSearch and interact with them in Kibana. +* The ability to publish Operator and WebLogic Server logs into Elasticsearch and interact with them in Kibana. + + As part of Oracle’s ongoing commitment to open source in general, and to Kubernetes and the Cloud Native Computing Foundation specifically, Oracle has open sourced the Operator and is committed to enhancing it with additional features. Oracle welcomes feedback, issues, pull requests, and feature requests from the WebLogic community. @@ -155,11 +158,18 @@ Please refer to [Scaling a WebLogic cluster](site/scaling.md) for more informati Please refer to [Shutting down a domain](site/shutdown-domain.md) for information about how to shut down a domain running in Kubernetes. -## Load balancing with the Traefik Ingress controller +## Load balancing + +### Traefik -This release of the operator supports only the Traefik load balancer/Ingress controller. Support for other load balancers is planned in the future. Please refer to [Load balancing with Traefik](site/traefik.md) for information about the current capabilities. +### Voyager + + +### Apache + + [comment]: # (Exporting operator logs to ELK. The operator provides an option to export its log files to the ELK stack. Please refer to [ELK integration]site/elk.md for information about this capability.) ## Removing a domain From 4f46f0725dc442a21682f36d7f34a197191c59d0 Mon Sep 17 00:00:00 2001 From: Ryan Eberhard Date: Thu, 26 Apr 2018 17:46:17 -0400 Subject: [PATCH 101/186] Adjust timeout --- wercker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/wercker.yml b/wercker.yml index 4d58eb82098..1a07d22ca15 100644 --- a/wercker.yml +++ b/wercker.yml @@ -69,6 +69,7 @@ build: env: "PATH=$PATH:/operator" # This pipeline runs the integration tests against a k8s cluster on OCI. +command-timeout: 60 integration-test: steps: - script: From 940b5b4ace2735cbbf107afcb17af631c7f2a190 Mon Sep 17 00:00:00 2001 From: Dongbo Xiao Date: Thu, 26 Apr 2018 17:27:17 -0700 Subject: [PATCH 102/186] Update apache docker image Signed-off-by: Dongbo Xiao --- kubernetes/internal/weblogic-domain-apache-template.yaml | 2 +- src/integration-tests/bash/run.sh | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/kubernetes/internal/weblogic-domain-apache-template.yaml b/kubernetes/internal/weblogic-domain-apache-template.yaml index 1d4b0689d59..dedfd79ea85 100755 --- a/kubernetes/internal/weblogic-domain-apache-template.yaml +++ b/kubernetes/internal/weblogic-domain-apache-template.yaml @@ -40,7 +40,7 @@ spec: # path: %LOAD_BALANCER_VOLUME_PATH% containers: - name: %DOMAIN_UID%-apache-webtier - image: 12213-apache:latest + image: store/oracle/apache:12.2.1.3 imagePullPolicy: Never # volumeMounts: # - name: "%DOMAIN_UID%-apache-webtier" diff --git a/src/integration-tests/bash/run.sh b/src/integration-tests/bash/run.sh index c6dd3fbcee9..3cb02172ed6 100755 --- a/src/integration-tests/bash/run.sh +++ b/src/integration-tests/bash/run.sh @@ -525,6 +525,9 @@ function setup_jenkins { docker pull wlsldi-v2.docker.oraclecorp.com/store-serverjre-8:latest docker tag wlsldi-v2.docker.oraclecorp.com/store-serverjre-8:latest store/oracle/serverjre:8 + docker pull wlsldi-v2.docker.oraclecorp.com/weblogic-webtier-apache-12.2.1.3.0:latest + docker tag wlsldi-v2.docker.oraclecorp.com/weblogic-webtier-apache-12.2.1.3.0:latest store/oracle/apache:12.2.1.3 + # create a docker image for the operator code being tested docker build -t "${IMAGE_NAME_OPERATOR}:${IMAGE_TAG_OPERATOR}" --no-cache=true . @@ -541,6 +544,9 @@ function setup_local { docker pull wlsldi-v2.docker.oraclecorp.com/store-serverjre-8:latest docker tag wlsldi-v2.docker.oraclecorp.com/store-serverjre-8:latest store/oracle/serverjre:8 + docker pull wlsldi-v2.docker.oraclecorp.com/weblogic-webtier-apache-12.2.1.3.0:latest + docker tag wlsldi-v2.docker.oraclecorp.com/weblogic-webtier-apache-12.2.1.3.0:latest store/oracle/apache:12.2.1.3 + } function create_image_pull_secret_jenkins { From 309b825129b0504835800e10781e17e60140f921 Mon Sep 17 00:00:00 2001 From: Dongbo Xiao Date: Thu, 26 Apr 2018 17:50:51 -0700 Subject: [PATCH 103/186] Fix unit test Signed-off-by: Dongbo Xiao --- .../operator/create/CreateDomainGeneratedFilesBaseTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainGeneratedFilesBaseTest.java b/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainGeneratedFilesBaseTest.java index a0a5ee10744..1cb3465e049 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainGeneratedFilesBaseTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainGeneratedFilesBaseTest.java @@ -467,7 +467,7 @@ protected ExtensionsV1beta1Deployment getExpectedApacheDeployment() { .terminationGracePeriodSeconds(60L) .addContainersItem(newContainer() .name(getApacheName()) - .image("12213-apache:latest") + .image("store/oracle/apache:12.2.1.3") .imagePullPolicy("Never") .addEnvItem(newEnvVar() .name("WEBLOGIC_CLUSTER") From df00dd9267e1364bb8611b55176af371f5e55fd7 Mon Sep 17 00:00:00 2001 From: Dongbo Xiao Date: Fri, 27 Apr 2018 06:43:20 -0700 Subject: [PATCH 104/186] Add documentation for Apache load balancer Signed-off-by: Dongbo Xiao --- README.md | 5 +- site/apache.md | 476 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 478 insertions(+), 3 deletions(-) create mode 100644 site/apache.md diff --git a/README.md b/README.md index 0edf61eb4dc..0b10f9ed42a 100644 --- a/README.md +++ b/README.md @@ -156,10 +156,9 @@ Please refer to [Scaling a WebLogic cluster](site/scaling.md) for more informati Please refer to [Shutting down a domain](site/shutdown-domain.md) for information about how to shut down a domain running in Kubernetes. -## Load balancing with the Traefik Ingress controller +##Load balancing with an Ingress Controller or a Web Server -The initial Technology Preview release of the operator supports only the Traefik load balancer/Ingress controller. Support for other load balancers is planned in the future. -Please refer to [Load balancing with Traefik](site/traefik.md) for information about current capabilities. +You can choose a load balancer provider for your WebLogic domains running in a Kubernetes cluster. Please refer to Load balancing with Voyager Ingress Controller (link TBD), [Load balancing with Traefik Ingress Controller](site/traefik.md), and [Load balancing with Apache Web Server](site/apache.md) for information about the current capabilities and setup instructions for each of the supported load balancers. [comment]: # (Exporting operator logs to ELK. The operator provides an option to export its log files to the ELK stack. Please refer to [ELK integration]site/elk.md for information about this capability.) diff --git a/site/apache.md b/site/apache.md new file mode 100644 index 00000000000..4cdcc705b8b --- /dev/null +++ b/site/apache.md @@ -0,0 +1,476 @@ + +# Load Balancing with Apache Web Server + +This page describes how to setup and start a Apache Web Server for load balancing inside a Kubernets cluster. The configuration and startup can either be automatic when you create a domain using the WebLogic Operator's `create-weblogic-domain.sh` script, or manually if you have an existing WebLogic domain configuration. + +## Build Docker Image for Apache Web Server + +You need to prepare the Docker image for Apache Web Server that enbeds Oracle WebLogic Server Proxy Plugin. + + 1. Download and build the Docker image for the Apache Web Server with 12.2.1.3.0 Oracle WebLogic Server Proxy Plugin. See the instructions in + +[https://github.com/oracle/docker-images/tree/master/OracleWebLogic/samples/12213-webtier-apache). + + 2. tag your Docker image to `store/oracle/apache:12.2.1.3` using `docker tag` command. + +``` + +docker tag 12213-apache:latest store/oracle/apache:12.2.1. + +``` + +More information about the Apache plugin can be found at: [https://docs.oracle.com/middleware/1213/webtier/develop-plugin/apache.htm#PLGWL395) + +Once you have access to the Docker image of the Apache Web Server, you can go ahead follow the instructions below to setup and start Kubernetes artifacts for Apache Web Server. + + +## When use Apache load balancer with a WebLogic domain created with the WebLogic Operator + +Please refer to [Creating a domain using the WebLogic Operator](creating-domain.md) for how to create a domain with the WebLogic Operator. + +You need to configure Apache Web Server as your load balancer for a WebLogic domain by setting the `loadBalancer` option to `APACHE` in `create-weblogic-domain-inputs.yaml` (as shown below) when running the `create-weblogic-domain.sh` script to create a domain. + +``` + +# Load balancer to deploy. Supported values are: VOYAGER, TRAEFIK, APACHE, NONE + +loadBalancer: APACHE + +``` + +The `create-weblogic-domain.sh` script installs the Apache Web Server with Oracle WebLogic Server Proxy Plugin into the Kubernetes *cluster* in the same namespace of the *domain*. + +The Apache Web Server will expose a `NodePort` that allows access to the load balancer from the outside of the Kubernetes cluster. The port is controlled by the following setting in `create-weblogic-domain-inputs.yaml` file: + +``` + +# Load balancer web port + +loadBalancerWebPort: 30305 + +``` + +The user can access an application from utsidie of the Kubernetes cluster via http://:30305/. + +### Use the default plugin WL module configuration + +By default, the Apache Docker image supports a simple WebLogic server proxy plugin configuration for a single WebLogic domain with an admin server and a cluster. The `create-weblogic-domain.sh` script automatically customizes the default behavior based on your domain configuration. The default setting only supports the type of load balancing that uses the root path ("/"). You can further customize the root path of the load balancer withloadBalancerAppPrepath property in the `create-weblogic-domain-inputs.yaml` file. + +``` + +# Load balancer app prepath + +loadBalancerAppPrepath: /weblogic + +``` + +The user can then access an application from utsidie of the Kubernetes cluster via `http://:30305/weblogic/,` and the admin can then access the admin console via `http://:30305/console`. + +The generated Kubernetes yaml files look like the following given the domainUID "domain1". + +`weblogic-domain-apache.yaml` for Apache web server deployment + +``` + +-- + +apiVersion: v1 + +kind: ServiceAccount + +metadata: + + name: domain1-apache-webtier + + namespace: default + + labels: + + weblogic.domainUID: domain1 + + weblogic.domainName: base_domain + + app: apache-webtier + +--- + +kind: Deployment + +apiVersion: extensions/v1beta1 + +metadata: + + name: domain1-apache-webtier + + namespace: default + + labels: + + weblogic.domainUID: domain1 + + weblogic.domainName: base_domain + + app: apache-webtier + +spec: + + replicas: 1 + + selector: + + matchLabels: + + weblogic.domainUID: domain1 + + weblogic.domainName: base_domain + + app: apache-webtier + + template: + + metadata: + + labels: + + weblogic.domainUID: domain1 + + weblogic.domainName: base_domain + + app: apache-webtier + + spec: + + serviceAccountName: domain1-apache-webtier + + terminationGracePeriodSeconds: 60 + + # volumes: + + # - name: "domain1-apache-webtier" + + # hostPath: + + # path: %LOAD_BALANCER_VOLUME_PATH% + + containers: + + - name: domain1-apache-webtier + + image: store/oracle/apache:12.2.1.3 + + imagePullPolicy: Never + + # volumeMounts: + + # - name: "domain1-apache-webtier" + + # mountPath: "/config" + + env: + + - name: WEBLOGIC_CLUSTER + + value: 'domain1-cluster-cluster-1:8001' + + - name: LOCATION + + value: '/weblogic' + + - name: WEBLOGIC_HOST + + value: 'domain1-admin-server' + + - name: WEBLOGIC_PORT + + value: '7001' + + readinessProbe: + + tcpSocket: + + port: 80 + + failureThreshold: 1 + + initialDelaySeconds: 10 + + periodSeconds: 10 + + successThreshold: 1 + + timeoutSeconds: 2 + + livenessProbe: + + tcpSocket: + + port: 80 + + failureThreshold: 3 + + initialDelaySeconds: 10 + + periodSeconds: 10 + + successThreshold: 1 + + timeoutSeconds: 2 + +--- + +apiVersion: v1 + +kind: Service + +metadata: + + name: domain1-apache-webtier + + namespace: default + + labels: + + weblogic.domainUID: domain1 + + weblogic.domainName: base_domain + +spec: + + type: NodePort + + selector: + + weblogic.domainUID: domain1 + + weblogic.domainName: base_domain + + app: apache-webtier + + ports: + + - port: 80 + + nodePort: 30305 + + name: rest-https + +``` + +`weblogic-domain-apache-security.yaml` for associated RBAC roles and role bindings + +``` + +--- + +kind: ClusterRole + +apiVersion: rbac.authorization.k8s.io/v1beta1 + +metadata: + + name: domain1-apache-webtier + + labels: + + weblogic.domainUID: domain1 + + weblogic.domainName: base_domain + +rules: + + - apiGroups: + + - "" + + resources: + + - pods + + - services + + - endpoints + + - secrets + + verbs: + + - get + + - list + + - watch + + - apiGroups: + + - extensions + + resources: + + - ingresses + + verbs: + + - get + + - list + + - watch + +--- + +kind: ClusterRoleBinding + +apiVersion: rbac.authorization.k8s.io/v1beta1 + +metadata: + + name: domain1-apache-webtier + + labels: + + weblogic.domainUID: domain1 + + weblogic.domainName: base_domain + +roleRef: + + apiGroup: rbac.authorization.k8s.io + + kind: ClusterRole + + name: domain1-apache-webtier + +subjects: + +- kind: ServiceAccount + + name: domain1-apache-webtier + + namespace: default + +``` + + +Here are examples of the Kubernetes artifacts created by the WebLogic Operator: + + +``` + + +AMESPACE NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE + +default deploy/domain1-apache-webtier 1 1 1 1 4h + + + +default po/domain1-apache-webtier-7f7cf44dcf-pqzn7 1/1 Running 1 4h + +PORT(S) AGE + +default svc/domain1-apache-webtier NodePort 10.96.36.137 80:30305/TCP 4h + + +NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE + +bash-4.2$ kubectl get all |grep apache + +deploy/domain1-apache-webtier 1 1 1 1 2h + + +rs/domain1-apache-webtier-7b8b789797 1 1 1 2h + + +deploy/domain1-apache-webtier 1 1 1 1 2h + + +rs/domain1-apache-webtier-7b8b789797 1 1 1 2h + + + +po/domain1-apache-webtier-7b8b789797-lm45z 1/1 Running 0 2h + + +svc/domain1-apache-webtier NodePort 10.111.114.67 80:30305/TCP 2h + + + +bash-4.2$ kubectl get clusterroles |grep apache + +domain1-apache-webtier 2h + + + +bash-4.2$ kubectl get clusterrolebindings |grep apache + +domain1-apache-webtier 2h + +``` + + +### Use your own plugin WL module configuration + + +Optionally you can fine turn the behavior of the Apache plugin by providing your own Apache plugin configuration for `create-weblogic-domain.sh` script to use. You put your custom_mod_wl_apache.conf file in a local directory, say `/scratch/apache/config` , and specify this location in the `create-weblogic-domain-inputs.yaml` file as follows. + +``` + +# Docker volume path for APACHE + +# By default, the VolumePath is empty, which will cause the volume mount be disabled + +loadBalancerVolumePath: /scratch/apache/config + +``` + +Once the loadBalancerVolumePath is specified, the `create-weblogic-domain.sh` script will use the custom_mod_wl_apache.config file in `/scratch/apache/config` directory to replace what is in the Docker image. + +The generated yaml files will look similar except that the lines that start with "#" will be un-commented like the following. + +``` + + volumes: + + - name: "domain1-apache-webtier" + + hostPath: + + path: /scratch/apache/config + + containers: + + - name: domain1-apache-webtier + + image: store/oracle/apache:12.2.1.3 + + imagePullPolicy: Never + + volumeMounts: + + - name: "domain1-apache-webtier" + + mountPath: "/config" + +``` + + + +## When use Apache load balancer with a manually created WebLogic Domain + +If your WebLogic domain is not created by the WebLogic Operator, you need to manually create and start all Kubernetes' artifacts for Apache Web Server. + + + 1. Create your own custom_mod_wl_apache.conf file, and put it in a local dir, say ``. See the instructions in [https://docs.oracle.com/middleware/1213/webtier/develop-plugin/apache.htm#PLGWL395). + + 2. Create the Apache deployment yaml file. See the example above. Note that you need to use the **volumes** and **volumeMounts** to mount `` in to `/config` directory inside the pod that runs Apache web tier. Note that the Apache Web Server needs to be in the same Kubernetes namespace as the WebLogic domains that it needs to access. + + 3. Create a RBAC yaml file. See the example above + +Note that you can choose to run one Apache Web Server to load balance to multiple domains/clusters inside the same Kubernetes cluster as long as the Apache Web Server and the domains are all in the same namespace. + + + + + + + + From e7a0e391139057b55a2c261bbce76b1e71d31294 Mon Sep 17 00:00:00 2001 From: Dongbo Xiao Date: Fri, 27 Apr 2018 06:58:27 -0700 Subject: [PATCH 105/186] Fix references Signed-off-by: Dongbo Xiao --- README.md | 2 +- site/apache.md | 24 ++++-------------------- 2 files changed, 5 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 0b10f9ed42a..6e1ecee7320 100644 --- a/README.md +++ b/README.md @@ -156,7 +156,7 @@ Please refer to [Scaling a WebLogic cluster](site/scaling.md) for more informati Please refer to [Shutting down a domain](site/shutdown-domain.md) for information about how to shut down a domain running in Kubernetes. -##Load balancing with an Ingress Controller or a Web Server +## Load balancing with an Ingress Controller or a Web Server You can choose a load balancer provider for your WebLogic domains running in a Kubernetes cluster. Please refer to Load balancing with Voyager Ingress Controller (link TBD), [Load balancing with Traefik Ingress Controller](site/traefik.md), and [Load balancing with Apache Web Server](site/apache.md) for information about the current capabilities and setup instructions for each of the supported load balancers. diff --git a/site/apache.md b/site/apache.md index 4cdcc705b8b..77849b8d523 100644 --- a/site/apache.md +++ b/site/apache.md @@ -7,9 +7,7 @@ This page describes how to setup and start a Apache Web Server for load balancin You need to prepare the Docker image for Apache Web Server that enbeds Oracle WebLogic Server Proxy Plugin. - 1. Download and build the Docker image for the Apache Web Server with 12.2.1.3.0 Oracle WebLogic Server Proxy Plugin. See the instructions in - -[https://github.com/oracle/docker-images/tree/master/OracleWebLogic/samples/12213-webtier-apache). + 1. Download and build the Docker image for the Apache Web Server with 12.2.1.3.0 Oracle WebLogic Server Proxy Plugin. See the instructions in [Apache Web Server with Oracle WebLogic Server Proxy Plugin on Docker] (https://github.com/oracle/docker-images/tree/master/OracleWebLogic/samples/12213-webtier-apache). 2. tag your Docker image to `store/oracle/apache:12.2.1.3` using `docker tag` command. @@ -19,7 +17,7 @@ docker tag 12213-apache:latest store/oracle/apache:12.2.1. ``` -More information about the Apache plugin can be found at: [https://docs.oracle.com/middleware/1213/webtier/develop-plugin/apache.htm#PLGWL395) +More information about the Apache plugin can be found at: [Apache Web Server with Oracle WebLogic Server Proxy Plugin on Docker] (https://docs.oracle.com/middleware/1213/webtier/develop-plugin/apache.htm#PLGWL395). Once you have access to the Docker image of the Apache Web Server, you can go ahead follow the instructions below to setup and start Kubernetes artifacts for Apache Web Server. @@ -64,7 +62,7 @@ loadBalancerAppPrepath: /weblogic ``` -The user can then access an application from utsidie of the Kubernetes cluster via `http://:30305/weblogic/,` and the admin can then access the admin console via `http://:30305/console`. +The user can then access an application from utsidie of the Kubernetes cluster via `http://:30305/weblogic/,` and the admin can access the admin console via `http://:30305/console`. The generated Kubernetes yaml files look like the following given the domainUID "domain1". @@ -356,20 +354,6 @@ Here are examples of the Kubernetes artifacts created by the WebLogic Operator: ``` - -AMESPACE NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE - -default deploy/domain1-apache-webtier 1 1 1 1 4h - - - -default po/domain1-apache-webtier-7f7cf44dcf-pqzn7 1/1 Running 1 4h - -PORT(S) AGE - -default svc/domain1-apache-webtier NodePort 10.96.36.137 80:30305/TCP 4h - - NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE bash-4.2$ kubectl get all |grep apache @@ -459,7 +443,7 @@ The generated yaml files will look similar except that the lines that start with If your WebLogic domain is not created by the WebLogic Operator, you need to manually create and start all Kubernetes' artifacts for Apache Web Server. - 1. Create your own custom_mod_wl_apache.conf file, and put it in a local dir, say ``. See the instructions in [https://docs.oracle.com/middleware/1213/webtier/develop-plugin/apache.htm#PLGWL395). + 1. Create your own custom_mod_wl_apache.conf file, and put it in a local dir, say ``. See the instructions in [Apache Web Server with Oracle WebLogic Server Proxy Plugin on Docker](https://docs.oracle.com/middleware/1213/webtier/develop-plugin/apache.htm#PLGWL395). 2. Create the Apache deployment yaml file. See the example above. Note that you need to use the **volumes** and **volumeMounts** to mount `` in to `/config` directory inside the pod that runs Apache web tier. Note that the Apache Web Server needs to be in the same Kubernetes namespace as the WebLogic domains that it needs to access. From 3d48c44e31e57c64552350ff31bd118a9d180da5 Mon Sep 17 00:00:00 2001 From: Dongbo Xiao Date: Fri, 27 Apr 2018 07:01:42 -0700 Subject: [PATCH 106/186] Fix more references Signed-off-by: Dongbo Xiao --- site/apache.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/site/apache.md b/site/apache.md index 77849b8d523..d19a9cf32ca 100644 --- a/site/apache.md +++ b/site/apache.md @@ -7,7 +7,7 @@ This page describes how to setup and start a Apache Web Server for load balancin You need to prepare the Docker image for Apache Web Server that enbeds Oracle WebLogic Server Proxy Plugin. - 1. Download and build the Docker image for the Apache Web Server with 12.2.1.3.0 Oracle WebLogic Server Proxy Plugin. See the instructions in [Apache Web Server with Oracle WebLogic Server Proxy Plugin on Docker] (https://github.com/oracle/docker-images/tree/master/OracleWebLogic/samples/12213-webtier-apache). + 1. Download and build the Docker image for the Apache Web Server with 12.2.1.3.0 Oracle WebLogic Server Proxy Plugin. See the instructions in [Apache Web Server with Oracle WebLogic Server Proxy Plugin on Docker](https://github.com/oracle/docker-images/tree/master/OracleWebLogic/samples/12213-webtier-apache). 2. tag your Docker image to `store/oracle/apache:12.2.1.3` using `docker tag` command. @@ -17,7 +17,7 @@ docker tag 12213-apache:latest store/oracle/apache:12.2.1. ``` -More information about the Apache plugin can be found at: [Apache Web Server with Oracle WebLogic Server Proxy Plugin on Docker] (https://docs.oracle.com/middleware/1213/webtier/develop-plugin/apache.htm#PLGWL395). +More information about the Apache plugin can be found at: [Apache Web Server with Oracle WebLogic Server Proxy Plugin on Docker](https://docs.oracle.com/middleware/1213/webtier/develop-plugin/apache.htm#PLGWL395). Once you have access to the Docker image of the Apache Web Server, you can go ahead follow the instructions below to setup and start Kubernetes artifacts for Apache Web Server. From f28ab3dff1f2dc18a579ca3c27a47fcea5174235 Mon Sep 17 00:00:00 2001 From: Dongbo Xiao Date: Fri, 27 Apr 2018 07:05:27 -0700 Subject: [PATCH 107/186] Minor formating change Signed-off-by: Dongbo Xiao --- site/apache.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/apache.md b/site/apache.md index d19a9cf32ca..cbfacb425f2 100644 --- a/site/apache.md +++ b/site/apache.md @@ -13,7 +13,7 @@ You need to prepare the Docker image for Apache Web Server that enbeds Oracle We ``` -docker tag 12213-apache:latest store/oracle/apache:12.2.1. + docker tag 12213-apache:latest store/oracle/apache:12.2.1. ``` From 4f29d3b6e718f23d6bbedbe0a3053123bfa51f03 Mon Sep 17 00:00:00 2001 From: Dongbo Xiao Date: Fri, 27 Apr 2018 07:06:43 -0700 Subject: [PATCH 108/186] Change command format Signed-off-by: Dongbo Xiao --- site/apache.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/apache.md b/site/apache.md index cbfacb425f2..bba8234a3f8 100644 --- a/site/apache.md +++ b/site/apache.md @@ -13,7 +13,7 @@ You need to prepare the Docker image for Apache Web Server that enbeds Oracle We ``` - docker tag 12213-apache:latest store/oracle/apache:12.2.1. + $ docker tag 12213-apache:latest store/oracle/apache:12.2.1. ``` From a00b89f075e78ea260ea818767c05e35ced1d623 Mon Sep 17 00:00:00 2001 From: Dongbo Xiao Date: Fri, 27 Apr 2018 07:08:22 -0700 Subject: [PATCH 109/186] Fix a typo in docker image name Signed-off-by: Dongbo Xiao --- site/apache.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/apache.md b/site/apache.md index bba8234a3f8..76887b45a57 100644 --- a/site/apache.md +++ b/site/apache.md @@ -13,7 +13,7 @@ You need to prepare the Docker image for Apache Web Server that enbeds Oracle We ``` - $ docker tag 12213-apache:latest store/oracle/apache:12.2.1. + $ docker tag 12213-apache:latest store/oracle/apache:12.2.1.3 ``` From fb8dbc4d79d77317972bb440b4ded9a5247c9c40 Mon Sep 17 00:00:00 2001 From: Dongbo Xiao Date: Fri, 27 Apr 2018 08:10:12 -0700 Subject: [PATCH 110/186] Address review comments in apache.md Signed-off-by: Dongbo Xiao --- site/apache.md | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/site/apache.md b/site/apache.md index 76887b45a57..e776e533812 100644 --- a/site/apache.md +++ b/site/apache.md @@ -5,7 +5,7 @@ This page describes how to setup and start a Apache Web Server for load balancin ## Build Docker Image for Apache Web Server -You need to prepare the Docker image for Apache Web Server that enbeds Oracle WebLogic Server Proxy Plugin. +You need to build the Docker image for Apache Web Server that enbeds Oracle WebLogic Server Proxy Plugin. 1. Download and build the Docker image for the Apache Web Server with 12.2.1.3.0 Oracle WebLogic Server Proxy Plugin. See the instructions in [Apache Web Server with Oracle WebLogic Server Proxy Plugin on Docker](https://github.com/oracle/docker-images/tree/master/OracleWebLogic/samples/12213-webtier-apache). @@ -22,7 +22,7 @@ More information about the Apache plugin can be found at: [Apache Web Server wit Once you have access to the Docker image of the Apache Web Server, you can go ahead follow the instructions below to setup and start Kubernetes artifacts for Apache Web Server. -## When use Apache load balancer with a WebLogic domain created with the WebLogic Operator +## Use Apache load balancer with a WebLogic domain created with the WebLogic Operator Please refer to [Creating a domain using the WebLogic Operator](creating-domain.md) for how to create a domain with the WebLogic Operator. @@ -36,9 +36,9 @@ loadBalancer: APACHE ``` -The `create-weblogic-domain.sh` script installs the Apache Web Server with Oracle WebLogic Server Proxy Plugin into the Kubernetes *cluster* in the same namespace of the *domain*. +The `create-weblogic-domain.sh` script installs the Apache Web Server with Oracle WebLogic Server Proxy Plugin into the Kubernetes *cluster* in the same namespace as the *domain*. -The Apache Web Server will expose a `NodePort` that allows access to the load balancer from the outside of the Kubernetes cluster. The port is controlled by the following setting in `create-weblogic-domain-inputs.yaml` file: +The Apache Web Server will expose a `NodePort` that allows access to the load balancer from outside of the Kubernetes cluster. The port is configured by setting 'loadBalancerWebPort' in `create-weblogic-domain-inputs.yaml` file. ``` @@ -48,7 +48,7 @@ loadBalancerWebPort: 30305 ``` -The user can access an application from utsidie of the Kubernetes cluster via http://:30305/. +The user can access an application from outside of the Kubernetes cluster via http://:30305/. ### Use the default plugin WL module configuration @@ -394,7 +394,7 @@ domain1-apache-webtier 2h ### Use your own plugin WL module configuration -Optionally you can fine turn the behavior of the Apache plugin by providing your own Apache plugin configuration for `create-weblogic-domain.sh` script to use. You put your custom_mod_wl_apache.conf file in a local directory, say `/scratch/apache/config` , and specify this location in the `create-weblogic-domain-inputs.yaml` file as follows. +You can fine turn the behavior of the Apache plugin by providing your own Apache plugin configuration. You put your custom_mod_wl_apache.conf file in a local directory, for example `` , and specify this location in the `create-weblogic-domain-inputs.yaml` file as follows. ``` @@ -402,13 +402,13 @@ Optionally you can fine turn the behavior of the Apache plugin by providing your # By default, the VolumePath is empty, which will cause the volume mount be disabled -loadBalancerVolumePath: /scratch/apache/config +loadBalancerVolumePath: ``` -Once the loadBalancerVolumePath is specified, the `create-weblogic-domain.sh` script will use the custom_mod_wl_apache.config file in `/scratch/apache/config` directory to replace what is in the Docker image. +After the loadBalancerVolumePath is specified, the `create-weblogic-domain.sh` script will use the custom_mod_wl_apache.config file in `` directory to replace what is in the Docker image. -The generated yaml files will look similar except that the lines that start with "#" will be un-commented like the following. +The generated yaml files will look similar except with un-commented entries like bellow. ``` @@ -418,7 +418,7 @@ The generated yaml files will look similar except that the lines that start with hostPath: - path: /scratch/apache/config + path: containers: @@ -438,14 +438,14 @@ The generated yaml files will look similar except that the lines that start with -## When use Apache load balancer with a manually created WebLogic Domain +## Use the Apache load balancer with a manually created WebLogic Domain If your WebLogic domain is not created by the WebLogic Operator, you need to manually create and start all Kubernetes' artifacts for Apache Web Server. - 1. Create your own custom_mod_wl_apache.conf file, and put it in a local dir, say ``. See the instructions in [Apache Web Server with Oracle WebLogic Server Proxy Plugin on Docker](https://docs.oracle.com/middleware/1213/webtier/develop-plugin/apache.htm#PLGWL395). + 1. Create your own custom_mod_wl_apache.conf file, and put it in a local dir, say ``. See the instructions in [Apache Web Server with Oracle WebLogic Server Proxy Plugin on Docker](https://docs.oracle.com/middleware/1213/webtier/develop-plugin/apache.htm#PLGWL395). - 2. Create the Apache deployment yaml file. See the example above. Note that you need to use the **volumes** and **volumeMounts** to mount `` in to `/config` directory inside the pod that runs Apache web tier. Note that the Apache Web Server needs to be in the same Kubernetes namespace as the WebLogic domains that it needs to access. + 2. Create the Apache deployment yaml file. See the example above. Note that you need to use the **volumes** and **volumeMounts** to mount `` in to `/config` directory inside the pod that runs Apache web tier. Note that the Apache Web Server needs to be in the same Kubernetes namespace as the WebLogic domains that it needs to access. 3. Create a RBAC yaml file. See the example above From d62efb36586a3a5d9acfc55c85e8dd787c62378f Mon Sep 17 00:00:00 2001 From: Dongbo Xiao Date: Fri, 27 Apr 2018 08:13:49 -0700 Subject: [PATCH 111/186] Fix backticks Signed-off-by: Dongbo Xiao --- site/apache.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/site/apache.md b/site/apache.md index e776e533812..06a707e20f0 100644 --- a/site/apache.md +++ b/site/apache.md @@ -52,7 +52,7 @@ The user can access an application from outside of the Kubernetes cluster via ht ### Use the default plugin WL module configuration -By default, the Apache Docker image supports a simple WebLogic server proxy plugin configuration for a single WebLogic domain with an admin server and a cluster. The `create-weblogic-domain.sh` script automatically customizes the default behavior based on your domain configuration. The default setting only supports the type of load balancing that uses the root path ("/"). You can further customize the root path of the load balancer withloadBalancerAppPrepath property in the `create-weblogic-domain-inputs.yaml` file. +By default, the Apache Docker image supports a simple WebLogic server proxy plugin configuration for a single WebLogic domain with an admin server and a cluster. The `create-weblogic-domain.sh` script automatically customizes the default behavior based on your domain configuration. The default setting only supports the type of load balancing that uses the root path ("/"). You can further customize the root path of the load balancer with `loadBalancerAppPrepath` property in the `create-weblogic-domain-inputs.yaml` file. ``` @@ -406,7 +406,7 @@ loadBalancerVolumePath: ``` -After the loadBalancerVolumePath is specified, the `create-weblogic-domain.sh` script will use the custom_mod_wl_apache.config file in `` directory to replace what is in the Docker image. +After the `loadBalancerVolumePath` property is specified, the `create-weblogic-domain.sh` script will use the custom_mod_wl_apache.config file in `` directory to replace what is in the Docker image. The generated yaml files will look similar except with un-commented entries like bellow. From c95e6c23dadb2e33ed5ebf48ccf4338e5f3f3a05 Mon Sep 17 00:00:00 2001 From: Russell Gold Date: Fri, 27 Apr 2018 12:11:55 -0400 Subject: [PATCH 112/186] enable unit tests during integration testing --- operator/src/test/java/oracle/kubernetes/TestUtils.java | 2 +- pom.xml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/operator/src/test/java/oracle/kubernetes/TestUtils.java b/operator/src/test/java/oracle/kubernetes/TestUtils.java index c3fcb878384..fbcc95a8e7e 100644 --- a/operator/src/test/java/oracle/kubernetes/TestUtils.java +++ b/operator/src/test/java/oracle/kubernetes/TestUtils.java @@ -38,7 +38,7 @@ private static Boolean checkKubernetes() { PrintStream savedOut = System.out; System.setOut(new PrintStream(new ByteArrayOutputStream())); try { - CommandLine cmdLine = CommandLine.parse("kubectl cluster-info"); + CommandLine cmdLine = CommandLine.parse("kubectl cluster-info dump"); DefaultExecutor executor = new DefaultExecutor(); executor.execute(cmdLine); return true; diff --git a/pom.xml b/pom.xml index 786cc658ee9..ef6ecf9f01d 100644 --- a/pom.xml +++ b/pom.xml @@ -117,7 +117,6 @@ integration-tests ${project.basedir}/src/integration-test/java - true From f5122e661f99839fe9ecefffe8bcbf6f32541ba5 Mon Sep 17 00:00:00 2001 From: Rosemary Marano Date: Fri, 27 Apr 2018 12:13:02 -0400 Subject: [PATCH 113/186] incorporated review comments --- README.md | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/README.md b/README.md index 68d4e196972..fa26a7dab72 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Many organizations are exploring, testing, or actively moving application worklo Oracle has been working with the WebLogic community to find ways to make it as easy as possible for organizations using WebLogic Server to run important workloads, to move those workloads into the cloud. One aspect of that effort is the creation of the Oracle WebLogic Server Kubernetes Operator. This release of the Operator provides a number of features to assist with the management of WebLogic domains in a Kubernetes environment, including: -* A mechanism to create a WebLogic domain on a Kubernetes persistent volume.This persistent volume can reside in NFS. +* A mechanism to create a WebLogic domain on a Kubernetes persistent volume. This persistent volume can reside in NFS. * A mechanism to define a WebLogic domain as a Kubernetes resource (using a Kubernetes custom resource definition). * The ability to automatically start servers based on declarative startup parameters and desired states. * The ability to manage a WebLogic configured or dynamic cluster. @@ -160,15 +160,6 @@ Please refer to [Shutting down a domain](site/shutdown-domain.md) for informatio ## Load balancing -### Traefik - -Please refer to [Load balancing with Traefik](site/traefik.md) for information about the current capabilities. - -### Voyager - - -### Apache - [comment]: # (Exporting operator logs to ELK. The operator provides an option to export its log files to the ELK stack. Please refer to [ELK integration]site/elk.md for information about this capability.) From dbd436db48e14c2cc16059d23c3555fac9ae324f Mon Sep 17 00:00:00 2001 From: Russell Gold Date: Fri, 27 Apr 2018 12:14:44 -0400 Subject: [PATCH 114/186] Start idolate domain presence logic --- .../operator/DomainPresenceControl.java | 84 ++++++++ .../java/oracle/kubernetes/operator/Main.java | 116 +++-------- .../operator/helpers/CallBuilder.java | 8 +- .../helpers/SynchronousCallFactory.java | 8 +- .../operator/DomainNormalizationTest.java | 188 ++++++++++++++++++ .../operator/DomainPresenceTest.java | 18 +- 6 files changed, 325 insertions(+), 97 deletions(-) create mode 100644 operator/src/main/java/oracle/kubernetes/operator/DomainPresenceControl.java create mode 100644 operator/src/test/java/oracle/kubernetes/operator/DomainNormalizationTest.java diff --git a/operator/src/main/java/oracle/kubernetes/operator/DomainPresenceControl.java b/operator/src/main/java/oracle/kubernetes/operator/DomainPresenceControl.java new file mode 100644 index 00000000000..d8876df9ae4 --- /dev/null +++ b/operator/src/main/java/oracle/kubernetes/operator/DomainPresenceControl.java @@ -0,0 +1,84 @@ +// Copyright 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator; + +import java.util.ArrayList; +import java.util.concurrent.ScheduledFuture; + +import oracle.kubernetes.operator.helpers.DomainPresenceInfo; +import oracle.kubernetes.weblogic.domain.v1.ClusterStartup; +import oracle.kubernetes.weblogic.domain.v1.DomainSpec; +import oracle.kubernetes.weblogic.domain.v1.ServerStartup; + +class DomainPresenceControl { + + // This method fills in null values which would interfere with the general DomainSpec.equals() method + static void normalizeDomainSpec(DomainSpec spec) { + normalizeImage(spec); + normalizeImagePullPolicy(spec); + normalizeExportT3Channels(spec); + normalizeStartupControl(spec); + normalizeServerStartup(spec); + normalizeClusterStartup(spec); + normalizeReplicas(spec); + } + + private static void normalizeImage(DomainSpec spec) { + if (isNotDefined(spec.getImage())) spec.setImage(KubernetesConstants.DEFAULT_IMAGE); + } + + private static void normalizeImagePullPolicy(DomainSpec spec) { + if (isNotDefined(spec.getImagePullPolicy())) { + spec.setImagePullPolicy((spec.getImage().endsWith(KubernetesConstants.LATEST_IMAGE_SUFFIX)) + ? KubernetesConstants.ALWAYS_IMAGEPULLPOLICY + : KubernetesConstants.IFNOTPRESENT_IMAGEPULLPOLICY); + } + } + + private static void normalizeExportT3Channels(DomainSpec spec) { + if (spec.getExportT3Channels() == null) + spec.setExportT3Channels(new ArrayList<>()); + } + + private static void normalizeStartupControl(DomainSpec spec) { + if (isNotDefined(spec.getStartupControl())) + spec.setStartupControl(StartupControlConstants.AUTO_STARTUPCONTROL); + } + + private static void normalizeServerStartup(DomainSpec spec) { + if (spec.getServerStartup() == null) + spec.setServerStartup(new ArrayList<>()); + else + for (ServerStartup ss : spec.getServerStartup()) { + if (ss.getDesiredState() == null) ss.setDesiredState(WebLogicConstants.RUNNING_STATE); + if (ss.getEnv() == null) ss.setEnv(new ArrayList<>()); + } + } + + private static void normalizeClusterStartup(DomainSpec spec) { + if (spec.getClusterStartup() == null) + spec.setClusterStartup(new ArrayList<>()); + else + for (ClusterStartup cs : spec.getClusterStartup()) { + if (cs.getDesiredState() == null) cs.setDesiredState(WebLogicConstants.RUNNING_STATE); + if (cs.getEnv() == null) cs.setEnv(new ArrayList<>()); + if (cs.getReplicas() == null) cs.setReplicas(1); + } + } + + private static void normalizeReplicas(DomainSpec spec) { + if (spec.getReplicas() == null) spec.setReplicas(1); + } + + private static boolean isNotDefined(String value) { + return value == null || value.length() == 0; + } + + static void cancelDomainStatusUpdating(DomainPresenceInfo info) { + ScheduledFuture statusUpdater = info.getStatusUpdater().getAndSet(null); + if (statusUpdater != null) { + statusUpdater.cancel(true); + } + } +} diff --git a/operator/src/main/java/oracle/kubernetes/operator/Main.java b/operator/src/main/java/oracle/kubernetes/operator/Main.java index 20e0a72f233..58243519671 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/Main.java +++ b/operator/src/main/java/oracle/kubernetes/operator/Main.java @@ -75,11 +75,9 @@ import oracle.kubernetes.operator.work.NextAction; import oracle.kubernetes.operator.work.Packet; import oracle.kubernetes.operator.work.Step; -import oracle.kubernetes.weblogic.domain.v1.ClusterStartup; import oracle.kubernetes.weblogic.domain.v1.Domain; import oracle.kubernetes.weblogic.domain.v1.DomainList; import oracle.kubernetes.weblogic.domain.v1.DomainSpec; -import oracle.kubernetes.weblogic.domain.v1.ServerStartup; /** * A Kubernetes Operator for WebLogic. @@ -275,60 +273,6 @@ public void onThrowable(Packet packet, Throwable throwable) { // // ----------------------------------------------------------------------------- - private static void normalizeDomainSpec(DomainSpec spec) { - // Normalize DomainSpec so that equals() will work correctly - String imageName = spec.getImage(); - if (isNotDefined(imageName)) { - spec.setImage(imageName = KubernetesConstants.DEFAULT_IMAGE); - } - if (isNotDefined(spec.getImagePullPolicy())) { - spec.setImagePullPolicy((imageName.endsWith(KubernetesConstants.LATEST_IMAGE_SUFFIX)) - ? KubernetesConstants.ALWAYS_IMAGEPULLPOLICY - : KubernetesConstants.IFNOTPRESENT_IMAGEPULLPOLICY); - } - if (spec.getExportT3Channels() == null) { - spec.setExportT3Channels(new ArrayList<>()); - } - String startupControl = spec.getStartupControl(); - if (isNotDefined(startupControl)) { - spec.setStartupControl(StartupControlConstants.AUTO_STARTUPCONTROL); - } - if (spec.getServerStartup() == null) { - spec.setServerStartup(new ArrayList<>()); - } else { - for (ServerStartup ss : spec.getServerStartup()) { - if (ss.getDesiredState() == null) { - ss.setDesiredState(WebLogicConstants.RUNNING_STATE); - } - if (ss.getEnv() == null) { - ss.setEnv(new ArrayList<>()); - } - } - } - if (spec.getClusterStartup() == null) { - spec.setClusterStartup(new ArrayList<>()); - } else { - for (ClusterStartup cs : spec.getClusterStartup()) { - if (cs.getDesiredState() == null) { - cs.setDesiredState(WebLogicConstants.RUNNING_STATE); - } - if (cs.getEnv() == null) { - cs.setEnv(new ArrayList<>()); - } - if (cs.getReplicas() == null) { - cs.setReplicas(1); - } - } - } - if (spec.getReplicas() == null) { - spec.setReplicas(1); - } - } - - private static boolean isNotDefined(String value) { - return value == null || value.length() == 0; - } - /** * Restarts the admin server, if already running * @@ -454,13 +398,6 @@ public void onThrowable(Packet packet, Throwable throwable) { } } - private static void cancelDomainStatusUpdating(DomainPresenceInfo info) { - ScheduledFuture statusUpdater = info.getStatusUpdater().getAndSet(null); - if (statusUpdater != null) { - statusUpdater.cancel(true); - } - } - private static void doCheckAndCreateDomainPresence(Domain dom) { doCheckAndCreateDomainPresence(dom, false, false, null, null); } @@ -477,7 +414,7 @@ private static void doCheckAndCreateDomainPresence(Domain dom, boolean explicitR || explicitRestartClusters != null; DomainSpec spec = dom.getSpec(); - normalizeDomainSpec(spec); + DomainPresenceControl.normalizeDomainSpec(spec); String domainUID = spec.getDomainUID(); DomainPresenceInfo created = new DomainPresenceInfo(dom); @@ -599,7 +536,7 @@ private static void deleteDomainPresence(String namespace, String domainUID) { DomainPresenceInfo info = domains.remove(domainUID); if (info != null) { - cancelDomainStatusUpdating(info); + DomainPresenceControl.cancelDomainStatusUpdating(info); } domainUpdaters.startFiber(domainUID, new DeleteDomainStep(namespace, domainUID), new Packet(), new CompletionCallback() { @@ -623,7 +560,7 @@ public void onThrowable(Packet packet, Throwable throwable) { * @return the collection of target namespace names */ private static Collection getTargetNamespaces(String namespace) { - Collection targetNamespaces = new ArrayList(); + Collection targetNamespaces = new ArrayList<>(); String tnValue = tuningAndConfig.get("targetNamespaces"); if (tnValue != null) { @@ -678,10 +615,6 @@ public static boolean getStopping() { return stopping.get(); } - private static DomainWatcher createDomainWatcher(String namespace, String initialResourceVersion) { - return DomainWatcher.create(factory, namespace, initialResourceVersion, Main::dispatchDomainWatch, stopping); - } - private static EventWatcher createEventWatcher(String namespace, String initialResourceVersion) { return EventWatcher.create(factory, namespace, READINESS_PROBE_FAILURE_EVENT_FILTER, initialResourceVersion, Main::dispatchEventWatch, stopping); @@ -958,7 +891,7 @@ private static String getOperatorNamespace() { private static class V1beta1IngressListResponseStep extends ResponseStep { private final String ns; - public V1beta1IngressListResponseStep(Step domainList, String ns) { + V1beta1IngressListResponseStep(Step domainList, String ns) { super(domainList); this.ns = ns; } @@ -998,10 +931,10 @@ public NextAction onSuccess(Packet packet, V1beta1IngressList result, int status private static class V1ServiceListResponseStep extends ResponseStep { private final String ns; - public V1ServiceListResponseStep(String ns, V1beta1IngressListResponseStep ingressListResponseStep) { - super(Main.callBuilderFactory.create().with($ -> { - $.labelSelector = LabelConstants.DOMAINUID_LABEL + "," + LabelConstants.CREATEDBYOPERATOR_LABEL; - }).listIngressAsync(ns, ingressListResponseStep)); + V1ServiceListResponseStep(String ns, V1beta1IngressListResponseStep ingressListResponseStep) { + super(Main.callBuilderFactory.create() + .with($ -> $.labelSelector = LabelConstants.DOMAINUID_LABEL + "," + LabelConstants.CREATEDBYOPERATOR_LABEL) + .listIngressAsync(ns, ingressListResponseStep)); this.ns = ns; } @@ -1046,10 +979,10 @@ public NextAction onSuccess(Packet packet, V1ServiceList result, int statusCode, private static class V1EventListResponseStep extends ResponseStep { private final String ns; - public V1EventListResponseStep(String ns, V1ServiceListResponseStep serviceListResponseStep) { - super(Main.callBuilderFactory.create().with($ -> { - $.labelSelector = LabelConstants.DOMAINUID_LABEL + "," + LabelConstants.CREATEDBYOPERATOR_LABEL; - }).listServiceAsync(ns, serviceListResponseStep)); + V1EventListResponseStep(String ns, V1ServiceListResponseStep serviceListResponseStep) { + super(Main.callBuilderFactory.create() + .with($ -> $.labelSelector = LabelConstants.DOMAINUID_LABEL + "," + LabelConstants.CREATEDBYOPERATOR_LABEL) + .listServiceAsync(ns, serviceListResponseStep)); this.ns = ns; } @@ -1079,10 +1012,10 @@ public NextAction onSuccess(Packet packet, V1EventList result, int statusCode, private static class V1PodListResponseStep extends ResponseStep { private final String ns; - public V1PodListResponseStep(String ns, V1EventListResponseStep eventListResponseStep) { - super(Main.callBuilderFactory.create().with($ -> { - $.fieldSelector = Main.READINESS_PROBE_FAILURE_EVENT_FILTER; - }).listEventAsync(ns, eventListResponseStep)); + V1PodListResponseStep(String ns, V1EventListResponseStep eventListResponseStep) { + super(Main.callBuilderFactory.create() + .with($ -> $.fieldSelector = Main.READINESS_PROBE_FAILURE_EVENT_FILTER) + .listEventAsync(ns, eventListResponseStep)); this.ns = ns; } @@ -1122,7 +1055,7 @@ public NextAction onSuccess(Packet packet, V1PodList result, int statusCode, private static class ExistingDomainListResponseStep extends ResponseStep { private final String ns; - public ExistingDomainListResponseStep(String ns) { + ExistingDomainListResponseStep(String ns) { super(null); this.ns = ns; } @@ -1137,18 +1070,23 @@ public NextAction onFailure(Packet packet, ApiException e, int statusCode, } @Override - public NextAction onSuccess(Packet packet, DomainList result, int statusCode, - Map> responseHeaders) { + public NextAction onSuccess(Packet packet, DomainList result, int statusCode, Map> responseHeaders) { if (result != null) { for (Domain dom : result.getItems()) { doCheckAndCreateDomainPresence(dom); } } - // main logic now happens in the watch handlers - domainWatchers.put(ns, - createDomainWatcher(ns, result != null ? result.getMetadata().getResourceVersion() : "")); + domainWatchers.put(ns, createDomainWatcher(ns, getResourceVersion(result))); return doNext(packet); } + + String getResourceVersion(DomainList result) { + return result != null ? result.getMetadata().getResourceVersion() : ""; + } + + private static DomainWatcher createDomainWatcher(String namespace, String initialResourceVersion) { + return DomainWatcher.create(factory, namespace, initialResourceVersion, Main::dispatchDomainWatch, stopping); + } } } diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java index df2b6458c54..18cb86931b3 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java @@ -174,7 +174,7 @@ public DomainList listDomain(String namespace) throws ApiException { String _continue = ""; ApiClient client = helper.take(); try { - return CALL_FACTORY.getDomainList(client, namespace, _continue, pretty, fieldSelector, includeUninitialized, labelSelector, + return CALL_FACTORY.getDomainList(client, namespace, pretty, _continue, fieldSelector, includeUninitialized, labelSelector, limit, resourceVersion, timeoutSeconds, watch); } finally { helper.recycle(client); @@ -614,7 +614,7 @@ public V1PersistentVolumeList listPersistentVolume() throws ApiException { String _continue = ""; ApiClient client = helper.take(); try { - return CALL_FACTORY.listPersistentVolumes(_continue, client, pretty, fieldSelector, includeUninitialized, + return CALL_FACTORY.listPersistentVolumes(client, pretty, _continue, fieldSelector, includeUninitialized, labelSelector, limit, resourceVersion, timeoutSeconds, watch); } finally { helper.recycle(client); @@ -984,7 +984,7 @@ public V1SelfSubjectRulesReview createSelfSubjectRulesReview(ApiClient client, V } @Override - public V1PersistentVolumeList listPersistentVolumes(String _continue, ApiClient client, String pretty, String fieldSelector, Boolean includeUninitialized, String labelSelector, Integer limit, String resourceVersion, Integer timeoutSeconds, Boolean watch) throws ApiException { + public V1PersistentVolumeList listPersistentVolumes(ApiClient client, String pretty, String _continue, String fieldSelector, Boolean includeUninitialized, String labelSelector, Integer limit, String resourceVersion, Integer timeoutSeconds, Boolean watch) throws ApiException { return new CoreV1Api(client).listPersistentVolume(pretty, _continue, fieldSelector, includeUninitialized, labelSelector, limit, resourceVersion, timeoutSeconds, watch); } @@ -995,7 +995,7 @@ public VersionInfo getVersionCode(ApiClient client) throws ApiException { } @Override - public DomainList getDomainList(ApiClient client, String namespace, String _continue, String pretty, String fieldSelector, Boolean includeUninitialized, String labelSelector, Integer limit, String resourceVersion, Integer timeoutSeconds, Boolean watch) throws ApiException { + public DomainList getDomainList(ApiClient client, String namespace, String pretty, String _continue, String fieldSelector, Boolean includeUninitialized, String labelSelector, Integer limit, String resourceVersion, Integer timeoutSeconds, Boolean watch) throws ApiException { return new WeblogicApi(client).listWebLogicOracleV1NamespacedDomain(namespace, pretty, _continue, fieldSelector, includeUninitialized, labelSelector, limit, resourceVersion, timeoutSeconds, watch); } diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/SynchronousCallFactory.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/SynchronousCallFactory.java index d81b331b0f8..366a4fb659c 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/SynchronousCallFactory.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/SynchronousCallFactory.java @@ -1,3 +1,6 @@ +// Copyright 2018 Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + package oracle.kubernetes.operator.helpers; import io.kubernetes.client.ApiClient; @@ -8,6 +11,7 @@ import io.kubernetes.client.models.VersionInfo; import oracle.kubernetes.weblogic.domain.v1.DomainList; + public interface SynchronousCallFactory { V1beta1CustomResourceDefinition readCustomResourceDefinition(ApiClient client, String name, String pretty, Boolean exact, Boolean export) throws ApiException; @@ -16,9 +20,9 @@ public interface SynchronousCallFactory { V1SelfSubjectRulesReview createSelfSubjectRulesReview(ApiClient client, V1SelfSubjectRulesReview body, String pretty) throws ApiException; - V1PersistentVolumeList listPersistentVolumes(String _continue, ApiClient client, String pretty, String fieldSelector, Boolean includeUninitialized, String labelSelector, Integer limit, String resourceVersion, Integer timeoutSeconds, Boolean watch) throws ApiException; + V1PersistentVolumeList listPersistentVolumes(ApiClient client, String pretty, String _continue, String fieldSelector, Boolean includeUninitialized, String labelSelector, Integer limit, String resourceVersion, Integer timeoutSeconds, Boolean watch) throws ApiException; VersionInfo getVersionCode(ApiClient client) throws ApiException; - DomainList getDomainList(ApiClient client, String namespace, String _continue, String pretty, String fieldSelector, Boolean includeUninitialized, String labelSelector, Integer limit, String resourceVersion, Integer timeoutSeconds, Boolean watch) throws ApiException; + DomainList getDomainList(ApiClient client, String namespace, String pretty, String _continue, String fieldSelector, Boolean includeUninitialized, String labelSelector, Integer limit, String resourceVersion, Integer timeoutSeconds, Boolean watch) throws ApiException; } diff --git a/operator/src/test/java/oracle/kubernetes/operator/DomainNormalizationTest.java b/operator/src/test/java/oracle/kubernetes/operator/DomainNormalizationTest.java new file mode 100644 index 00000000000..4337f6251fd --- /dev/null +++ b/operator/src/test/java/oracle/kubernetes/operator/DomainNormalizationTest.java @@ -0,0 +1,188 @@ +// Copyright 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator; + +import java.util.ArrayList; +import java.util.List; + +import com.meterware.simplestub.Memento; + +import io.kubernetes.client.models.V1EnvVar; +import oracle.kubernetes.TestUtils; +import oracle.kubernetes.weblogic.domain.v1.ClusterStartup; +import oracle.kubernetes.weblogic.domain.v1.DomainSpec; +import oracle.kubernetes.weblogic.domain.v1.ServerStartup; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static java.util.Arrays.asList; +import static java.util.Collections.singletonList; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; + +public class DomainNormalizationTest { + + private static final String LATEST_IMAGE = "store/oracle/weblogic:latest"; + private static final String IMAGE_PULL_POLICY = "Never"; + private static final String[] T3_CHANNELS = {"channel1", "channel2"}; + private static final String STARTUP_CONTROL = "ADMIN"; + private static final ServerStartup[] SERVER_STARTUPS = createServerStartups(); + private static final ClusterStartup[] CLUSTER_STARTUPS = createClusterStartups(); + private static final int REPLICAS = 5; + private static final V1EnvVar ENV_VAR1 = new V1EnvVar().name("name1").value("value1"); + private static final V1EnvVar ENV_VAR2 = new V1EnvVar().name("name2").value("value2"); + private static final V1EnvVar ENV_VAR3 = new V1EnvVar().name("name3").value("value3"); + + private final DomainSpec domainSpec = new DomainSpec(); + private List mementos = new ArrayList<>(); + + @Before + public void setUp() throws Exception { + mementos.add(TestUtils.silenceOperatorLogger()); + } + + @After + public void tearDown() throws Exception { + for (Memento memento : mementos) memento.revert(); + } + + @Test + public void whenDomainSpecHasNulls_normalizationSetsDefaultValues() throws Exception { + domainSpec.setExportT3Channels(null); + domainSpec.setServerStartup(null); + domainSpec.setClusterStartup(null); + domainSpec.setReplicas(null); + + DomainPresenceControl.normalizeDomainSpec(domainSpec); + + assertThat(domainSpec.getImage(), equalTo(KubernetesConstants.DEFAULT_IMAGE)); + assertThat(domainSpec.getImagePullPolicy(), equalTo(KubernetesConstants.IFNOTPRESENT_IMAGEPULLPOLICY)); + assertThat(domainSpec.getExportT3Channels(), empty()); + assertThat(domainSpec.getStartupControl(), equalTo(StartupControlConstants.AUTO_STARTUPCONTROL)); + assertThat(domainSpec.getServerStartup(), empty()); + assertThat(domainSpec.getClusterStartup(), empty()); + assertThat(domainSpec.getReplicas(), equalTo(1)); + } + + @Test + public void whenDomainSpecHasDefinedValues_normalizationDoesNotChangeThem() throws Exception { + domainSpec.setImage(LATEST_IMAGE); + domainSpec.setImagePullPolicy(IMAGE_PULL_POLICY); + domainSpec.setExportT3Channels(asList(T3_CHANNELS)); + domainSpec.setStartupControl(STARTUP_CONTROL); + domainSpec.setServerStartup(asList(SERVER_STARTUPS)); + domainSpec.setClusterStartup(asList(CLUSTER_STARTUPS)); + domainSpec.setReplicas(REPLICAS); + + DomainPresenceControl.normalizeDomainSpec(domainSpec); + + assertThat(domainSpec.getImage(), equalTo(LATEST_IMAGE)); + assertThat(domainSpec.getImagePullPolicy(), equalTo(IMAGE_PULL_POLICY)); + assertThat(domainSpec.getExportT3Channels(), contains(T3_CHANNELS)); + assertThat(domainSpec.getStartupControl(), equalTo(STARTUP_CONTROL)); + assertThat(domainSpec.getServerStartup(), contains(SERVER_STARTUPS)); + assertThat(domainSpec.getClusterStartup(), contains(CLUSTER_STARTUPS)); + assertThat(domainSpec.getReplicas(), equalTo(REPLICAS)); + } + + private static ServerStartup[] createServerStartups() { + return new ServerStartup[] { + new ServerStartup().withDesiredState("STANDBY") + .withEnv(asList(ENV_VAR1, ENV_VAR2)), + new ServerStartup().withDesiredState("RUNNING") + .withEnv(singletonList(ENV_VAR3)) + }; + } + + private static ClusterStartup[] createClusterStartups() { + return new ClusterStartup[]{ + new ClusterStartup() + .withDesiredState("ADMIN") + .withEnv(asList(ENV_VAR1, ENV_VAR2, ENV_VAR3)) + .withReplicas(3) + }; + } + + @Test + public void whenDomainSpecHasLatestImageAndNoPullPolicy_normalizationSetsAlwaysPull() throws Exception { + domainSpec.setImage(LATEST_IMAGE); + + DomainPresenceControl.normalizeDomainSpec(domainSpec); + + assertThat(domainSpec.getImagePullPolicy(), equalTo(KubernetesConstants.ALWAYS_IMAGEPULLPOLICY)); + } + + @Test + public void whenDomainSpecHasServerStartupsWithoutDesiredState_normalizationSetsRunningState() throws Exception { + domainSpec.setServerStartup(asList( + new ServerStartup().withServerName("server1").withEnv(singletonList(ENV_VAR1)), + new ServerStartup().withServerName("server2").withEnv(asList(ENV_VAR2, ENV_VAR3)))); + + DomainPresenceControl.normalizeDomainSpec(domainSpec); + + assertThat(domainSpec.getServerStartup(), hasSize(2)); + assertThat(domainSpec.getServerStartup().get(0).getServerName(), equalTo("server1")); + assertThat(domainSpec.getServerStartup().get(0).getDesiredState(), equalTo(WebLogicConstants.RUNNING_STATE)); + assertThat(domainSpec.getServerStartup().get(0).getEnv(), contains(ENV_VAR1)); + + assertThat(domainSpec.getServerStartup().get(1).getServerName(), equalTo("server2")); + assertThat(domainSpec.getServerStartup().get(1).getDesiredState(), equalTo(WebLogicConstants.RUNNING_STATE)); + assertThat(domainSpec.getServerStartup().get(1).getEnv(), contains(ENV_VAR2, ENV_VAR3)); + } + + @Test + public void whenDomainSpecHasServerStartupsWithoutEnv_normalizationSetsEmptyList() throws Exception { + domainSpec.setServerStartup(asList( + new ServerStartup().withServerName("server1").withDesiredState("ADMIN").withEnv(null), + new ServerStartup().withServerName("server2").withDesiredState("STANDBY"))); + + DomainPresenceControl.normalizeDomainSpec(domainSpec); + + assertThat(domainSpec.getServerStartup(), hasSize(2)); + assertThat(domainSpec.getServerStartup().get(0).getServerName(), equalTo("server1")); + assertThat(domainSpec.getServerStartup().get(0).getDesiredState(), equalTo("ADMIN")); + assertThat(domainSpec.getServerStartup().get(0).getEnv(), empty()); + + assertThat(domainSpec.getServerStartup().get(1).getServerName(), equalTo("server2")); + assertThat(domainSpec.getServerStartup().get(1).getDesiredState(), equalTo("STANDBY")); + assertThat(domainSpec.getServerStartup().get(1).getEnv(), empty()); + } + + @Test + public void whenDomainSpecHasClusterStartupsWithoutDesiredState_normalizationSetsRunningState() throws Exception { + domainSpec.setClusterStartup(singletonList( + new ClusterStartup().withClusterName("cluster1").withEnv(asList(ENV_VAR2, ENV_VAR3)))); + + DomainPresenceControl.normalizeDomainSpec(domainSpec); + + assertThat(domainSpec.getClusterStartup(), hasSize(1)); + assertThat(domainSpec.getClusterStartup().get(0).getClusterName(), equalTo("cluster1")); + assertThat(domainSpec.getClusterStartup().get(0).getDesiredState(), equalTo(WebLogicConstants.RUNNING_STATE)); + assertThat(domainSpec.getClusterStartup().get(0).getEnv(), contains(ENV_VAR2, ENV_VAR3)); + } + + @Test + public void whenDomainSpecHasClusterStartupsWithoutEnv_normalizationSetsEmptyList() throws Exception { + domainSpec.setClusterStartup(asList( + new ClusterStartup().withClusterName("cluster1").withDesiredState("ADMIN"), + new ClusterStartup().withClusterName("cluster2").withDesiredState("STANDBY").withEnv(null))); + + DomainPresenceControl.normalizeDomainSpec(domainSpec); + + assertThat(domainSpec.getClusterStartup(), hasSize(2)); + assertThat(domainSpec.getClusterStartup().get(0).getClusterName(), equalTo("cluster1")); + assertThat(domainSpec.getClusterStartup().get(0).getDesiredState(), equalTo("ADMIN")); + assertThat(domainSpec.getClusterStartup().get(0).getEnv(), empty()); + + assertThat(domainSpec.getClusterStartup().get(1).getClusterName(), equalTo("cluster2")); + assertThat(domainSpec.getClusterStartup().get(1).getDesiredState(), equalTo("STANDBY")); + assertThat(domainSpec.getClusterStartup().get(1).getEnv(), empty()); + } + +} diff --git a/operator/src/test/java/oracle/kubernetes/operator/DomainPresenceTest.java b/operator/src/test/java/oracle/kubernetes/operator/DomainPresenceTest.java index 0d54d8c3da8..1c0085779ce 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/DomainPresenceTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/DomainPresenceTest.java @@ -5,6 +5,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.concurrent.ScheduledFuture; import com.meterware.simplestub.Memento; import com.meterware.simplestub.StaticStubSupport; @@ -22,6 +23,7 @@ import oracle.kubernetes.TestUtils; import oracle.kubernetes.operator.builders.StubWatchFactory; import oracle.kubernetes.operator.helpers.CallBuilder; +import oracle.kubernetes.operator.helpers.DomainPresenceInfo; import oracle.kubernetes.operator.helpers.SynchronousCallFactory; import oracle.kubernetes.operator.work.AsyncCallTestSupport; import oracle.kubernetes.weblogic.domain.v1.DomainList; @@ -31,6 +33,8 @@ import org.junit.Test; import static com.meterware.simplestub.Stub.createStub; +import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.junit.MatcherAssert.assertThat; public class DomainPresenceTest { @@ -61,6 +65,16 @@ public void watchPresenceWithNoPreexistingData_doesNothing() throws Exception { Main.begin(); } + @Test + public void afterCancelDomainStatusUpdating_statusUpdaterIsNull() throws Exception { + DomainPresenceInfo info = new DomainPresenceInfo("namespace"); + info.getStatusUpdater().getAndSet(createStub(ScheduledFuture.class)); + + DomainPresenceControl.cancelDomainStatusUpdating(info); + + assertThat(info.getStatusUpdater().get(), nullValue()); + } + static abstract class SynchronousCallFactoryStub implements SynchronousCallFactory { @Override public VersionInfo getVersionCode(ApiClient client) throws ApiException { @@ -68,12 +82,12 @@ public VersionInfo getVersionCode(ApiClient client) throws ApiException { } @Override - public DomainList getDomainList(ApiClient client, String namespace, String _continue, String pretty, String fieldSelector, Boolean includeUninitialized, String labelSelector, Integer limit, String resourceVersion, Integer timeoutSeconds, Boolean watch) throws ApiException { + public DomainList getDomainList(ApiClient client, String namespace, String pretty, String _continue, String fieldSelector, Boolean includeUninitialized, String labelSelector, Integer limit, String resourceVersion, Integer timeoutSeconds, Boolean watch) throws ApiException { return new DomainList(); } @Override - public V1PersistentVolumeList listPersistentVolumes(String _continue, ApiClient client, String pretty, String fieldSelector, Boolean includeUninitialized, String labelSelector, Integer limit, String resourceVersion, Integer timeoutSeconds, Boolean watch) throws ApiException { + public V1PersistentVolumeList listPersistentVolumes(ApiClient client, String pretty, String _continue, String fieldSelector, Boolean includeUninitialized, String labelSelector, Integer limit, String resourceVersion, Integer timeoutSeconds, Boolean watch) throws ApiException { return new V1PersistentVolumeList(); } From 768b8a3ee899e8c362858c34a5eb4aae8239c6bc Mon Sep 17 00:00:00 2001 From: Rosemary Marano Date: Fri, 27 Apr 2018 12:20:00 -0400 Subject: [PATCH 115/186] fixed typos --- site/database.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/database.md b/site/database.md index e20d5eeedf7..b58623479fb 100644 --- a/site/database.md +++ b/site/database.md @@ -1,6 +1,6 @@ # Running the Oracle Database in Kubernetes -**PLEASE NOTE** this page is a work in progress. We will update this with either better details about how to put the datafiles onto a persistent volume, or a pointer to the official Oracle Database Kubernetes pages, or both. +**PLEASE NOTE**: This page is a work in progress. We will update this with either better details about how to put the data files onto a persistent volume, or a pointer to the official Oracle Database Kubernetes pages, or both. ``` From 72c4fae69353a17004645449a39e58920f65e468 Mon Sep 17 00:00:00 2001 From: Dongbo Xiao Date: Fri, 27 Apr 2018 09:48:34 -0700 Subject: [PATCH 116/186] Fix a typo Signed-off-by: Dongbo Xiao --- site/apache.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/apache.md b/site/apache.md index 06a707e20f0..7b7f79a04c8 100644 --- a/site/apache.md +++ b/site/apache.md @@ -394,7 +394,7 @@ domain1-apache-webtier 2h ### Use your own plugin WL module configuration -You can fine turn the behavior of the Apache plugin by providing your own Apache plugin configuration. You put your custom_mod_wl_apache.conf file in a local directory, for example `` , and specify this location in the `create-weblogic-domain-inputs.yaml` file as follows. +You can fine tune the behavior of the Apache plugin by providing your own Apache plugin configuration. You put your custom_mod_wl_apache.conf file in a local directory, for example `` , and specify this location in the `create-weblogic-domain-inputs.yaml` file as follows. ``` From 272c193e8d48b2d80bb4b4b55c0295161482ac9d Mon Sep 17 00:00:00 2001 From: Rosemary Marano Date: Fri, 27 Apr 2018 13:53:12 -0400 Subject: [PATCH 117/186] minor edits --- site/design.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/site/design.md b/site/design.md index 26f58798dfb..94fa85c4eba 100644 --- a/site/design.md +++ b/site/design.md @@ -3,9 +3,9 @@ The Oracle WebLogic Server Kubernetes Operator (the “operator”) is designed to fulfill a similar role to that which a human operator would fill in a traditional data center deployment. It contains a set of useful built-in knowledge about how to perform various lifecycle operations on a domain correctly. -Human operators are normally responsible for starting and stopping environments, initiating backups, performing scaling operations, performing manual tasks associated with disaster recovery and high availability needs and coordinating actions with other operators in other data centers. It is envisaged that the operator will have similar responsibilities in a Kubernetes environment. The initial technology preview version of the operator does not have the capability to take on all of those responsibilities, but enumerating them here gives insight into the background context for making various design choices. +Human operators are normally responsible for starting and stopping environments, initiating backups, performing scaling operations, performing manual tasks associated with disaster recovery and high availability needs and coordinating actions with other operators in other data centers. It is envisaged that the operator will have similar responsibilities in a Kubernetes environment. -It is important to note the distinction between an *operator* and an *administrator*. A WebLogic Server administrator typically has different responsibilities centered around managing the detailed configuration of the WebLogic domains. The operator has only limited interest in the domain configuration, with its main concern being the high-level topology of the domain; e.g., how many clusters and servers, and information about network access points, such as channels. +It is important to note the distinction between an *operator* and an *administrator*. A WebLogic Server administrator typically has different responsibilities centered around managing the detailed configuration of the WebLogic domains. The operator has only limited interest in the domain configuration, with its main concern being the high-level topology of the domain; for example, how many clusters and servers, and information about network access points, such as channels. Human operators may manage more than one domain, and the operator is also designed to be able to manage more than one domain. Like its human counterpart, the operator will only take actions against domains that it is told to manage, and will ignore any other domains that may be present in the same environment. @@ -21,7 +21,7 @@ The operator is designed with security in mind from the outset. Some examples o The operator is designed to avoid imposing any arbitrary restriction on how WebLogic Server may be configured or used in Kubernetes. Where there are restrictions, these are based on the availability of some specific feature in Kubernetes; for example, multicast support. -The operator learns of WebLogic domains through instances of a domain Kubernetes resource. When the operator is installed, it creates a Kubernetes [Custom Resource Definition](https://kubernetes.io/docs/concepts/api-extension/custom-resources/). This custom resource definition defines the domain resource type. Once this type is defined, you can manage domain resources using `kubectl` just like any other resource type. For instance, `kubectl get domain` or `kubectl edit domain domain1`. +The operator learns of WebLogic domains through instances of a domain Kubernetes resource. When the operator is installed, it creates a Kubernetes [Custom Resource Definition](https://kubernetes.io/docs/concepts/api-extension/custom-resources/). This custom resource definition defines the domain resource type. After this type is defined, you can manage domain resources using `kubectl` just like any other resource type. For instance, `kubectl get domain` or `kubectl edit domain domain1`. Schema for domain resources: * [Domain](../model/src/main/resources/schema/domain.json) @@ -34,4 +34,4 @@ Schema for domain resources: * [ServerHealth](../model/src/main/resources/schema/serverhealth.json) * [SubsystemHealth](../model/src/main/resources/schema/subsystemhealth.json) -The schema for the domain resource is designed to be as sparse as possible. It includes the connection details for the Administration Server, but all of the other content are operational details about which servers should be started, environment variables, and details about what should be exposed outside the Kubernetes cluster. This way, the WebLogic domain's configuration remains the normative configuration. +The schema for the domain resource is designed to be as sparse as possible. It includes the connection details for the Administration Server, but all of the other content is operational details about which servers should be started, environment variables, and details about what should be exposed outside the Kubernetes cluster. This way, the WebLogic domain's configuration remains the normative configuration. From 5e4dfb0898e2e69772ff07f4605cbbad5ee61883 Mon Sep 17 00:00:00 2001 From: Ryan Eberhard Date: Fri, 27 Apr 2018 14:35:51 -0400 Subject: [PATCH 118/186] Use repository variables for wercker --- wercker.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/wercker.yml b/wercker.yml index 1a07d22ca15..792fe813ead 100644 --- a/wercker.yml +++ b/wercker.yml @@ -45,7 +45,7 @@ build: cp -R src/scripts/* /operator/ cp operator/target/weblogic-kubernetes-operator-0.2.jar /operator/weblogic-kubernetes-operator.jar cp operator/target/lib/*.jar /operator/lib/ - export IMAGE_TAG_OPERATOR="${WERCKER_GIT_BRANCH//[_\/]/-}" + export IMAGE_TAG_OPERATOR="${IMAGE_TAG_OPERATOR:-${WERCKER_GIT_BRANCH//[_\/]/-}}" if [ "$IMAGE_TAG_OPERATOR" = "master" ]; then export IMAGE_TAG_OPERATOR="latest" fi @@ -59,10 +59,10 @@ build: # push the image to quay.io using the GIT branch as the tag # this image needs to be available to the integration-test pipeline for testing - internal/docker-push: - username: $QUAY_USERNAME - password: $QUAY_PASSWORD - email: $QUAY_EMAIL - repository: quay.io/markxnelson/weblogic-kubernetes-operator + username: $REPO_USERNAME + password: $REPO_PASSWORD + email: $REPO_EMAIL + repository: $REPO_LOCATION tag: $IMAGE_TAG_OPERATOR working-dir: "/operator" cmd: "operator.sh" From dc2bf1c2bb483d6e5baf14155c8b2e008fd0b411 Mon Sep 17 00:00:00 2001 From: Rosemary Marano Date: Fri, 27 Apr 2018 14:38:24 -0400 Subject: [PATCH 119/186] mostly minor edits --- site/developer.md | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/site/developer.md b/site/developer.md index 8f8df345804..5f903695465 100644 --- a/site/developer.md +++ b/site/developer.md @@ -1,10 +1,10 @@ # Developer guide -This page provides information for developers who wish to understand or contribute to the code. +This guide provides information for developers who wish to understand or contribute to the code. ## Requirements -The following software are required to obtain and build the operator: +The following software is required to obtain and build the operator: * Git (1.8 or later recommended) * Apache Maven (3.3 or later recommended) @@ -77,9 +77,9 @@ These tests assume that the RBAC definitions exist on the Kubernetes cluster. To create them, first, make a copy of the inputs file (`create-weblogic-operator-inputs.yaml`) and update it. -Next, choose and create a directory that generated operator related files will be stored in, e.g. /path/to/weblogic-operator-output-directory. +Next, choose and create a directory that generated operator-related files will be stored in, for example, `/path/to/weblogic-operator-output-directory`. -Finally, run the operator installation script with the "generate only" option as shown below, pointing it at your inputs file and your output directory. (see the [installation](installation.md) page for details about this script and the inputs): +Finally, run the operator installation script with the "generate only" option as shown below, pointing it at your inputs file and your output directory. (See the [installation](installation.md) page for details about this script and the inputs): ``` ./create-weblogic-operator.sh -g \ @@ -113,7 +113,7 @@ After you have run the build (that is, `mvn clean install`), create the Docker i docker build -t weblogic-kubernetes-operator:some-tag --no-cache=true . ``` -We recommend that you use a tag other than `latest` to make it easy to distinguish your image from the "real" one. In the example above, we just put in the GitHub ID of the developer. +We recommend that you use a tag other than `latest` to make it easy to distinguish your image from the "real" one. In the example above, we used the GitHub ID of the developer. Next, upload your image to your Kubernetes server as follows: @@ -125,11 +125,11 @@ scp operator.tar YOUR_USER@YOUR_SERVER:/some/path/operator.tar docker load < /some/path/operator.tar ``` -Verify that you have the right image by running `docker images | grep webloogic-kubernetes-operator` on both machines and comparing the image ID. +Verify that you have the right image by running `docker images | grep webloogic-kubernetes-operator` on both machines and comparing the image IDs. To create and deploy the operator, first, make a copy of the inputs file (`create-weblogic-operator-inputs.yaml`) and update it, making sure that `weblogicOperatorImagePullPolicy` is set to `Never` and `weblogicOperatorImage` matches the name you used in your `docker build` command. -Next, choose and create a directory that generated operator related files will be stored in, e.g. /path/to/weblogic-operator-output-directory. +Next, choose and create a directory that generated operator-related files will be stored in, for example, `/path/to/weblogic-operator-output-directory`. Finally, run the operator installation script to deploy the operator, pointing it at your inputs file and your output directory: @@ -145,7 +145,7 @@ Finally, run the operator installation script to deploy the operator, pointing i This project has adopted the following coding standards: * All indents are two spaces. -* Javadoc must be provided for all public packages, classes and methods and must include all parameters and returns. Javadoc is not required for methods that override or implement methods that are already documented. +* Javadoc must be provided for all public packages, classes, and methods, and must include all parameters and returns. Javadoc is not required for methods that override or implement methods that are already documented. * All non-trivial methods should include `LOGGER.entering()` and `LOGGER.exiting()` calls. * The `LOGGER.exiting()` call should include the value that is going to be returned from the method, unless that value includes a credential or other sensitive information. * All logged messages must be internationalized using the resource bundle `src/main/resources/Operator.properties` and using a key itemized in `src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java`. @@ -156,28 +156,28 @@ This project has adopted the following coding standards: This project has the following directory structure: -* docs: Generated javadoc and swagger -* kubernetes: BASH scripts and YAML templates for operator installation and WebLogic domain creation job. -* site: This documentation -* src/main/java: Java source code for the operator -* src/test/java: Java unit-tests for the operator -* src-generated-swagger: Snapshot of Java source files generated from the domain custom resource's swagger -* swagger: Swagger files for Kubernetes API server and domain custom resource +* `docs`: Generated javadoc and Swagger +* `kubernetes`: BASH scripts and YAML templates for operator installation and WebLogic domain creation job. +* `site`: This documentation +* `src/main/java`: Java source code for the operator +* `src/test/java`: Java unit-tests for the operator +* `src-generated-swagger`: Snapshot of Java source files generated from the domain custom resource's Swagger +* `swagger`: Swagger files for the Kubernetes API server and domain custom resource ### Watch package -The Watch API in the Kubernetes Java client provides a watch capability across a specific list of resources for a limited amount of time. As such it is not ideally suited for our use case, where a continuous stream of watches was desired, with watch events generated in real time. The watch-wrapper in this repository extends the default Watch API to provide a continuous stream of watch events until the stream is specifically closed. It also provides `resourceVersion` tracking to exclude events that have already been seen. The watch-wrapper provides callbacks so events, as they occur, can trigger actions. +The Watch API in the Kubernetes Java client provides a watch capability across a specific list of resources for a limited amount of time. As such, it is not ideally suited for our use case, where a continuous stream of watches is desired, with watch events generated in real time. The watch-wrapper in this repository extends the default Watch API to provide a continuous stream of watch events until the stream is specifically closed. It also provides `resourceVersion` tracking to exclude events that have already been seen. The watch-wrapper provides callbacks so events, as they occur, can trigger actions. ## Asynchronous call model Our expectation is that certain customers will task the operator with managing thousands of WebLogic domains across dozens of Kubernetes namespaces. Therefore, we have designed the operator with an efficient user-level threads pattern based on a simplified version of the code from the JAX-WS reference implementation. We have then used that pattern to implement an asynchronous call model for Kubernetes API requests. This call model has built-in support for timeouts, retries with exponential back-off, and lists that exceed the requested maximum size using the continuance functionality. -### User-level Thread Pattern +### User-Level Thread Pattern The user-level thread pattern is implemented by the classes in the `oracle.kubernetes.operator.work` package. * `Engine`: The executor service and factory for `Fibers`. -* `Fiber`: The user-level thread. `Fibers` represent the execution of a single processing flow through a series of `Steps`. `Fibers` may be suspended and later resumed and do not consume a `Thread` while suspended. +* `Fiber`: The user-level thread. `Fibers` represent the execution of a single processing flow through a series of `Steps`. `Fibers` may be suspended and later resumed, and do not consume a `Thread` while suspended. * `Step`: Individual CPU-bound activity in a processing flow. * `Packet`: Context of the processing flow. * `NextAction`: Used by a `Step` when it returns control to the `Fiber` to indicate what should happen next. Common 'next actions' are to execute another `Step` or to suspend the `Fiber`. @@ -298,6 +298,6 @@ In this sample, the developer is using the pattern to list pods from the default Notice that the required parameters, such as `namespace`, are method arguments, but optional parameters are designated using a simplified builder pattern using `with()` and a lambda. -The default behavior of `onFailure()` will retry with exponential backoff the request on status codes `429 (TooManyRequests)`, `500 (InternalServerError)`, `503 (ServiceUnavailable)`, `504 (ServerTimeout)` or a simple timeout with no response from the server. +The default behavior of `onFailure()` will retry with an exponential backoff the request on status codes `429 (TooManyRequests)`, `500 (InternalServerError)`, `503 (ServiceUnavailable)`, `504 (ServerTimeout)` or a simple timeout with no response from the server. If the server responds with status code `409 (Conflict)`, then this indicates an optimistic locking failure. Common use cases are that the code read a Kubernetes object in one asynchronous step, modified the object, and attempted to replace the object in another asynchronous step; however, another activity replaced that same object in the interim. In this case, retrying the request would give the same result. Therefore, developers may provide an "on conflict" step when calling `super.onFailure()`. The conflict step will be invoked after an exponential backoff delay. In this example, that conflict step should be the step that reads the existing Kubernetes object. From 92f720536c68cecebb2cb74eaf2672aa77565440 Mon Sep 17 00:00:00 2001 From: Rosemary Marano Date: Fri, 27 Apr 2018 16:05:17 -0400 Subject: [PATCH 120/186] incorporated Removing a Domain info from Tom B --- README.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fa26a7dab72..782e9dd2c8c 100644 --- a/README.md +++ b/README.md @@ -165,7 +165,15 @@ Please refer to [Shutting down a domain](site/shutdown-domain.md) for informatio ## Removing a domain -To permanently remove a domain from a Kubernetes cluster, first shut down the domain using the instructions provided in [Shutting down a domain](site/shutdown-domain.md), then remove the persistent volume claim and the persistent volume using these commands: +To permanently remove the Kubernetes resources for a domain from a Kubernetes cluster, run the [Delete WebLogic domain resources](kubernetes/delete-weblogic-domain-resources.sh) script. This script will delete a specific domain, or all domains, and all the Kubernetes resources associated with a set of given domains. The script will also attempt a clean shutdown of a domain’s WebLogic pods before deleting its resources.  You can run the script in a test mode to show what would be shutdown and deleted without actually performing the shutdowns and deletions.   For script help, use its `-h` option. + +The script will remove only domain-related resources which are labeled with the `domainUID` label, such as resources created by the [Create WebLogic domain](kubernetes/create-weblogic-domain.sh) script or the [integration tests](src/integration-tests/bash/run.sh).  If you manually created resources and have not labelled them with a `domainUID`, the script will not remove them.   One way to label a resource that has already been deployed is: + +``` +kubectl -n  label domainUID= +``` + +To manually remove the persistent volume claim and the persistent volume, use these commands: ``` kubectl delete pvc PVC-NAME -n NAMESPACE @@ -174,7 +182,7 @@ kubectl delete pv PV-NAME Find the names of the persistent volume claim (represented above as `PVC-NAME`) and the persistent volume (represented as `PV-NAME`) in the domain custom resource YAML file, or if it is not available, check for the `domainUID` in the metadata on the persistent volumes. Replace `NAMESPACE` with the namespace that the operator is running in. -To permanently delete the actual domain configuration, delete the physical volume using the appropriate tools. For example, if the persistent volume used the `HostPath provider`, then delete the corresponding directory on the Kubernetes master. +To permanently delete the actual WebLogic domain configuration and domain home, delete the physical volume using the appropriate tools. For example, if the persistent volume used the `HostPath` provider, then delete the corresponding directory on the Kubernetes master. ## Removing the operator From dafc9bd0de838d86f2de81f461520da01a385b7d Mon Sep 17 00:00:00 2001 From: Rosemary Marano Date: Sat, 28 Apr 2018 14:30:00 -0400 Subject: [PATCH 121/186] re-worked this doc --- site/elk.md | 76 ++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 52 insertions(+), 24 deletions(-) diff --git a/site/elk.md b/site/elk.md index b472e043715..04e6f32f7bd 100644 --- a/site/elk.md +++ b/site/elk.md @@ -1,27 +1,55 @@ -# ELK integration - -**PLEASE NOTE** This page is a work in progress, we have some rough notes in here, we are working on writing better doc for how to set up this integration. - -TODO Update this whole section -Turn on Logstash in operator -Add the ENABLE_LOGSTASH env in weblogic-operator.yaml to the env section -name: ENABLE_LOGSTASH value: "true" -Create the PV and PVC for the operator logs -kubectl create -f elk-pv.yaml -Verify if PV and PVC are created kubectl get pv -n weblogic-operator kubectl get pvc -n weblogic-operator -Deploying the ELK stack -ELK stack consists of Elasticsearch, Logstash, and Kibana. Logstash is configured to pickup logs from the operator and push to Elasticsearch. Kibana is setup to connect to Elasticsearch to read the log and shows it on the dashboard. -Deploying Elasticsearch -kubectl create -f elasticsearch.yaml -Deploying Kibana -kubectl create -f kibana.yaml -Deploying Logstash -kubectl create -f logstash.yaml -Verify if ELK stack pods are created and running kubectl get pods -n weblogic-operator kubectl get pods -Accessing the Kibana dashboard -Get the NodePort from kibana services kubectl describe service kibana -Access Kibana dashboard using NodePort from output of the above kubctl command and the hostname http://hostname:NodePort (eg. http://slcac571:30211) -Select the Management tab to configure an index pattern. You will see logstash-* prepopulated, select the time filter then press the create button Then select the Discover tab to see the logs +# Elastic Stack integration + +**PLEASE NOTE**: This page is a work in progress. We have some rough notes in here and are working on writing better documentation for how to set up this integration. + + +## Turn on Logstash in the operator + +In `weblogic-operator.yaml`, add `ENABLE_LOGSTASH` in the `env` section: + +`name: ENABLE_LOGSTASH value: "true"` + +## Create the PV and PVC for the operator logs + +`kubectl create -f elk-pv.yaml` + +Verify that the PV and PVC are created: + +`kubectl get pv -n weblogic-operator kubectl get pvc -n weblogic-operator` + +## Deploy the Elastic Stack + +The Elastic Stack consists of Elasticsearch, Logstash, and Kibana. Logstash is configured to pickup logs from the operator and push them to Elasticsearch. Kibana connects to Elasticsearch to read the logs and displays them on the dashboard. + +### Deploy Elasticsearch + +`kubectl create -f elasticsearch.yaml` + +### Deploy Kibana + +`kubectl create -f kibana.yaml` + +### Deploy Logstash + +`kubectl create -f logstash.yaml` + +Verify that the Elastic Stack pods are created and running: + +`kubectl get pods -n weblogic-operator kubectl get pods` + +## Access the Kibana dashboard + +Get the `NodePort` from the Kibana services: + +`kubectl describe service kibana` + +Access the Kibana dashboard using the `NodePort` from the kubctl command output and the hostname, `http://hostname:NodePort` (for example, `http://slcac571:30211`). + +Select **Management** to configure an index pattern. You will see `logstash-*` prepopulated. + +Select the time filter, then click **Create**. + +To see the logs, select **Discover**. ![Kibana dashboard](images/kibana.png) From 2e92d144c325f0f5586b861984688aa1e5d8b4c5 Mon Sep 17 00:00:00 2001 From: Ryan Eberhard Date: Sat, 28 Apr 2018 16:12:33 -0400 Subject: [PATCH 122/186] Use OCIR registry --- wercker.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wercker.yml b/wercker.yml index 792fe813ead..4b77ec429db 100644 --- a/wercker.yml +++ b/wercker.yml @@ -62,7 +62,8 @@ build: username: $REPO_USERNAME password: $REPO_PASSWORD email: $REPO_EMAIL - repository: $REPO_LOCATION + repository: $REPO_REPOSITORY + registry: $REPO_REGISTRY tag: $IMAGE_TAG_OPERATOR working-dir: "/operator" cmd: "operator.sh" From 131e781a6506a923a524ee5d40744d99b5cb5279 Mon Sep 17 00:00:00 2001 From: Ryan Eberhard Date: Sat, 28 Apr 2018 16:20:45 -0400 Subject: [PATCH 123/186] Remove e-mail --- wercker.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/wercker.yml b/wercker.yml index 4b77ec429db..7235754ded7 100644 --- a/wercker.yml +++ b/wercker.yml @@ -61,7 +61,6 @@ build: - internal/docker-push: username: $REPO_USERNAME password: $REPO_PASSWORD - email: $REPO_EMAIL repository: $REPO_REPOSITORY registry: $REPO_REGISTRY tag: $IMAGE_TAG_OPERATOR From cd37103285465916a12cea66ce04a356db6c4459 Mon Sep 17 00:00:00 2001 From: Ryan Eberhard Date: Sun, 29 Apr 2018 11:42:20 -0400 Subject: [PATCH 124/186] Move to OCIR from quay.io --- wercker.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/wercker.yml b/wercker.yml index 7235754ded7..5f61a0fff49 100644 --- a/wercker.yml +++ b/wercker.yml @@ -56,7 +56,7 @@ build: rpm -e --nodeps gzip yum clean all rm -rf /var/cache/yum - # push the image to quay.io using the GIT branch as the tag + # push the image to Docker using the GIT branch as the tag # this image needs to be available to the integration-test pipeline for testing - internal/docker-push: username: $REPO_USERNAME @@ -215,19 +215,19 @@ integration-test: kubectl delete secret docker-store -n test2 --ignore-not-found=true kubectl create secret docker-registry docker-store -n test2 --docker-server=index.docker.io/v1/ --docker-username=$DOCKER_USERNAME --docker-password=$DOCKER_PASSWORD --docker-email=$DOCKER_EMAIL - kubectl delete secret quay-io -n weblogic-operator-1 --ignore-not-found=true - kubectl create secret docker-registry quay-io -n weblogic-operator-1 --docker-server=quay.io --docker-username=$QUAY_USERNAME --docker-password=$QUAY_PASSWORD --docker-email=$QUAY_EMAIL + kubectl delete secret test-registry -n weblogic-operator-1 --ignore-not-found=true + kubectl create secret docker-registry test-registry -n weblogic-operator-1 --docker-server=$REPO_REGISTRY --docker-username=$REPO_USERNAME --docker-password=$REPO_PASSWORD - kubectl delete secret quay-io -n weblogic-operator-2 --ignore-not-found=true - kubectl create secret docker-registry quay-io -n weblogic-operator-2 --docker-server=quay.io --docker-username=$QUAY_USERNAME --docker-password=$QUAY_PASSWORD --docker-email=$QUAY_EMAIL + kubectl delete secret test-registry -n weblogic-operator-2 --ignore-not-found=true + kubectl create secret docker-registry test-registry -n weblogic-operator-2 --docker-server=$REPO_REGISTRY --docker-username=$REPO_USERNAME --docker-password=$REPO_PASSWORD - export IMAGE_NAME_OPERATOR="quay.io/markxnelson/weblogic-kubernetes-operator" + export IMAGE_NAME_OPERATOR="${REPO_REPOSITORY}" export IMAGE_TAG_OPERATOR="${WERCKER_GIT_BRANCH//[_\/]/-}" if [ "$IMAGE_TAG_OPERATOR" = "master" ]; then export IMAGE_TAG_OPERATOR="latest" fi export IMAGE_PULL_POLICY_OPERATOR="Always" - export IMAGE_PULL_SECRET_OPERATOR="quay-io" + export IMAGE_PULL_SECRET_OPERATOR="test-registry" export IMAGE_PULL_SECRET_WEBLOGIC="docker-store" echo "Integration test suite against the test image which is:" From cfae327c1cf8cd27ddaf0f89456948d9d0cc2e31 Mon Sep 17 00:00:00 2001 From: Ryan Eberhard Date: Sun, 29 Apr 2018 13:19:03 -0400 Subject: [PATCH 125/186] Move generation of secrets --- src/integration-tests/bash/run.sh | 31 ++++++++++++++++++ wercker.yml | 53 +------------------------------ 2 files changed, 32 insertions(+), 52 deletions(-) diff --git a/src/integration-tests/bash/run.sh b/src/integration-tests/bash/run.sh index 3cb02172ed6..2c6d20cc295 100755 --- a/src/integration-tests/bash/run.sh +++ b/src/integration-tests/bash/run.sh @@ -566,6 +566,35 @@ function create_image_pull_secret_jenkins { } +function create_image_pull_secret_wercker { + + trace "Creating Docker Secret" + kubectl create secret docker-registry $IMAGE_PULL_SECRET_WEBLOGIC \ + --docker-server=index.docker.io/v1/ \ + --docker-username=$DOCKER_USERNAME \ + --docker-password=$DOCKER_PASSWORD \ + --docker-email=$DOCKER_EMAIL 2>&1 | sed 's/^/+' 2>&1 + + trace "Checking Secret" + local SECRET="`kubectl get secret $IMAGE_PULL_SECRET_WEBLOGIC | grep $IMAGE_PULL_SECRET_WEBLOGIC | wc | awk ' { print $1; }'`" + if [ "$SECRET" != "1" ]; then + fail 'secret $IMAGE_PULL_SECRET_WEBLOGIC was not created successfully' + fi + + trace "Creating Registry Secret" + kubectl create secret docker-registry $IMAGE_PULL_SECRET_OPERATOR \ + --docker-server=$REPO_REGISTRY \ + --docker-username=$REPO_USERNAME \ + --docker-password=$REPO_PASSWORD 2>&1 | sed 's/^/+' 2>&1 + + trace "Checking Secret" + local SECRET="`kubectl get secret $IMAGE_PULL_SECRET_OPERATOR | grep $IMAGE_PULL_SECRET_OPERATOR | wc | awk ' { print $1; }'`" + if [ "$SECRET" != "1" ]; then + fail 'secret $IMAGE_PULL_SECRET_OPERATOR was not created successfully' + fi + +} + # op_define OP_KEY NAMESPACE TARGET_NAMESPACES EXTERNAL_REST_HTTPSPORT # sets up table of operator values. # @@ -2577,6 +2606,8 @@ function test_suite_init { mkdir -p $RESULT_ROOT/acceptance_test_tmp || fail "Could not mkdir -p RESULT_ROOT/acceptance_test_tmp (RESULT_ROOT=$RESULT_ROOT)" + create_image_pull_secret_wercker + elif [ "$JENKINS" = "true" ]; then trace "Test Suite is running on Jenkins and k8s is running locally on the same node." diff --git a/wercker.yml b/wercker.yml index 5f61a0fff49..1a2722b5add 100644 --- a/wercker.yml +++ b/wercker.yml @@ -163,11 +163,6 @@ integration-test: $WERCKER_SOURCE_DIR/src/integration-tests/bash/lease.sh -o "$LEASE_ID" -t $((100 * 60)) echo @@ - # create pull secrets - echo @@ "Creating pull secrets" - kubectl delete secret docker-store --ignore-not-found=true - kubectl create secret docker-registry docker-store --docker-server=index.docker.io/v1/ --docker-username=$DOCKER_USERNAME --docker-password=$DOCKER_PASSWORD --docker-email=$DOCKER_EMAIL - export HOST_PATH="/scratch" export PV_ROOT=$HOST_PATH export RESULT_ROOT="$WERCKER_OUTPUT_DIR/k8s_dir" @@ -175,59 +170,13 @@ integration-test: export PROJECT_ROOT="${WERCKER_SOURCE_DIR}" $WERCKER_SOURCE_DIR/src/integration-tests/bash/cleanup.sh - # create pull secrets - echo @@ "Creating pull secrets" - kubectl delete secret docker-store --ignore-not-found=true - kubectl create secret docker-registry docker-store --docker-server=index.docker.io/v1/ --docker-username=$DOCKER_USERNAME --docker-password=$DOCKER_PASSWORD --docker-email=$DOCKER_EMAIL - - if [ `kubectl get ns test1 | grep Error | wc -l` = 0 ]; then - kubectl delete ns test1 --ignore-not-found=true - while [`kubectl get ns test1 -o jsonpath='{.status.phase}' | grep Terminating | wc -l` = 1 ]; do - sleep 5 - done - fi - kubectl create ns test1 - if [ `kubectl get ns test2 | grep Error | wc -l` = 0 ]; then - kubectl delete ns test2 --ignore-not-found=true - while [`kubectl get ns test2 -o jsonpath='{.status.phase}' | grep Terminating | wc -l` = 1 ]; do - sleep 5 - done - fi - kubectl create ns test2 - if [ `kubectl get ns weblogic-operator-1 | grep Error | wc -l` = 0 ]; then - kubectl delete ns weblogic-operator-1 --ignore-not-found=true - while [`kubectl get ns weblogic-operator-1 -o jsonpath='{.status.phase}' | grep Terminating | wc -l` = 1 ]; do - sleep 5 - done - fi - kubectl create ns weblogic-operator-1 - if [ `kubectl get ns weblogic-operator-2 | grep Error | wc -l` = 0 ]; then - kubectl delete ns weblogic-operator-2 --ignore-not-found=true - while [`kubectl get ns weblogic-operator-2 -o jsonpath='{.status.phase}' | grep Terminating | wc -l` = 1 ]; do - sleep 5 - done - fi - kubectl create ns weblogic-operator-2 - - kubectl delete secret docker-store -n test1 --ignore-not-found=true - kubectl create secret docker-registry docker-store -n test1 --docker-server=index.docker.io/v1/ --docker-username=$DOCKER_USERNAME --docker-password=$DOCKER_PASSWORD --docker-email=$DOCKER_EMAIL - - kubectl delete secret docker-store -n test2 --ignore-not-found=true - kubectl create secret docker-registry docker-store -n test2 --docker-server=index.docker.io/v1/ --docker-username=$DOCKER_USERNAME --docker-password=$DOCKER_PASSWORD --docker-email=$DOCKER_EMAIL - - kubectl delete secret test-registry -n weblogic-operator-1 --ignore-not-found=true - kubectl create secret docker-registry test-registry -n weblogic-operator-1 --docker-server=$REPO_REGISTRY --docker-username=$REPO_USERNAME --docker-password=$REPO_PASSWORD - - kubectl delete secret test-registry -n weblogic-operator-2 --ignore-not-found=true - kubectl create secret docker-registry test-registry -n weblogic-operator-2 --docker-server=$REPO_REGISTRY --docker-username=$REPO_USERNAME --docker-password=$REPO_PASSWORD - export IMAGE_NAME_OPERATOR="${REPO_REPOSITORY}" export IMAGE_TAG_OPERATOR="${WERCKER_GIT_BRANCH//[_\/]/-}" if [ "$IMAGE_TAG_OPERATOR" = "master" ]; then export IMAGE_TAG_OPERATOR="latest" fi export IMAGE_PULL_POLICY_OPERATOR="Always" - export IMAGE_PULL_SECRET_OPERATOR="test-registry" + export IMAGE_PULL_SECRET_OPERATOR="ocir-registry" export IMAGE_PULL_SECRET_WEBLOGIC="docker-store" echo "Integration test suite against the test image which is:" From 0164e0cc5bc12edd68fc4469bf973fc113076e78 Mon Sep 17 00:00:00 2001 From: Rosemary Marano Date: Mon, 30 Apr 2018 07:20:04 -0400 Subject: [PATCH 126/186] incorporate Monica's edits --- site/developer.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/site/developer.md b/site/developer.md index 5f903695465..9251bfb7c98 100644 --- a/site/developer.md +++ b/site/developer.md @@ -22,7 +22,7 @@ The operator source code is published on GitHub at https://github.com/oracle/web To clone the repository from GitHub, issue this command: ``` -git clone https://github.com/oracle/weblogic-kubernetes-operator +git clone https://github.com/oracle/weblogic-kubernetes-operator.git ``` ## Building the operator @@ -49,7 +49,7 @@ The Javadoc is also available in the GitHub repository [here](https://oracle.git ## Running integration tests -The project includes integration tests that can be run against a Kubernetes cluster. If you want to use these tests, you will need to provide your own Kubernetes cluster. You will need to obtain the `kube.config` file for an administrator user and make it available on the machine running the build. Tests will run against Kubernetes 1.7.x and 1.8.x currently. There are some issues with 1.9, which are being worked on. +The project includes integration tests that can be run against a Kubernetes cluster. If you want to use these tests, you will need to provide your own Kubernetes cluster. You will need to obtain the `kube.config` file for an administrator user and make it available on the machine running the build. Tests will run against Kubernetes 1.7.5+, 1.8.0+, 1.9.0+, and 1.10.0. To run the tests, uncomment the following `execution` element in the `pom.xml` file and update the `KUBECONFIG` to point to your kube config file. From b7cee733a70e1f1205b5c2ec7713c8a61304cb23 Mon Sep 17 00:00:00 2001 From: Ryan Eberhard Date: Mon, 30 Apr 2018 07:36:40 -0400 Subject: [PATCH 127/186] Updates for branching and using Wercker --- site/developer.md | 18 ++++++++++++++++++ site/installation.md | 41 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/site/developer.md b/site/developer.md index 5f903695465..0e94babc95d 100644 --- a/site/developer.md +++ b/site/developer.md @@ -25,6 +25,24 @@ To clone the repository from GitHub, issue this command: git clone https://github.com/oracle/weblogic-kubernetes-operator ``` +## Operator branching model + +The ```master``` branch is protected and will always contain source for the latest, generally available (GA) +release of the operator, including any critical hot fixes. No general pull requests will be merged to this branch. + +Active work will be performed on the ```develop``` branch. This branch is also protected. Please submit pull +requests to this branch unless you are collaborating on a feature and have another target branch. +Please see details on the Oracle Contributor Agreement (OCA) and guidelines for pull requests on the [README] (README.md). + +Longer running feature work will be performed on specific branches, such as ```feature/dynamic-clusters```. Since we want +to balance separating destabilizing work into feature branches against the possibility of later difficult merges, we +encourage developers working on features to pull out any necessary refactoring or improvements that are general purpose into +their own shorter-lived branches and create pull requests to ```develop``` when these smaller work items are complete. + +When it is time for a release, we will branch off ```develop``` to create a per-release branch. Here, we will update version +numbers, rebuild javadoc, if necessary, and perform any other pre-release updates. Finally, this release branch will be merged +to ```master```. + ## Building the operator The operator is built using [Apache Maven](http://maven.apache.org). The build machine will also need to have Docker installed. diff --git a/site/installation.md b/site/installation.md index ba6ba87c314..189b6963888 100644 --- a/site/installation.md +++ b/site/installation.md @@ -11,7 +11,46 @@ Note that there is a short video demonstration of the installation process avail [comment]: # ( Note that you *must* create the `docker-registry` secret in the `weblogic-operator` namespace, so you will need to create the namespace first. ) [comment]: # ( In this command, replace the uppercase items with the appropriate values. The `SECRET_NAME` will be needed in later parameter files. The `NAMESPACE` must match the namespace where the operator will be deployed. ) -## Build the Docker image for the operator +## Build the Docker image for the operator using Wercker + +You can build, test, and publish the Docker image for the operator directly from Wercker using the ```wercker.yml``` from this repository. + +If you haven't done so already, navigate to [wercker.com](https://www.wercker.com) and create an account. Once you are logged in, +on the [app.wercker.com] (https://app.wercker.com) page press, "Create your first application." + +Select GitHub (the default, if you are new to Wercker). If you haven't done so already, press the "Connect" button within the +larger GitHub button and follow the prompts to provide a login for GitHub. This connects your Wercker and GitHub accounts so +that Wercker pipelines will later be able to clone this repository. Press, "Next." + +Select the repository from GitHub. This will be "oracle / weblogic-kubernetes-operator" or a different value if you +forked this repository. Press, "Next." + +Configure Wercker's access to the GitHub repository. The default choice, "wercker will check out the code without using an SSH key", +is typically sufficient. Press, "Next." + +Verify the settings so far on the review page and press, "Create." + +Since this GitHub repository already has a ```wercker.yml``` file, you can skip directly to the "Environment" tab. + +Please provide the following key/value pairs on the environment page. Remember that these values will be +visible to anyone to whom you give access to the Wercker application, therefore, select "Protected" for any +values that should remain hidden, including all passwords. + +| Key | Value | OCIR Sample | +| --- | --- | --- | +| DOCKER_USERNAME | Username for the Docker store for pulling serverjre image | | +| DOCKER_PASSWORD | Password for the Docker store | | +| REPO_REGISTRY| Registry address | https://phx.ocir.io/v2 | +| REPO_REPOSITORY | Repository value | phx.ocir.io//weblogic-kubernetes-operator | +| REPO_USERNAME | Username for registry | / | +| REPO_PASSWORD | Password for registry | Use generated SWIFT password | +| IMAGE_TAG_OPERATOR | Image tag, such as 0.2 or latest | | + +Select the "Runs" tab. Skip to the bottom and press, "Trigger your first build now." + +When the run completes successfully, the Docker image for the operator will be built and published to your repository. + +## Build the Docker image for the operator locally The following software are required to obtain and build the operator: From c77c00d95a9c702320b4b3fb8a3f7dd072f93fba Mon Sep 17 00:00:00 2001 From: Rosemary Marano Date: Mon, 30 Apr 2018 09:42:03 -0400 Subject: [PATCH 128/186] first pass edits on install doc --- site/installation.md | 72 ++++++++++++++++++++++---------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/site/installation.md b/site/installation.md index ba6ba87c314..a378f149b67 100644 --- a/site/installation.md +++ b/site/installation.md @@ -13,7 +13,7 @@ Note that there is a short video demonstration of the installation process avail ## Build the Docker image for the operator -The following software are required to obtain and build the operator: +The following software is required to obtain and build the operator: * Git (1.8 or later recommended) * Apache Maven (3.3 or later recommended) @@ -32,28 +32,28 @@ Then run the build using this command: mvn clean install ``` -Then login to Docker Store so that you will be able to pull the base image, and create the Docker image as follows. These commands should be executed in the project root directory.: +Log in to the Docker Store so that you will be able to pull the base image and create the Docker image as follows. These commands should be executed in the project root directory: ``` docker login docker build -t weblogic-kubernetes-operator:developer --no-cache=true . ``` -Note: If you have not used the base image (`store/oracle/serverjre:8`) before you will need to visit the [Docker Store web interface](https://store.docker.com/images/oracle-serverjre-8) and accept the license agreement before Docker Store will give you permission to pull that image. +**Note**: If you have not used the base image (`store/oracle/serverjre:8`) before, you will need to visit the [Docker Store web interface](https://store.docker.com/images/oracle-serverjre-8) and accept the license agreement before the Docker Store will give you permission to pull that image. -We recommend that you use a tag other than `latest` to make it easy to distinguish your image. In the example above, the tag could be the GitHub ID of the developer. +We recommend that you use a tag other than `latest`, to make it easy to distinguish your image. In the example above, the tag could be the GitHub ID of the developer. Next, upload your image to your Kubernetes server as follows: ``` -# on your build machine +# on your build machine: docker save weblogic-kubernetes-operator:developer > operator.tar scp operator.tar YOUR_USER@YOUR_SERVER:/some/path/operator.tar # on each Kubernetes worker: docker load < /some/path/operator.tar ``` -Verify that you have the right image by running `docker images | grep webloogic-kubernetes-operator` on both machines and comparing the image ID. +Verify that you have the right image by running `docker images | grep webloogic-kubernetes-operator` on both machines and comparing the image IDs. [comment]: # ( Pull the operator image ) [comment]: # ( You can let Kubernetes pull the Docker image for you the first time you try to create a pod that uses the image, but we have found that you can generally avoid various common issues like putting the secret in the wrong namespace or getting the credentials wrong by just manually pulling the image by running these commands *on the Kubernetes master*: ) @@ -64,26 +64,26 @@ Verify that you have the right image by running `docker images | grep webloogic- The operator is deployed with the provided installation script (`create-weblogic-operator.sh`). The default input to this script is the file `create-weblogic-operator-inputs.yaml`. It contains the following parameters: -### CONFIGURATION PARAMETERS FOR THE OPERATOR +### Configuration parameters for the operator | Parameter | Definition | Default | | --- | --- | --- | -| elkIntegrationEnabled | Determines whether the ELK integration will be enabled. If set to `true`, then ElasticSearch, logstash and Kibana will be installed, and logstash will be configured to export the operator’s logs to ElasticSearch. | false | -| externalDebugHttpPort | The port number of the operator's debugging port outside of the Kubernetes cluster. | 30999 | -| externalOperatorCert | A base64 encoded string containing the X.509 certificate that the operator will present to clients accessing its REST endpoints. This value is only used when `externalRestOption` is set to `CUSTOM_CERT`. | | -| externalOperatorKey | A base64 encoded string containing the private key of the operator's X.509 certificate. This value is only used when externalRestOption is set to `CUSTOM_CERT`. | | -| externalRestHttpsPort| The NodePort number that should be allocated for the operator REST server should listen for HTTPS requests on. | 31001 | -| externalRestOption | Which of the available REST options is desired. Allowed values:
- `NONE` Disable the REST interface.
- `SELF_SIGNED_CERT` The operator will use a self-signed certificate for its REST server. If this value is specified, then the `externalSans` parameter must also be set.
- `CUSTOM_CERT` Provide custom certificates, for example from an external certification authority. If this value is specified, then the `externalOperatorCert` and `externalOperatorKey` must also be provided.| NONE | -| externalSans| A comma-separated list of Subject Alternative Names that should be included in the X.509 Certificate. This list should include ...
Example: `DNS:myhost,DNS:localhost,IP:127.0.0.1` | | -| internalDebugHttpPort | The port number of the operator's debugging port inside the Kubernetes cluster. | 30999 | -| javaLoggingLevel | The level of Java logging that should be enabled in the operator. Allowed values are `SEVERE`, `WARNING`, `INFO`, `CONFIG`, `FINE`, `FINER`, and `FINEST` | INFO | -| namespace | The Kubernetes namespace that the operator will be deployed in. It is recommended that a namespace be created for the operator rather than using the `default` namespace.| weblogic-operator | -| remoteDebugNodePortEnabled | Controls whether or not the operator will start a Java remote debug server on the provided port and suspend execution until a remote debugger has attached. | false | -| serviceAccount| The name of the service account that the operator will use to make requests to the Kubernetes API server. | weblogic-operator | -| targetNamespaces | A list of the Kubernetes namespaces that may contain WebLogic domains that the operator will manage. The operator will not take any action against a domain that is in a namespace not listed here. | default | -| weblogicOperatorImage | The Docker image containing the operator code. | container-registry.oracle.com/middleware/weblogic-kubernetes-operator:latest | -| weblogicOperatorImagePullPolicy | The image pull policy for the operator docker image. Allowed values are 'Always', 'Never' and 'IfNotPresent' | IfNotPresent | -| weblogicOperatorImagePullSecretName | Name of the Kubernetes secret to access the Docker Store to pull the WebLogic Server Docker image. The presence of the secret will be validated when this parameter is enabled. | | +| `elkIntegrationEnabled` | Determines whether the Elastic Stack integration will be enabled. If set to `true`, then Elasticsearch, Logstash, and Kibana will be installed, and Logstash will be configured to export the operator’s logs to Elasticsearch. | `false` | +| `externalDebugHttpPort` | The port number of the operator's debugging port outside of the Kubernetes cluster. | `3099`9 | +| `externalOperatorCert` | A base64 encoded string containing the X.509 certificate that the operator will present to clients accessing its REST endpoints. This value is only used when `externalRestOption` is set to `CUSTOM_CERT`. | | +| `externalOperatorKey` | A base64 encoded string containing the private key of the operator's X.509 certificate. This value is used only when `externalRestOption` is set to `CUSTOM_CERT`. | | +| `externalRestHttpsPort`| The `NodePort` number that should be allocated on which the operator REST server should listen for HTTPS requests. | `31001` | +| `externalRestOption` | The available REST options. Allowed values are:
- `NONE` Disable the REST interface.
- `SELF_SIGNED_CERT` The operator will use a self-signed certificate for its REST server. If this value is specified, then the `externalSans` parameter must also be set.
- `CUSTOM_CERT` Provide custom certificates, for example, from an external certification authority. If this value is specified, then `externalOperatorCert` and `externalOperatorKey` must also be provided.| `NONE` | +| `externalSans`| A comma-separated list of Subject Alternative Names that should be included in the X.509 Certificate. This list should include ...
Example: `DNS:myhost,DNS:localhost,IP:127.0.0.1` | | +| `internalDebugHttpPort` | The port number of the operator's debugging port inside the Kubernetes cluster. | `30999` | +| `javaLoggingLevel` | The level of Java logging that should be enabled in the operator. Allowed values are: `SEVERE`, `WARNING`, `INFO`, `CONFIG`, `FINE`, `FINER`, and `FINEST` | `INFO` | +| `namespace` | The Kubernetes namespace that the operator will be deployed in. It is recommended that a namespace be created for the operator rather than using the `default` namespace.| `weblogic-operator` | +| `remoteDebugNodePortEnabled` | Controls whether or not the operator will start a Java remote debug server on the provided port and suspend execution until a remote debugger has attached. | `false` | +| `serviceAccount`| The name of the service account that the operator will use to make requests to the Kubernetes API server. | `weblogic-operator` | +| `targetNamespaces` | A list of the Kubernetes namespaces that may contain WebLogic domains that the operator will manage. The operator will not take any action against a domain that is in a namespace not listed here. | `default` | +| `weblogicOperatorImage` | The Docker image containing the operator code. | `container-registry.oracle.com/middleware/weblogic-kubernetes-operator:latest` | +| `weblogicOperatorImagePullPolicy` | The image pull policy for the operator Docker image. Allowed values are: `Always`, `Never` and `IfNotPresent` | `IfNotPresent` | +| `weblogicOperatorImagePullSecretName` | Name of the Kubernetes secret to access the Docker Store to pull the WebLogic Server Docker image. The presence of the secret will be validated when this parameter is enabled. | | Review the default values to see if any need to be updated to reflect the target environment. If so, then make a copy of the input file and modify it. Otherwise, you can use the default input file. @@ -91,19 +91,19 @@ Review the default values to see if any need to be updated to reflect the target The operator provides three REST certificate options: -* `NONE` will disable the REST server. -* `SELF_SIGNED_CERT` will generate self-signed certificates. -* `CUSTOM_CERT` provides a mechanism to provide certificates that were created and signed by some other means. +* `NONE` Disables the REST server. +* `SELF_SIGNED_CERT` Generates self-signed certificates. +* `CUSTOM_CERT` Provides a mechanism to provide certificates that were created and signed by some other means. ## Decide which options to enable The operator provides some optional features that can be enabled in the configuration file. -### Load Balancing +### Load balancing with an Ingress controller or a web server -The operator can install the Traefik Ingress provider to provide load balancing for web applications running in WebLogic clusters. If enabled, an instance of Traefik and an Ingress will be created for each WebLogic cluster. Additional configuration is performed when creating the domain. +You can choose a load balancer provider for your WebLogic domains running in a Kubernetes cluster. Please refer to Load balancing with Voyager Ingress Controller, [Load balancing with Traefik Ingress Controller](site/traefik.md), and [Load balancing with Apache Web Server](site/apache.md) for information about the current capabilities and setup instructions for each of the supported load balancers. -Note that the technology preview release provides only basic load balancing: +Note that this release provides only basic load balancing: * Only HTTP(S) is supported. Other protocols are not supported. * A root path rule is created for each cluster. Rules based on the DNS name, or on URL paths other than ‘/’, are not supported. @@ -111,17 +111,17 @@ Note that the technology preview release provides only basic load balancing: Note that Ingresses are not created for servers that are not part of a WebLogic cluster, including the Administration Server. Such servers are exposed externally using NodePort services. -### Log integration with ELK +### Log integration with Elastic Stack -The operator can install the ELK stack and publish its logs into ELK. If enabled, ElasticSearch and Kibana will be installed in the `default` namespace, and a logstash container will be created in the operator pod. Logstash will be configured to publish the operator’s logs into ElasticSearch, and the log data will be available for visualization and analysis in Kibana. +The operator can install the Elastic Stack and publish its logs into it. If enabled, Elasticsearch and Kibana will be installed in the `default` namespace, and a Logstash container will be created in the operator pod. Logstash will be configured to publish the operator’s logs into Elasticsearch, and the log data will be available for visualization and analysis in Kibana. -To enable the ELK integration, set the `elkIntegrationEnabled` option to `true`. +To enable the Elastic Stack integration, set the `elkIntegrationEnabled` option to `true`. ## Deploying the operator to a Kubernetes cluster At this point, you've created a custom inputs file, or you've decided to use the default one. -Next, choose and create a directory that generated operator related files will be stored in, e.g. /path/to/weblogic-operator-output-directory. +Next, choose and create a directory that generated operator-related files will be stored in, for example, `/path/to/weblogic-operator-output-directory`. Finally, run the operator installation script to deploy the operator, pointing it at your inputs file and your output directory: @@ -135,20 +135,20 @@ Finally, run the operator installation script to deploy the operator, pointing i The script will carry out the following actions: -* Create a directory for the generated Kubernetes YAML files for this operator. The pathname is /path/to/weblogic-operator-output-directory/weblogic-operators/ Date: Mon, 30 Apr 2018 10:14:22 -0400 Subject: [PATCH 129/186] minor edits --- site/k8s_setup.md | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/site/k8s_setup.md b/site/k8s_setup.md index 50a05078098..948670a9934 100644 --- a/site/k8s_setup.md +++ b/site/k8s_setup.md @@ -11,18 +11,18 @@ If you need some help setting up a Kubernetes environment to experiment with the * Set up your own Kubernetes environment on bare compute resources on a cloud. * Use you cloud provider's management console to provision a managed Kubernetes environment. -* Install Kubernetes on your own compute resources (i.e. "real" computers, outside a cloud). +* Install Kubernetes on your own compute resources (for example, "real" computers, outside a cloud). "Development/test" options: * Install [Docker for Mac](https://docs.docker.com/docker-for-mac/#kubernetes) and enable its embedded Kubernetes cluster (or register for the [Docker for Windows](https://beta.docker.com/form) beta and wait until Kubernetes is available there). * Install [Minikube](https://github.com/kubernetes/minikube) on your Windows/Linux/Mac computer. -We have provided our hints and tips for several of these options in the sections below: +We have provided our hints and tips for several of these options in the sections below. ## Set up Kubernetes on bare compute resources in a cloud -Follow the basic steps from the [Terraform Kubernetes installer for Oracle Cloud Infrastructure](https://github.com/oracle/terraform-kubernetes-installer): +Follow the basic steps from the [Terraform Kubernetes installer for Oracle Cloud Infrastructure](https://github.com/oracle/terraform-kubernetes-installer). ### Prerequisites @@ -34,7 +34,7 @@ providers { oci = "/terraform-provider-oci" } ``` -4. Ensure you have [Kubectl][Kubectl] installed if you plan to interact with the cluster locally. +4. Ensure that you have [Kubectl][Kubectl] installed if you plan to interact with the cluster locally. ### Quick Start @@ -50,13 +50,13 @@ cd terraform-kubernetes-installer terraform init ``` -3. Copy the example terraform.tvfars: +3. Copy the example `terraform.tvfars`: ``` cp terraform.example.tfvars terraform.tfvars ``` -4. Edit the `terraform.tvfars` file to include values for your tenancy, user, and compartment. Optionally edit variables to change the `Shape` of the VMs for your Kubernetes master and workers, and your etcd cluster. For example: +4. Edit the `terraform.tvfars` file to include values for your tenancy, user, and compartment. Optionally, edit variables to change the `Shape` of the VMs for your Kubernetes master and workers, and your `etcd` cluster. For example: ``` #give a label to your cluster to help identify it if you have multiple @@ -138,20 +138,20 @@ $ ``` -## Install Kubernetes on your own compute resources (e.g. Oracle Linux servers outside a cloud) +## Install Kubernetes on your own compute resources (for example, Oracle Linux servers outside a cloud) -These instructions are for Oracle Linux 7u2+. If you are using a different flavor of Linux, you will need to adjust accordingly. +These instructions are for Oracle Linux 7u2+. If you are using a different flavor of Linux, you will need to adjust them accordingly. -**NOTE** These steps must be run with the `root` user, until specified otherwise! Any time you see `YOUR_USERID` in a command, you should replace it with your actual userid. +**NOTE**: These steps must be run with the `root` user, until specified otherwise! Any time you see `YOUR_USERID` in a command, you should replace it with your actual `userid`. -Choose the directories where your Docker and Kubernetes files will be stored. The Docker directory should be on a disk with a lot of free space (more than 100GB) because it will be used for the `/var/lib/docker` file syste, which contains all of your images and containers. The Kubernetes directory will be used for the `/var/lib/kubelet` file system and persistent volume storage. +Choose the directories where your Docker and Kubernetes files will be stored. The Docker directory should be on a disk with a lot of free space (more than 100GB) because it will be used for the `/var/lib/docker` file system, which contains all of your images and containers. The Kubernetes directory will be used for the `/var/lib/kubelet` file system and persistent volume storage. ``` export docker_dir=/scratch/docker export k8s_dir=/scratch/k8s_dir ``` -Create a shell script that sets up the necessary environment variables. You should probably just append this to the user's `.bashrc` so that it will get executed at login. You will also need to configure your proxy settings in here if you are behind an HTTP proxy: +Create a shell script that sets up the necessary environment variables. You should probably just append this to the user's `.bashrc` so that it will get executed at login. You will also need to configure your proxy settings here if you are behind an HTTP proxy: ``` export PATH=$PATH:/sbin:/usr/sbin @@ -186,14 +186,14 @@ If you want command completion, you can add the following to the script: source <(kubectl completion bash) ``` -Create the directories we need: +Create the directories you need: ``` mkdir -p $docker_dir $k8s_dir/kubelet ln -s $k8s_dir/kubelet /var/lib/kubelet ``` -Set an environment variable with the Docker version we want to install: +Set an environment variable with the Docker version you want to install: ``` docker_version="17.03.1.ce" @@ -219,7 +219,7 @@ diff /etc/sysconfig/docker /tmp/docker.out mv /tmp/docker.out /etc/sysconfig/docker ``` -Set up the Docker network, including the HTTP proxy configuration if you need it: +Set up the Docker network, including the HTTP proxy configuration, if you need it: ``` # generate a custom /setc/sysconfig/docker-network @@ -238,7 +238,7 @@ Add your user to the `docker` group: usermod -aG docker YOUR_USERID ``` -Enable and start the Docker service that we just installed and configured: +Enable and start the Docker service that you just installed and configured: ``` systemctl enable docker && systemctl start docker @@ -314,7 +314,7 @@ chown YOUR_USERID:YOUR_GROUP $KUBECONFIG chmod 644 $KUBECONFIG ``` -**NOTE** The following steps should be run with your normal (non-`root`) user. +**NOTE**: The following steps should be run with your normal (non-`root`) user. Configure CNI: @@ -357,7 +357,7 @@ Congratulations! Docker and Kubernetes are installed and configured! ## Install Docker for Mac with Kubernetes -Docker for Mac 17.12 CE Edge provides an [embedded Kubernetes environment](https://docs.docker.com/docker-for-mac/#kubernetes) that is a pretty quick and easy way to get a simple test environment set up on your Mac. To set it up, follow these instructions: +Docker for Mac 17.12 CE Edge provides an [embedded Kubernetes environment](https://docs.docker.com/docker-for-mac/#kubernetes) that is a quick and easy way to get a simple test environment set up on your Mac. To set it up, follow these instructions: Install "Docker for Mac" from the Edge channel [https://download.docker.com/mac/edge/Docker.dmg](https://download.docker.com/mac/edge/Docker.dmg). Then start up the Docker application (press Command-Space bar, type in `Docker` and run it). After it is running you will see the Docker icon appear in your status bar: @@ -371,7 +371,7 @@ Go to the "Kubernetes" tab and click on the option to enable Kubernetes: ![Enable Kubernetes setting](images/docker-enable-k8s.png) -**Note** If you are behind an HTTP proxy, then you should also go to the "Proxies" tab and enter your proxy details. +**Note**: If you are behind an HTTP proxy, then you should also go to the "Proxies" tab and enter your proxy details. Docker will download the Kuberentes components and start them up for you. When it is done, you will see the Kubernetes status go to green/running in the menu: From a75880aeb08072f0d61498b6e285799a5b7a13a2 Mon Sep 17 00:00:00 2001 From: Rosemary Marano Date: Mon, 30 Apr 2018 13:29:16 -0400 Subject: [PATCH 130/186] incorporate Monica's review comment --- site/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/installation.md b/site/installation.md index a378f149b67..3f682f3178f 100644 --- a/site/installation.md +++ b/site/installation.md @@ -103,7 +103,7 @@ The operator provides some optional features that can be enabled in the configur You can choose a load balancer provider for your WebLogic domains running in a Kubernetes cluster. Please refer to Load balancing with Voyager Ingress Controller, [Load balancing with Traefik Ingress Controller](site/traefik.md), and [Load balancing with Apache Web Server](site/apache.md) for information about the current capabilities and setup instructions for each of the supported load balancers. -Note that this release provides only basic load balancing: +Note these limitations: * Only HTTP(S) is supported. Other protocols are not supported. * A root path rule is created for each cluster. Rules based on the DNS name, or on URL paths other than ‘/’, are not supported. From b4302b834be723e44ef41969de505a59f632fc2a Mon Sep 17 00:00:00 2001 From: Rosemary Marano Date: Mon, 30 Apr 2018 13:49:16 -0400 Subject: [PATCH 131/186] minor edits --- site/manually-creating-domain.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/site/manually-creating-domain.md b/site/manually-creating-domain.md index a1c3263d72a..6cd10298907 100644 --- a/site/manually-creating-domain.md +++ b/site/manually-creating-domain.md @@ -1,8 +1,8 @@ # Manually creating a WebLogic domain -**PLEASE NOTE** This page is a work in progress, we have some rough notes in here, we are working on writing better doc for how to set up this integration. +**PLEASE NOTE**: This page is a work in progress. We have some rough notes here and are working on writing better documentation for how to set up this procedure. -If creating the domain manually, using a WLST script for example, the domain must be configured to meet these requirements: +If you are creating the domain manually, for example, using a WLST script, the domain must be configured to meet these requirements: * Domain directory should be in `/shared/domain`. * Applications directory should be in `/shared/applications`. @@ -11,11 +11,11 @@ If creating the domain manually, using a WLST script for example, the domain mus ## Use the scripts to create the sample YAML files -The `create-weblogic-domain.sh` script described in the previous section can be executed with the “-g” option, which will cause it to generate the YAML files but take no action at all against the Kubernetes environment. This is a useful way to create the sample YAML files needed to manually create a domain. +The `create-weblogic-domain.sh` script can be executed with the `-g` option, which will cause it to generate the YAML files but take no action at all against the Kubernetes environment. This is a useful way to create the sample YAML files needed to manually create a domain. First, make a copy of `create-weblogic-domain-inputs.yaml` and customize it. -Next, choose and create a directory that generated operator related files will be stored in, e.g. /path/to/weblogic-operator-output-directory +Next, choose and create a directory that generated operator-related files will be stored in, for example, `/path/to/weblogic-operator-output-directory`. Then, execute the script, pointing it at your inputs file and output directory: @@ -25,7 +25,7 @@ Then, execute the script, pointing it at your inputs file and output directory: -o /path/to/weblogic-operator-output-directory ``` -The following YAML files will be generated in the /path/to/weblogic-operator-output-directory/weblogic-domains/ directory: +The following YAML files will be generated in the `/path/to/weblogic-operator-output-directory/weblogic-domains/` directory: * `weblogic-domain-pv.yaml` can be customized and used to create the persistent volume for this domain. * `weblogic-domain-pvc.yaml` can be customized and used to create the persistent volume claim for this domain. @@ -48,13 +48,13 @@ The file `weblogic-domain-pv.yaml` contains a template to create a persistent vo * Persistent volume name * Storage class name * The amount of storage to allocate -* The physical location of the persistent volume (edit). Prior to creating the persistent volume you need to ensure this location exists and has read/write/execute permission set for the account that Kubernetes is running from. +* The physical location of the persistent volume (edit). Prior to creating the persistent volume, you need to ensure this location exists and has read/write/execute permissions set for the account that Kubernetes is running from. * The access mode is Read/Write Many. You must use a provider that supports Read/Write Many. * The contents of the volume are retained across restarts of the Kubernetes environment. ## Creating the persistent volume -To create the persistent volume issue the following command: +To create the persistent volume, issue the following command: ``` kubectl create –f weblogic-domain-pv.yaml @@ -70,7 +70,7 @@ Replace `PV_NAME` with the name of the persistent volume. ## Preparing to create the persistent volume claim -The file `weblogic-domain-pvc.yaml` contains a template to claim a portion of the persistent volume storage. The customizable items are listed below: +The file, `weblogic-domain-pvc.yaml`, contains a template to claim a portion of the persistent volume storage. The customizable items are: * Persistent volume claim name * Namespace name @@ -80,7 +80,7 @@ The file `weblogic-domain-pvc.yaml` contains a template to claim a portion of th ## Creating the persistent volume claim -To create the persistent volume claim issue the following command: +To create the persistent volume claim, issue the following command: ``` kubectl create –f weblogic-domain-pvc.yaml @@ -96,7 +96,7 @@ Replace `PVC_NAME` with the name of the persistent volume claim. ## Preparing to create the domain custom resource -The file `domain-custom-resource.yaml` contains a template to create the domain custom resource. The customizable items are listed below: +The file, `domain-custom-resource.yaml`, contains a template to create the domain custom resource. The customizable items are: * Domain name * Namespace name From 8bab431028221c77b33336a01dce242af68cb2b0 Mon Sep 17 00:00:00 2001 From: Rosemary Marano Date: Mon, 30 Apr 2018 14:47:24 -0400 Subject: [PATCH 132/186] editing pass --- site/apache.md | 64 ++++++++++++++++++++++---------------------------- 1 file changed, 28 insertions(+), 36 deletions(-) diff --git a/site/apache.md b/site/apache.md index 7b7f79a04c8..ecc9c6caa7b 100644 --- a/site/apache.md +++ b/site/apache.md @@ -1,15 +1,15 @@ -# Load Balancing with Apache Web Server +# Load balancing with the Apache web server -This page describes how to setup and start a Apache Web Server for load balancing inside a Kubernets cluster. The configuration and startup can either be automatic when you create a domain using the WebLogic Operator's `create-weblogic-domain.sh` script, or manually if you have an existing WebLogic domain configuration. +This document describes how to set up and start an Apache web server for load balancing inside a Kubernets cluster. The configuration and startup can be either automatic, when you create a domain using the WebLogic Operator's `create-weblogic-domain.sh` script, or manual, if you have an existing WebLogic domain configuration. -## Build Docker Image for Apache Web Server +## Build the Docker image for the Apache web server -You need to build the Docker image for Apache Web Server that enbeds Oracle WebLogic Server Proxy Plugin. +You need to build the Docker image for the Apache web server that embeds the Oracle WebLogic Server Proxy Plugin. - 1. Download and build the Docker image for the Apache Web Server with 12.2.1.3.0 Oracle WebLogic Server Proxy Plugin. See the instructions in [Apache Web Server with Oracle WebLogic Server Proxy Plugin on Docker](https://github.com/oracle/docker-images/tree/master/OracleWebLogic/samples/12213-webtier-apache). + 1. Download and build the Docker image for the Apache web server with the 12.2.1.3.0 Oracle WebLogic Server Proxy Plugin. See the instructions in [Apache Web Server with Oracle WebLogic Server Proxy Plugin on Docker](https://github.com/oracle/docker-images/tree/master/OracleWebLogic/samples/12213-webtier-apache). - 2. tag your Docker image to `store/oracle/apache:12.2.1.3` using `docker tag` command. + 2. Tag your Docker image, `store/oracle/apache:12.2.1.3`, using the `docker tag` command. ``` @@ -17,16 +17,16 @@ You need to build the Docker image for Apache Web Server that enbeds Oracle WebL ``` -More information about the Apache plugin can be found at: [Apache Web Server with Oracle WebLogic Server Proxy Plugin on Docker](https://docs.oracle.com/middleware/1213/webtier/develop-plugin/apache.htm#PLGWL395). +For more information about the Apache plugin, see [Apache Web Server with Oracle WebLogic Server Proxy Plugin on Docker](https://docs.oracle.com/middleware/1213/webtier/develop-plugin/apache.htm#PLGWL395). -Once you have access to the Docker image of the Apache Web Server, you can go ahead follow the instructions below to setup and start Kubernetes artifacts for Apache Web Server. +After you have access to the Docker image of the Apache web server, you can follow the instructions below to set up and start the Kubernetes artifacts for the Apache web server. -## Use Apache load balancer with a WebLogic domain created with the WebLogic Operator +## Use the Apache load balancer with a WebLogic domain created with the WebLogic Operator -Please refer to [Creating a domain using the WebLogic Operator](creating-domain.md) for how to create a domain with the WebLogic Operator. +For how to create a domain with the WebLogic Operator, please refer to [Creating a domain using the WebLogic Operator](creating-domain.md). -You need to configure Apache Web Server as your load balancer for a WebLogic domain by setting the `loadBalancer` option to `APACHE` in `create-weblogic-domain-inputs.yaml` (as shown below) when running the `create-weblogic-domain.sh` script to create a domain. +You need to configure the Apache web server as your load balancer for a WebLogic domain by setting the `loadBalancer` option to `APACHE` in the `create-weblogic-domain-inputs.yaml` (as shown below) when running the `create-weblogic-domain.sh` script to create a domain. ``` @@ -36,9 +36,9 @@ loadBalancer: APACHE ``` -The `create-weblogic-domain.sh` script installs the Apache Web Server with Oracle WebLogic Server Proxy Plugin into the Kubernetes *cluster* in the same namespace as the *domain*. +The `create-weblogic-domain.sh` script installs the Apache web server with the Oracle WebLogic Server Proxy Plugin into the Kubernetes *cluster* in the same namespace as the *domain*. -The Apache Web Server will expose a `NodePort` that allows access to the load balancer from outside of the Kubernetes cluster. The port is configured by setting 'loadBalancerWebPort' in `create-weblogic-domain-inputs.yaml` file. +The Apache web server will expose a `NodePort` that allows access to the load balancer from outside of the Kubernetes cluster. The port is configured by setting `loadBalancerWebPort` in the `create-weblogic-domain-inputs.yaml` file. ``` @@ -48,11 +48,11 @@ loadBalancerWebPort: 30305 ``` -The user can access an application from outside of the Kubernetes cluster via http://:30305/. +Users can access an application from outside of the Kubernetes cluster by using `http://:30305/`. ### Use the default plugin WL module configuration -By default, the Apache Docker image supports a simple WebLogic server proxy plugin configuration for a single WebLogic domain with an admin server and a cluster. The `create-weblogic-domain.sh` script automatically customizes the default behavior based on your domain configuration. The default setting only supports the type of load balancing that uses the root path ("/"). You can further customize the root path of the load balancer with `loadBalancerAppPrepath` property in the `create-weblogic-domain-inputs.yaml` file. +By default, the Apache Docker image supports a simple WebLogic Server proxy plugin configuration for a single WebLogic domain with an Administration Server and a cluster. The `create-weblogic-domain.sh` script automatically customizes the default behavior based on your domain configuration. The default setting supports only the type of load balancing that uses the root path ("/"). You can further customize the root path of the load balancer with the `loadBalancerAppPrepath` property in the `create-weblogic-domain-inputs.yaml` file. ``` @@ -62,11 +62,11 @@ loadBalancerAppPrepath: /weblogic ``` -The user can then access an application from utsidie of the Kubernetes cluster via `http://:30305/weblogic/,` and the admin can access the admin console via `http://:30305/console`. +Users can then access an application from outside of the Kubernetes cluster by using `http://:30305/weblogic/,` and the administrator can access the Administration Console by using `http://:30305/console`. -The generated Kubernetes yaml files look like the following given the domainUID "domain1". +The generated Kubernetes YAML files look like the following, given the `domainUID`, "`domain1`". -`weblogic-domain-apache.yaml` for Apache web server deployment +Sample `weblogic-domain-apache.yaml` file for Apache web server deployment. ``` @@ -254,7 +254,7 @@ spec: ``` -`weblogic-domain-apache-security.yaml` for associated RBAC roles and role bindings +Sample `weblogic-domain-apache-security.yaml` file for associated RBAC roles and role bindings. ``` @@ -394,7 +394,7 @@ domain1-apache-webtier 2h ### Use your own plugin WL module configuration -You can fine tune the behavior of the Apache plugin by providing your own Apache plugin configuration. You put your custom_mod_wl_apache.conf file in a local directory, for example `` , and specify this location in the `create-weblogic-domain-inputs.yaml` file as follows. +You can fine tune the behavior of the Apache plugin by providing your own Apache plugin configuration. You put your `custom_mod_wl_apache.conf` file in a local directory, for example, `` , and specify this location in the `create-weblogic-domain-inputs.yaml` file as follows: ``` @@ -406,9 +406,9 @@ loadBalancerVolumePath: ``` -After the `loadBalancerVolumePath` property is specified, the `create-weblogic-domain.sh` script will use the custom_mod_wl_apache.config file in `` directory to replace what is in the Docker image. +After the `loadBalancerVolumePath` property is specified, the `create-weblogic-domain.sh` script will use the `custom_mod_wl_apache.config` file in the `` directory to replace what is in the Docker image. -The generated yaml files will look similar except with un-commented entries like bellow. +The generated YAML files will look similar except with un-commented entries like below: ``` @@ -418,7 +418,7 @@ The generated yaml files will look similar except with un-commented entries like hostPath: - path: + path: containers: @@ -440,21 +440,13 @@ The generated yaml files will look similar except with un-commented entries like ## Use the Apache load balancer with a manually created WebLogic Domain -If your WebLogic domain is not created by the WebLogic Operator, you need to manually create and start all Kubernetes' artifacts for Apache Web Server. - - - 1. Create your own custom_mod_wl_apache.conf file, and put it in a local dir, say ``. See the instructions in [Apache Web Server with Oracle WebLogic Server Proxy Plugin on Docker](https://docs.oracle.com/middleware/1213/webtier/develop-plugin/apache.htm#PLGWL395). - - 2. Create the Apache deployment yaml file. See the example above. Note that you need to use the **volumes** and **volumeMounts** to mount `` in to `/config` directory inside the pod that runs Apache web tier. Note that the Apache Web Server needs to be in the same Kubernetes namespace as the WebLogic domains that it needs to access. - - 3. Create a RBAC yaml file. See the example above - -Note that you can choose to run one Apache Web Server to load balance to multiple domains/clusters inside the same Kubernetes cluster as long as the Apache Web Server and the domains are all in the same namespace. - - - +If your WebLogic domain is not created by the WebLogic Operator, you need to manually create and start all Kubernetes' artifacts for the Apache web server. + 1. Create your own `custom_mod_wl_apache.conf` file, and put it in a local directory, for example, ``. See the instructions in [Apache Web Server with Oracle WebLogic Server Proxy Plugin on Docker](https://docs.oracle.com/middleware/1213/webtier/develop-plugin/apache.htm#PLGWL395). + 2. Create the Apache deployment YAML file. See the example above. Note that you need to use the **volumes** and **volumeMounts** to mount `` into the `/config` directory inside the pod that runs the Apache web tier. Note that the Apache web server needs to be in the same Kubernetes namespace as the WebLogic domain that it needs to access. + 3. Create a RBAC YAML file. See the example above. +Note that you can choose to run one Apache web server to balance the loads from multiple domains/clusters inside the same Kubernetes cluster, as long as the Apache web server and the domains are all in the same namespace. From 0afc2b5994d8e62b1125896e400f72455e01973e Mon Sep 17 00:00:00 2001 From: Rosemary Marano Date: Mon, 30 Apr 2018 15:30:33 -0400 Subject: [PATCH 133/186] change capitalization scheme --- site/name-changes.md | 62 ++++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/site/name-changes.md b/site/name-changes.md index 16fc30bce54..ce0285b5349 100644 --- a/site/name-changes.md +++ b/site/name-changes.md @@ -1,4 +1,4 @@ -# Oracle WebLogic Server Kubernetes Operator Name Changes +# Oracle WebLogic Server Kubernetes Operator name changes The initial version of the WebLogic Server Kubernetes Operator did not use consistent conventions for many customer visible names. @@ -14,13 +14,13 @@ We're not providing an upgrade tool or backward compatibility with the previous This document lists the customer visible naming changes. Also, the WebLogic Server Kubernetes Operator documentation has been updated. -## Customer Visible Files +## Customer visible files -### Files for Creating and Deleting Operators and Domains +### Files for creating and deleting operators and domains -The following files are used to create the operator and to create and delete domains. +The following files are used to create the operator, and to create and delete domains. -| Previous File Name | New File Name | +| Previous file name | New file name | | --- | --- | | `kubernetes/create-domain-job.sh` | `kubernetes/create-weblogic-domain.sh` | | `kubernetes/create-domain-job-inputs.yaml` | `kubernetes/create-weblogic-domain-inputs.yaml` | @@ -28,27 +28,27 @@ The following files are used to create the operator and to create and delete dom | `kubernetes/create-weblogic-operator.sh` | same | | `kubernetes/delete-domain.sh` | `kubernetes/delete-weblogic-operator-resources.sh` | -### Generated YAML Files for Operators and Domains +### Generated YAML files for operators and domains The create scripts generate a number of YAML files that are used to configure the corresponding Kubernetes artifacts for the operator and the domains. Typically, customers do not use these YAML files. However, customers can look at them. They can also change the operator and domain configuration by editing these files and reapplying them. -#### Directory for the Generated YAML Files +#### Directory for the generated YAML files Previously, these files were placed in the `kubernetes` directory (for example, `kubernetes/weblogic-operator.yaml`). Now, they are placed in per-operator and per-domain directories (because a Kubernetes cluster can have more than one operator and an operator can manage more than one domain). -The customer must create a directory that will parent the per-operator and per-domain directories, and use the `-o` option to pass the name of that directory to the create script, for example: +The customer must create a directory that will parent the per-operator and per-domain directories, and use the `-o` option to pass the name of that directory to the create script, for example, `mkdir /scratch/my-user-projects create-weblogic-operator.sh -o /scratch/my-user-projects`. The pathname can be either a full path name or a relative path name. If it's a relative pathname, then it's relative to the directory of the shell invoking the create script. -The per-operator directory name is: +The per-operator directory name is `/weblogic-operators/`. -Similarly, the per-domain directory name is: +Similarly, the per-domain directory name is `/weblogic-domains/`. -#### What If I Mess Up Creating a Domain or Operator And Want To Do It Again? +#### What if I mess up creating a domain or operator and want to do it again? * Remove the resources that were created for the domain: * `kubernetes/delete-weblogic-domain-resources.sh yourDomainUID` @@ -61,7 +61,7 @@ Similarly, the per-domain directory name is: If you run the create script without cleaning up the previously generated directory, the create script will tell you about the offending files and then exit without creating anything. -#### Location of the Input YAML Files +#### Location of the input YAML files The create scripts support an `-i` option for specifying the location of the inputs file. Similar to the `-o` option, the path can be either a full path name or a relative path name. Relative path names are relative to the directory of the shell invoking the create script. @@ -72,19 +72,19 @@ Previously, `kubernetes/create-domain-job.sh` used `kubernetes/create-domain-job Also, we do not want the customer to have to change files in the operator's install directory. Because of this, the `-i` option MUST be specified when calling `kubernetes/create-weblogic-operator.sh`. The basic flow is: * Pick a user projects directory, for example, `/scratch/my-user-projects` -* `mkdir /scratch/my-user-projects` + * `mkdir /scratch/my-user-projects` * Pick a unique ID for the domain, for example, `foo.com` -* `cp kubernetes/create-weblogic-domain-inputs.yaml my-inputs.yaml` -* Set the domainUid in `my-inputs.yaml` to `foo.com` -* `kubernetes/create-weblogic-operator.sh -i my-inputs.yaml -o /scratch/my-user-projects` + * `cp kubernetes/create-weblogic-domain-inputs.yaml my-inputs.yaml` +* Set the `domainUid` in `my-inputs.yaml` to `foo.com` + * `kubernetes/create-weblogic-operator.sh -i my-inputs.yaml -o /scratch/my-user-projects` **Note:** `my-inputs.yaml` will be copied to `/scratch/my-user-projects/weblogic-domains/foo.com/create-weblogic-domain-inputs.yaml` -#### File Names of the Generated YAML Files +#### File names of the generated YAML files The names of several of the generated YAML files have changed. -| Previous File Name | New File Name | +| Previous file name | New file name | | --- | --- | | `domain-custom-resource.yaml` | same | | `domain-job.yaml` | `create-weblogic-domain-job.yaml` | @@ -95,7 +95,7 @@ The names of several of the generated YAML files have changed. | `traefik-rbac.yaml` | `weblogic-domain-traefik-security-${clusterName, lower case}.yaml` | | `weblogic-operator.yaml` | same | -## Input File Contents +## Input file contents Some of the contents of the inputs files have changed: * Some properties have been renamed * Some properties that are no longer needed have been removed @@ -104,17 +104,17 @@ Some of the contents of the inputs files have changed: ### create-weblogic-operator-inputs.yaml -#### Property Names +#### Property names -| Previous Property Name | New Property Name | +| Previous property name | New property name | | --- | --- | | `image` | `weblogicOperatorImage` | | `imagePullPolicy` | `weblogicOperatorImagePullPolicy` | | `imagePullSecretName` | `weblogicOperatorImagePullSecretName` | -#### Property Values +#### Property values -| Previous Property Name | Property Name | Old Property Value | New Property Value | +| Previous property name | Property name | Old property value | New property value | | --- | --- | --- | --- | | `externalRestOption` | `externalRestOption` | `none` | `NONE` | | `externalRestOption` | `externalRestOption` | `custom-cert` | `CUSTOM_CERT` | @@ -122,9 +122,9 @@ Some of the contents of the inputs files have changed: ### create-weblogic-domain-inputs.yaml -#### Property Names +#### Property names -| Previous Property Name | New Property Name | +| Previous property name | New property name | | --- | --- | | `createDomainScript` | This property has been removed | | `domainUid` | `domainUID` | @@ -141,27 +141,27 @@ Some of the contents of the inputs files have changed: | `persistenceVolumeName` | This property has been removed | | `secretName` | `weblogicCredentialsSecretName` | -#### Properties That Must be Customized +#### Properties that must be customized The following input properties, which used to have default values, now must be uncommented and customized. -| Previous Property Name | New Property Name | Previous Default Value | Notes | +| Previous property name | New property name | Previous Default Value | Notes | | --- | --- | --- | --- | | `domainUid` | `domainUID` | `domain1` | Because the domain UID is supposed to be unique across the Kubernetes cluster, the customer must choose one. | | `persistencePath` | `weblogicDomainStoragePath` | `/scratch/k8s_dir/persistentVolume001` | The customer must select a directory for the domain's storage. | | `nfsServer` | `weblogicDomainStorageNFSServer` | `nfsServer` | If `weblogicDomainStorageType` is NFS, then the customer must specify the name or IP of the NFS server. | -#### Property Values +#### Property values -| Previous Property Name | New Property Name | Old Property Value | New Property Value | +| Previous property name | New property name | Old property value | New property value | | --- | --- | --- | --- | | `loadBalancer` | `loadBalancer` | `none` | `NONE` | | `loadBalancer` | `loadBalancer` | `traefik` | `TRAEFIK` | | `persistenceType` | `weblogicDomainStorageType` | `hostPath` | `HOST_PATH` | | `persistenceType` | `weblogicDomainStorageType` | `nfs` | `NFS` | -## Kubernetes Artifact Names +## Kubernetes artifact names -| Artifact Type | Previous Name | New Name | +| Artifact type | Previous name | New name | | --- | --- | --- | | persistent volume | `${domainUid}-${persistenceVolume}` or `${persistenceVolume}` | `${domainUID}-weblogic-domain-pv` | | persistent volume claim | `${domainUid}-${persistenceVolumeClaim}` or `${persistenceVolumeClaim}` | `${domainUID}-weblogic-domain-pvc` | From 62487a733ea9682e3707ef7c2fd10af0955872b5 Mon Sep 17 00:00:00 2001 From: Tom Moreau Date: Mon, 30 Apr 2018 15:33:56 -0400 Subject: [PATCH 134/186] Version the input files --- kubernetes/create-weblogic-domain-inputs.yaml | 3 +++ .../create-weblogic-operator-inputs.yaml | 3 +++ kubernetes/internal/create-weblogic-domain.sh | 14 +++++++++++++- .../internal/create-weblogic-operator.sh | 14 +++++++++++++- .../operator/create/CreateDomainInputs.java | 14 ++++++++++++++ .../CreateDomainInputsValidationTest.java | 16 ++++++++++++++++ .../operator/create/CreateOperatorInputs.java | 5 +++++ .../CreateOperatorInputsValidationTest.java | 18 ++++++++++++++++++ 8 files changed, 85 insertions(+), 2 deletions(-) diff --git a/kubernetes/create-weblogic-domain-inputs.yaml b/kubernetes/create-weblogic-domain-inputs.yaml index 2e5e5b9f895..66a98bfd658 100644 --- a/kubernetes/create-weblogic-domain-inputs.yaml +++ b/kubernetes/create-weblogic-domain-inputs.yaml @@ -1,6 +1,9 @@ # Copyright 2018, Oracle Corporation and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. +# The version of this inputs file. Do not modify. +version: create-weblogic-domain-inputs/v1 + # Port number for admin server adminPort: 7001 diff --git a/kubernetes/create-weblogic-operator-inputs.yaml b/kubernetes/create-weblogic-operator-inputs.yaml index e1e1fb5529a..8681bac66e1 100644 --- a/kubernetes/create-weblogic-operator-inputs.yaml +++ b/kubernetes/create-weblogic-operator-inputs.yaml @@ -1,6 +1,9 @@ # Copyright 2017, 2018 Oracle Corporation and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. +# The version of this inputs file. Do not modify. +version: create-weblogic-operator-inputs/v1 + # The name of the service account that the operator will use to # make requests to the Kubernetes API server. # The name must be lowercase diff --git a/kubernetes/internal/create-weblogic-domain.sh b/kubernetes/internal/create-weblogic-domain.sh index 4e551e1fdad..8659101cffb 100755 --- a/kubernetes/internal/create-weblogic-domain.sh +++ b/kubernetes/internal/create-weblogic-domain.sh @@ -107,6 +107,16 @@ function initAndValidateOutputDir { domain-custom-resource.yaml } +# +# Function to validate the version of the inputs file +# +function validateVersion { + local requiredVersion='create-weblogic-domain-inputs/v1' + if [ "${version}" != "${requiredVersion}" ]; then + validationError "Invalid version: \"${version}\". Must be ${requiredVersion}." + fi +} + # # Function to ensure the domain uid is lowercase # @@ -376,7 +386,8 @@ function initialize { weblogicCredentialsSecretName \ namespace \ javaOptions \ - t3PublicAddress + t3PublicAddress \ + version validateIntegerInputParamsSpecified \ adminPort \ @@ -393,6 +404,7 @@ function initialize { exposeAdminT3Channel \ exposeAdminNodePort + validateVersion validateDomainUid validateNamespace validateClusterName diff --git a/kubernetes/internal/create-weblogic-operator.sh b/kubernetes/internal/create-weblogic-operator.sh index 0356a04df64..f8577020e4e 100755 --- a/kubernetes/internal/create-weblogic-operator.sh +++ b/kubernetes/internal/create-weblogic-operator.sh @@ -117,10 +117,12 @@ function initialize { # Parse the common inputs file parseCommonInputs - validateInputParamsSpecified serviceAccount namespace targetNamespaces weblogicOperatorImage + validateInputParamsSpecified version serviceAccount namespace targetNamespaces weblogicOperatorImage validateBooleanInputParamsSpecified elkIntegrationEnabled + validateVersion + validateServiceAccount validateNamespace @@ -203,6 +205,16 @@ function validateImagePullPolicy { fi } +# +# Function to validate the version of the inputs file +# +function validateVersion { + local requiredVersion='create-weblogic-operator-inputs/v1' + if [ "${version}" != "${requiredVersion}" ]; then + validationError "Invalid version: \"${version}\". Must be ${requiredVersion}." + fi +} + # # Function to validate the service account is lowercase # diff --git a/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputs.java b/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputs.java index c84ffa66666..bd41942dc6b 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputs.java +++ b/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputs.java @@ -76,6 +76,7 @@ public class CreateDomainInputs { private String loadBalancerVolumePath = ""; private String loadBalancerAppPrepath = ""; private String javaOptions = ""; + private String version = ""; public static CreateDomainInputs newInputs() throws Exception { return @@ -534,6 +535,19 @@ public CreateDomainInputs javaOptions(String javaOptions) { return this; } + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = convertNullToEmptyString(version); + } + + public CreateDomainInputs version(String version) { + setVersion(version); + return this; + } + private String convertNullToEmptyString(String val) { return Objects.toString(val, ""); } diff --git a/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputsValidationTest.java b/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputsValidationTest.java index e7c203c98b8..540ccf2f8d9 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputsValidationTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputsValidationTest.java @@ -48,6 +48,7 @@ public class CreateDomainInputsValidationTest { private static final String PARAM_LOAD_BALANCER_WEB_PORT = "loadBalancerWebPort"; private static final String PARAM_LOAD_BALANCER_DASHBOARD_PORT = "loadBalancerDashboardPort"; private static final String PARAM_JAVA_OPTIONS = "javaOptions"; + private static final String PARAM_VERSION = "version"; @Before public void setup() throws Exception { @@ -523,6 +524,21 @@ public void createDomain_with_missingJavaOptions_failsAndReturnsError() throws E failsAndPrints(paramMissingError(PARAM_JAVA_OPTIONS))); } + @Test + public void createDomain_with_missingVersion_failsAndReturnsError() throws Exception { + assertThat( + execCreateDomain(newInputs().version("")), + failsAndPrints(paramMissingError(PARAM_VERSION))); + } + + @Test + public void createDomainwith_invalidVersion_failsAndReturnsError() throws Exception { + String val = "no-such-version"; + assertThat( + execCreateDomain(newInputs().version(val)), + failsAndPrints(invalidEnumParamValueError(PARAM_VERSION, val))); + } + private void createDomain_with_validStartupControl_succeeds(String startupControl) throws Exception { createDomain_with_validInputs_succeeds(newInputs().startupControl(startupControl)); } diff --git a/operator/src/test/java/oracle/kubernetes/operator/create/CreateOperatorInputs.java b/operator/src/test/java/oracle/kubernetes/operator/create/CreateOperatorInputs.java index 36d0d56c02d..47e1f02b16a 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/create/CreateOperatorInputs.java +++ b/operator/src/test/java/oracle/kubernetes/operator/create/CreateOperatorInputs.java @@ -148,6 +148,11 @@ private String internalSans() { // Note: don't allow null strings since, if you use snakeyaml to write out the instance // to a yaml file, the nulls are written out as "null". Use "" instead. + private String version = ""; + public String getVersion() { return version; } + public void setVersion(String val) { version = convertNullToEmptyString(val); } + public CreateOperatorInputs version(String val) { setVersion(val); return this; } + private String serviceAccount = ""; public String getServiceAccount() { return serviceAccount; } public void setServiceAccount(String val) { serviceAccount = convertNullToEmptyString(val); } diff --git a/operator/src/test/java/oracle/kubernetes/operator/create/CreateOperatorInputsValidationTest.java b/operator/src/test/java/oracle/kubernetes/operator/create/CreateOperatorInputsValidationTest.java index ea006906937..e7438fa5725 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/create/CreateOperatorInputsValidationTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/create/CreateOperatorInputsValidationTest.java @@ -33,6 +33,7 @@ public void tearDown() throws Exception { } } + private static final String PARAM_VERSION = "version"; private static final String PARAM_SERVICE_ACCOUNT = "serviceAccount"; private static final String PARAM_NAMESPACE = "namespace"; private static final String PARAM_TARGET_NAMESPACES = "targetNamespaces"; @@ -50,6 +51,23 @@ public void tearDown() throws Exception { private static final String PARAM_JAVA_LOGGING_LEVEL = "javaLoggingLevel"; private static final String PARAM_ELK_INTEGRATION_ENABLED = "elkIntegrationEnabled"; + private static final String VERSION_V1 = "create-weblogic-operator-inputs/v1"; + + @Test + public void createOperator_with_missingVersion_failsAndReturnsError() throws Exception { + assertThat( + execCreateOperator(newInputs().version("")), + failsAndPrints(paramMissingError(PARAM_VERSION))); + } + + @Test + public void createOperator_with_invalidVersion_failsAndReturnsError() throws Exception { + String val = "no-such-version"; + assertThat( + execCreateOperator(newInputs().version(val)), + failsAndPrints(invalidEnumParamValueError(PARAM_VERSION, val))); + } + @Test public void createOperator_with_missingServiceAccount_failsAndReturnsError() throws Exception { assertThat( From e82a7ef4aaf9781cb23b5f08b25d2325d861deb6 Mon Sep 17 00:00:00 2001 From: Rosemary Marano Date: Mon, 30 Apr 2018 16:12:32 -0400 Subject: [PATCH 135/186] minor edits --- site/rbac.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/site/rbac.md b/site/rbac.md index 1b462f421b9..60e448410cb 100644 --- a/site/rbac.md +++ b/site/rbac.md @@ -1,24 +1,24 @@ -# RBAC +# Role-Based Access Control (RBAC) -The operator assumes that certain roles and role bindings are created on the Kubernetes cluster. The operator installation scripts create these, and the operator verifies that they are correct when the cluster starts up. This section lists the [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/) definitions that are created. +The operator assumes that certain roles and role bindings are created on the Kubernetes cluster. The operator installation scripts create these, and the operator verifies that they are correct when the cluster starts up. This document lists the [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/) definitions that are created. -The general design goal is to provide the operator with the minimum amount of permissions that it requires, and to favor built-in roles over custom roles where it make sense to do so. +The general design goal is to provide the operator with the minimum amount of permissions that it requires, and to favor built-in roles over custom roles, where it make sense to do so. -## KUBERNETES ROLE DEFINITIONS +## Kubernetes role definitions -| Cluster Role | Resources | Verbs | Notes | +| Cluster role | Resources | Verbs | Notes | | --- | --- | --- | --- | -| weblogic-operator-cluster-role | namespaces, persistentvolumes | get, list, watch | 1 | +| `weblogic-operator-cluster-role` | namespaces, persistentvolumes | get, list, watch | 1 | | | customresourcedefinitions in API group apiextensions.k8s.io | get, list, watch, create, update, patch, delete, deletecollection | | | | domains in API group weblogic.oracle | get, list, watch, update, patch | | | | Ingresses in API group extensions | get, list, watch, create, update, patch, delete, deletecollection | | -| weblogic-operator-cluster-role-nonresource | nonResourceURLs: ["/version/*"] | get | 1 | -|weblogic-operator-namespace-role | secrets, persistentvolumeclaims | get, list, watch | 2 | +| `weblogic-operator-cluster-role-nonresourc`e | nonResourceURLs: ["/version/*"] | get | 1 | +|`weblogic-operator-namespace-role` | secrets, persistentvolumeclaims | get, list, watch | 2 | | | services, pods, networkpolicies | get, list, watch, create, update, patch, delete, deletecollection | | -| NAMESPACE-operator-rolebinding-discovery | system:discovery in API group rbac.authorization.k8s.io | | 1 | -| NAMESPACE-operator-rolebinding-auth-delegator | system:auth-delegator in API group rbac.authorization.k8s.io | | 1 | +| `NAMESPACE-operator-rolebinding-discovery` | system:discovery in API group rbac.authorization.k8s.io | | 1 | +| `NAMESPACE-operator-rolebinding-auth-delegator` | system:auth-delegator in API group rbac.authorization.k8s.io | | 1 | -**Notes** +**Notes**: 1. This cluster role is assigned to the operator’s service account in the operator’s namespace. The uppercase text `NAMESPACE` in the cluster role name is replaced with the operator’s namespace. 2. This cluster role is assigned to the operator’s service account in each of the “target namespaces”; that is, each namespace that the operator is configured to manage. From b2f720172887297c214f50df1ef7ff87ed67449f Mon Sep 17 00:00:00 2001 From: Rosemary Marano Date: Mon, 30 Apr 2018 16:19:27 -0400 Subject: [PATCH 136/186] minor edits --- site/recent-changes.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/site/recent-changes.md b/site/recent-changes.md index ba64301c8ba..a2b91e096f7 100644 --- a/site/recent-changes.md +++ b/site/recent-changes.md @@ -1,8 +1,8 @@ -# Recent Changes to the Oracle WebLogic Server Kubernetes Operator +# Recent changes to the Oracle WebLogic Server Kubernetes Operator -This page tracks recent changes to the operator, especially ones that introduce backward incompatibilities. +This document tracks recent changes to the operator, especially ones that introduce backward incompatibilities. -| Date | Introduces Backward Incompatibilities | Change | +| Date | Introduces backward incompatibilities | Change | | --- | --- | --- | | March 20, 2018 | yes | See [Name Changes](name-changes.md). Several files and input parameters have been renamed. This affects how operators and domains are created. It also changes generated Kubernetes artifacts, therefore customers must recreate their operators and domains. | April 4, 2018 | yes | See [Name Changes](name-changes.md). Many Kubernetes artifact names and labels have changed. Also, the names of generated YAML files for creating a domain's PV and PVC have changed. Because of these changes, customers must recreate their operators and domains. From ce4bb72bf756d872970e23d0570cf9f70c611448 Mon Sep 17 00:00:00 2001 From: Rosemary Marano Date: Mon, 30 Apr 2018 16:46:26 -0400 Subject: [PATCH 137/186] changed Apache web server to Apache HTTP Server --- site/apache.md | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/site/apache.md b/site/apache.md index ecc9c6caa7b..b4915d5da8b 100644 --- a/site/apache.md +++ b/site/apache.md @@ -1,13 +1,13 @@ -# Load balancing with the Apache web server +# Load balancing with the Apache HTTP Server -This document describes how to set up and start an Apache web server for load balancing inside a Kubernets cluster. The configuration and startup can be either automatic, when you create a domain using the WebLogic Operator's `create-weblogic-domain.sh` script, or manual, if you have an existing WebLogic domain configuration. +This document describes how to set up and start an Apache HTTP Server for load balancing inside a Kubernets cluster. The configuration and startup can be either automatic, when you create a domain using the WebLogic Operator's `create-weblogic-domain.sh` script, or manual, if you have an existing WebLogic domain configuration. -## Build the Docker image for the Apache web server +## Build the Docker image for the Apache HTTP Server -You need to build the Docker image for the Apache web server that embeds the Oracle WebLogic Server Proxy Plugin. +You need to build the Docker image for the Apache HTTP Server that embeds the Oracle WebLogic Server Proxy Plugin. - 1. Download and build the Docker image for the Apache web server with the 12.2.1.3.0 Oracle WebLogic Server Proxy Plugin. See the instructions in [Apache Web Server with Oracle WebLogic Server Proxy Plugin on Docker](https://github.com/oracle/docker-images/tree/master/OracleWebLogic/samples/12213-webtier-apache). + 1. Download and build the Docker image for the Apache HTTP Server with the 12.2.1.3.0 Oracle WebLogic Server Proxy Plugin. See the instructions in [Apache HTTP Server with Oracle WebLogic Server Proxy Plugin on Docker](https://github.com/oracle/docker-images/tree/master/OracleWebLogic/samples/12213-webtier-apache). 2. Tag your Docker image, `store/oracle/apache:12.2.1.3`, using the `docker tag` command. @@ -17,16 +17,16 @@ You need to build the Docker image for the Apache web server that embeds the Ora ``` -For more information about the Apache plugin, see [Apache Web Server with Oracle WebLogic Server Proxy Plugin on Docker](https://docs.oracle.com/middleware/1213/webtier/develop-plugin/apache.htm#PLGWL395). +For more information about the Apache plugin, see [Apache HTTP Server with Oracle WebLogic Server Proxy Plugin on Docker](https://docs.oracle.com/middleware/1213/webtier/develop-plugin/apache.htm#PLGWL395). -After you have access to the Docker image of the Apache web server, you can follow the instructions below to set up and start the Kubernetes artifacts for the Apache web server. +After you have access to the Docker image of the Apache HTTP Server, you can follow the instructions below to set up and start the Kubernetes artifacts for the Apache HTTP Server. ## Use the Apache load balancer with a WebLogic domain created with the WebLogic Operator For how to create a domain with the WebLogic Operator, please refer to [Creating a domain using the WebLogic Operator](creating-domain.md). -You need to configure the Apache web server as your load balancer for a WebLogic domain by setting the `loadBalancer` option to `APACHE` in the `create-weblogic-domain-inputs.yaml` (as shown below) when running the `create-weblogic-domain.sh` script to create a domain. +You need to configure the Apache HTTP Server as your load balancer for a WebLogic domain by setting the `loadBalancer` option to `APACHE` in the `create-weblogic-domain-inputs.yaml` (as shown below) when running the `create-weblogic-domain.sh` script to create a domain. ``` @@ -36,9 +36,9 @@ loadBalancer: APACHE ``` -The `create-weblogic-domain.sh` script installs the Apache web server with the Oracle WebLogic Server Proxy Plugin into the Kubernetes *cluster* in the same namespace as the *domain*. +The `create-weblogic-domain.sh` script installs the Apache HTTP Server with the Oracle WebLogic Server Proxy Plugin into the Kubernetes *cluster* in the same namespace as the *domain*. -The Apache web server will expose a `NodePort` that allows access to the load balancer from outside of the Kubernetes cluster. The port is configured by setting `loadBalancerWebPort` in the `create-weblogic-domain-inputs.yaml` file. +The Apache HTTP Server will expose a `NodePort` that allows access to the load balancer from outside of the Kubernetes cluster. The port is configured by setting `loadBalancerWebPort` in the `create-weblogic-domain-inputs.yaml` file. ``` @@ -66,7 +66,7 @@ Users can then access an application from outside of the Kubernetes cluster by u The generated Kubernetes YAML files look like the following, given the `domainUID`, "`domain1`". -Sample `weblogic-domain-apache.yaml` file for Apache web server deployment. +Sample `weblogic-domain-apache.yaml` file for Apache HTTP Server deployment. ``` @@ -440,13 +440,13 @@ The generated YAML files will look similar except with un-commented entries like ## Use the Apache load balancer with a manually created WebLogic Domain -If your WebLogic domain is not created by the WebLogic Operator, you need to manually create and start all Kubernetes' artifacts for the Apache web server. +If your WebLogic domain is not created by the WebLogic Operator, you need to manually create and start all Kubernetes' artifacts for the Apache HTTP Server. 1. Create your own `custom_mod_wl_apache.conf` file, and put it in a local directory, for example, ``. See the instructions in [Apache Web Server with Oracle WebLogic Server Proxy Plugin on Docker](https://docs.oracle.com/middleware/1213/webtier/develop-plugin/apache.htm#PLGWL395). - 2. Create the Apache deployment YAML file. See the example above. Note that you need to use the **volumes** and **volumeMounts** to mount `` into the `/config` directory inside the pod that runs the Apache web tier. Note that the Apache web server needs to be in the same Kubernetes namespace as the WebLogic domain that it needs to access. + 2. Create the Apache deployment YAML file. See the example above. Note that you need to use the **volumes** and **volumeMounts** to mount `` into the `/config` directory inside the pod that runs the Apache web tier. Note that the Apache HTTP Server needs to be in the same Kubernetes namespace as the WebLogic domain that it needs to access. 3. Create a RBAC YAML file. See the example above. -Note that you can choose to run one Apache web server to balance the loads from multiple domains/clusters inside the same Kubernetes cluster, as long as the Apache web server and the domains are all in the same namespace. +Note that you can choose to run one Apache HTTP Server to balance the loads from multiple domains/clusters inside the same Kubernetes cluster, as long as the Apache HTTP Server and the domains are all in the same namespace. From aa6c5b1711545e098228f009a3ca1739b6272b7d Mon Sep 17 00:00:00 2001 From: Rosemary Marano Date: Mon, 30 Apr 2018 16:49:21 -0400 Subject: [PATCH 138/186] changed Apache web server to Apache HTTP Server --- site/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/installation.md b/site/installation.md index 3f682f3178f..0ae945228f4 100644 --- a/site/installation.md +++ b/site/installation.md @@ -101,7 +101,7 @@ The operator provides some optional features that can be enabled in the configur ### Load balancing with an Ingress controller or a web server -You can choose a load balancer provider for your WebLogic domains running in a Kubernetes cluster. Please refer to Load balancing with Voyager Ingress Controller, [Load balancing with Traefik Ingress Controller](site/traefik.md), and [Load balancing with Apache Web Server](site/apache.md) for information about the current capabilities and setup instructions for each of the supported load balancers. +You can choose a load balancer provider for your WebLogic domains running in a Kubernetes cluster. Please refer to Load balancing with Voyager Ingress Controller, [Load balancing with Traefik Ingress Controller](site/traefik.md), and [Load balancing with the Apache HTTP Server](site/apache.md) for information about the current capabilities and setup instructions for each of the supported load balancers. Note these limitations: From c391b80e8842d1e249f0c321e4dd7fb3cf1aef9d Mon Sep 17 00:00:00 2001 From: Rosemary Marano Date: Mon, 30 Apr 2018 16:51:36 -0400 Subject: [PATCH 139/186] changed Apache web server to Apache HTTP Server --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cfd487f2e4f..a51befa06ad 100644 --- a/README.md +++ b/README.md @@ -161,7 +161,7 @@ Please refer to [Shutting down a domain](site/shutdown-domain.md) for informatio ## Load balancing with an Ingress controller or a web server -You can choose a load balancer provider for your WebLogic domains running in a Kubernetes cluster. Please refer to Load balancing with Voyager Ingress Controller, [Load balancing with Traefik Ingress Controller](site/traefik.md), and [Load balancing with Apache Web Server](site/apache.md) for information about the current capabilities and setup instructions for each of the supported load balancers. +You can choose a load balancer provider for your WebLogic domains running in a Kubernetes cluster. Please refer to Load balancing with Voyager Ingress Controller, [Load balancing with Traefik Ingress Controller](site/traefik.md), and [Load balancing with Apache HTTP Server](site/apache.md) for information about the current capabilities and setup instructions for each of the supported load balancers. [comment]: # (Exporting operator logs to ELK. The operator provides an option to export its log files to the ELK stack. Please refer to [ELK integration]site/elk.md for information about this capability.) From 295c044e6311bdc9a6a911511304ececd9269857 Mon Sep 17 00:00:00 2001 From: Rosemary Marano Date: Mon, 30 Apr 2018 16:56:11 -0400 Subject: [PATCH 140/186] fix typo --- site/rbac.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/rbac.md b/site/rbac.md index 60e448410cb..f31f7d2771b 100644 --- a/site/rbac.md +++ b/site/rbac.md @@ -12,7 +12,7 @@ The general design goal is to provide the operator with the minimum amount of pe | | customresourcedefinitions in API group apiextensions.k8s.io | get, list, watch, create, update, patch, delete, deletecollection | | | | domains in API group weblogic.oracle | get, list, watch, update, patch | | | | Ingresses in API group extensions | get, list, watch, create, update, patch, delete, deletecollection | | -| `weblogic-operator-cluster-role-nonresourc`e | nonResourceURLs: ["/version/*"] | get | 1 | +| `weblogic-operator-cluster-role-nonresource` | nonResourceURLs: ["/version/*"] | get | 1 | |`weblogic-operator-namespace-role` | secrets, persistentvolumeclaims | get, list, watch | 2 | | | services, pods, networkpolicies | get, list, watch, create, update, patch, delete, deletecollection | | | `NAMESPACE-operator-rolebinding-discovery` | system:discovery in API group rbac.authorization.k8s.io | | 1 | From 9cb072f923bcd7fe16f92e62f4bd597e5691f08e Mon Sep 17 00:00:00 2001 From: Russell Gold Date: Mon, 30 Apr 2018 23:26:32 -0400 Subject: [PATCH 141/186] Clean up watcher test intermittent failures --- .../kubernetes/operator/builders/WatchI.java | 8 +++-- .../kubernetes/operator/PodWatcherTest.java | 19 +++++----- .../kubernetes/operator/WatcherTestBase.java | 35 ++++++++++++++++++- .../operator/builders/StubWatchFactory.java | 28 +++++++++++---- 4 files changed, 69 insertions(+), 21 deletions(-) diff --git a/operator/src/main/java/oracle/kubernetes/operator/builders/WatchI.java b/operator/src/main/java/oracle/kubernetes/operator/builders/WatchI.java index 5e37c07084b..ffa73434ff3 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/builders/WatchI.java +++ b/operator/src/main/java/oracle/kubernetes/operator/builders/WatchI.java @@ -3,12 +3,14 @@ package oracle.kubernetes.operator.builders; -import io.kubernetes.client.util.Watch; - import java.util.Iterator; +import io.kubernetes.client.util.Watch; + /** - * An interface that allows test-stubbing of the Kubernetes Watch class. + * An iterator over watch responses from the server. These objects maintain resources, + * which will be release when #close() is called. + * * @param the generic object type */ public interface WatchI diff --git a/operator/src/test/java/oracle/kubernetes/operator/PodWatcherTest.java b/operator/src/test/java/oracle/kubernetes/operator/PodWatcherTest.java index 23249a281ca..026bc37af19 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/PodWatcherTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/PodWatcherTest.java @@ -3,6 +3,12 @@ package oracle.kubernetes.operator; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +import com.google.common.collect.ImmutableMap; + import io.kubernetes.client.models.V1ObjectMeta; import io.kubernetes.client.models.V1Pod; import io.kubernetes.client.models.V1PodCondition; @@ -14,13 +20,6 @@ import oracle.kubernetes.operator.work.Packet; import oracle.kubernetes.operator.work.Step; -import com.google.common.collect.ImmutableMap; - -import java.util.Collections; -import java.util.List; -import java.util.concurrent.Executors; -import java.util.concurrent.atomic.AtomicBoolean; - import org.hamcrest.Matchers; import org.junit.Test; @@ -74,7 +73,7 @@ protected T createObjectWithMetaData(V1ObjectMeta metaData) { @Override protected PodWatcher createWatcher(String nameSpace, AtomicBoolean stopping, int initialResourceVersion) { - return PodWatcher.create(Executors.defaultThreadFactory(), nameSpace, + return PodWatcher.create(this, nameSpace, Integer.toString(initialResourceVersion), this, stopping); } @@ -171,7 +170,7 @@ public void whenPodHasServerName_returnIt() throws Exception { @Test public void waitForReady_returnsAStep() throws Exception { AtomicBoolean stopping = new AtomicBoolean(true); - PodWatcher watcher = PodWatcher.create(Executors.defaultThreadFactory(), "ns", + PodWatcher watcher = PodWatcher.create(this, "ns", Integer.toString(INITIAL_RESOURCE_VERSION), this, stopping); assertThat(watcher.waitForReady(pod, null), Matchers.instanceOf(Step.class)); @@ -180,7 +179,7 @@ public void waitForReady_returnsAStep() throws Exception { @Test public void WhenWaitForReadyAppliedToReadyPod_performNextStep() throws Exception { AtomicBoolean stopping = new AtomicBoolean(false); - PodWatcher watcher = PodWatcher.create(Executors.defaultThreadFactory(), "ns", + PodWatcher watcher = PodWatcher.create(this, "ns", Integer.toString(INITIAL_RESOURCE_VERSION), this, stopping); makePodReady(pod); diff --git a/operator/src/test/java/oracle/kubernetes/operator/WatcherTestBase.java b/operator/src/test/java/oracle/kubernetes/operator/WatcherTestBase.java index 59484c0ea4b..944263eba74 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/WatcherTestBase.java +++ b/operator/src/test/java/oracle/kubernetes/operator/WatcherTestBase.java @@ -5,6 +5,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.concurrent.ThreadFactory; import java.util.concurrent.atomic.AtomicBoolean; import com.meterware.simplestub.Memento; @@ -17,7 +18,11 @@ import org.junit.After; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TestRule; +import org.junit.rules.TestWatcher; +import org.junit.runner.Description; import static java.net.HttpURLConnection.HTTP_GONE; import static oracle.kubernetes.operator.builders.EventMatcher.addEvent; @@ -30,7 +35,7 @@ /** * Tests behavior of the Watcher class. */ -public abstract class WatcherTestBase implements StubWatchFactory.AllWatchesClosedListener { +public abstract class WatcherTestBase implements StubWatchFactory.AllWatchesClosedListener, ThreadFactory { private static final int NEXT_RESOURCE_VERSION = 123456; private static final int INITIAL_RESOURCE_VERSION = 123; private static final String NAMESPACE = "testspace"; @@ -47,6 +52,25 @@ private V1ObjectMeta createMetaData() { private AtomicBoolean stopping = new AtomicBoolean(false); + private String testName; + private List threads = new ArrayList<>(); + + @Rule + public TestRule watcher = new TestWatcher() { + @Override + protected void starting(Description description) { + testName = description.getMethodName(); + } + }; + + @Override + public Thread newThread(Runnable r) { + Thread thread = new Thread(r); + threads.add(thread); + thread.setName(String.format("Test thread %d for %s", threads.size(), testName)); + return thread; + } + @Override public void allWatchesClosed() { stopping.set(true); @@ -65,9 +89,18 @@ public void setUp() throws Exception { @After public void tearDown() throws Exception { + for (Thread thread : threads) shutDown(thread); for (Memento memento : mementos) memento.revert(); } + private void shutDown(Thread thread) { + try { + thread.interrupt(); + thread.join(); + } catch (InterruptedException ignored) { + } + } + @SuppressWarnings("unchecked") void sendInitialRequest(int initialResourceVersion) { StubWatchFactory.addCallResponses(createAddResponse(createObjectWithMetaData())); diff --git a/operator/src/test/java/oracle/kubernetes/operator/builders/StubWatchFactory.java b/operator/src/test/java/oracle/kubernetes/operator/builders/StubWatchFactory.java index 79332341450..436eaab8005 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/builders/StubWatchFactory.java +++ b/operator/src/test/java/oracle/kubernetes/operator/builders/StubWatchFactory.java @@ -6,6 +6,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -22,6 +23,8 @@ import io.kubernetes.client.util.Watch.Response; import oracle.kubernetes.operator.helpers.Pool; +import javax.annotation.Nonnull; + /** * A test-time replacement for the factory that creates Watch objects, allowing * tests to specify directly the events they want returned from the Watch. @@ -69,15 +72,26 @@ public static List> getRecordedParameters() { public WatchI createWatch(Pool pool, CallParams callParams, Class responseBodyType, BiFunction function) throws ApiException { getRecordedParameters().add(recordedParams(callParams)); - if (exceptionOnNext == null) - return new WatchStub((List)calls.remove(0)); - else try { - return new ExceptionThrowingWatchStub(exceptionOnNext); - } finally { - exceptionOnNext = null; + try { + if (nothingToDo()) + return new WatchStub<>(Collections.emptyList()); + else if (exceptionOnNext == null) + return new WatchStub((List)calls.remove(0)); + else try { + return new ExceptionThrowingWatchStub<>(exceptionOnNext); + } finally { + exceptionOnNext = null; + } + } catch (IndexOutOfBoundsException e) { + System.out.println("Failed in thread " + Thread.currentThread()); + throw e; } } + public boolean nothingToDo() { + return calls.isEmpty() && exceptionOnNext == null; + } + private Map recordedParams(CallParams callParams) { Map result = new HashMap<>(); if (callParams.getResourceVersion() != null) @@ -116,7 +130,7 @@ public void close() throws IOException { } @Override - public Iterator> iterator() { + public @Nonnull Iterator> iterator() { return responses.iterator(); } From 5a41ef94127805586fbb21848cd455c4ecb95de9 Mon Sep 17 00:00:00 2001 From: Rosemary Marano Date: Tue, 1 May 2018 08:12:27 -0400 Subject: [PATCH 142/186] minor text edits --- site/rest.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/site/rest.md b/site/rest.md index 6cd40c7de7b..31e610ab37b 100644 --- a/site/rest.md +++ b/site/rest.md @@ -2,16 +2,16 @@ The operator provides a REST server which can be used to get a list of WebLogic domains and clusters and to initiate scaling operations. Swagger documentation for the REST API is available [here](https://oracle.github.io/weblogic-kubernetes-operator/swagger/index.html). -Most of the services access a `GET`, for example: +You can access most of the REST services using `GET`, for example: * To obtain a list of domains, send a `GET` request to the URL `/operator/latest/domains`. * To obtain a list of clusters in a domain, send a `GET` request to the URL `/operator/latest/domains//clusters`. All of the REST services require authentication. Callers must pass in a valid token header and a CA certificate file. The `X-Requested-By` header is not required. Callers should pass in the `Accept:/application/json` header. -If using `curl`, the `-k` option can be used to bypass the check to verify that the operator's certificate is trusted (instead of `curl --cacert`). +If using `curl`, you can use the `-k` option to bypass the check to verify that the operator's certificate is trusted (instead of `curl --cacert`). -Here is a small BASH script that may help to prepare the necessary token, certificates, and so on, to call the operator's REST services: +Here is a small BASH script that may help to prepare the necessary token, certificates, and such, to call the operator's REST services: ``` #!/bin/bash From 994de44809c51886dc8dd8129f672cbba0d830e8 Mon Sep 17 00:00:00 2001 From: Tom Moreau Date: Tue, 1 May 2018 10:48:19 -0400 Subject: [PATCH 143/186] Add the constants necessary to update the unit tests, finish writing the input file version tests --- kubernetes/internal/voyager-ingress-template.yaml | 2 ++ .../oracle/kubernetes/operator/LabelConstants.java | 1 + .../kubernetes/operator/VersionConstants.java | 14 ++++++++++++++ .../create/CreateDomainInputsFileTest.java | 2 ++ .../create/CreateOperatorInputsFileTest.java | 2 ++ .../create/CreateOperatorInputsValidationTest.java | 4 +--- 6 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 operator/src/main/java/oracle/kubernetes/operator/VersionConstants.java diff --git a/kubernetes/internal/voyager-ingress-template.yaml b/kubernetes/internal/voyager-ingress-template.yaml index 4f9bed4e5e6..d63a136bb08 100644 --- a/kubernetes/internal/voyager-ingress-template.yaml +++ b/kubernetes/internal/voyager-ingress-template.yaml @@ -4,6 +4,7 @@ metadata: name: %DOMAIN_UID%-voyager namespace: %NAMESPACE% labels: + weblogic.resourceVersion: voyager-load-balancer/v1 weblogic.domainUID: %DOMAIN_UID% weblogic.domainName: %DOMAIN_NAME% annotations: @@ -27,6 +28,7 @@ metadata: namespace: %NAMESPACE% labels: app: %DOMAIN_UID%-voyager-stats + weblogic.resourceVersion: voyager-load-balancer/v1 weblogic.domainUID: %DOMAIN_UID% weblogic.domainName: %DOMAIN_NAME% spec: diff --git a/operator/src/main/java/oracle/kubernetes/operator/LabelConstants.java b/operator/src/main/java/oracle/kubernetes/operator/LabelConstants.java index 5cd0be0441f..830e1cd1bb8 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/LabelConstants.java +++ b/operator/src/main/java/oracle/kubernetes/operator/LabelConstants.java @@ -5,6 +5,7 @@ public interface LabelConstants { + public static final String RESOURCE_VERSION_LABEL = "weblogic.resourceVersion"; public static final String DOMAINUID_LABEL = "weblogic.domainUID"; public static final String DOMAINNAME_LABEL = "weblogic.domainName"; public static final String SERVERNAME_LABEL = "weblogic.serverName"; diff --git a/operator/src/main/java/oracle/kubernetes/operator/VersionConstants.java b/operator/src/main/java/oracle/kubernetes/operator/VersionConstants.java new file mode 100644 index 00000000000..45c631a90a3 --- /dev/null +++ b/operator/src/main/java/oracle/kubernetes/operator/VersionConstants.java @@ -0,0 +1,14 @@ +// Copyright 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator; + +public interface VersionConstants { + public static final String CREATE_WEBLOGIC_OPERATOR_INPUTS_V1 = "create-weblogic-operator-inputs/v1"; + public static final String CREATE_WEBLOGIC_DOMAIN_INPUTS_V1 = "create-weblogic-domain-inputs/v1"; + public static final String OPERATOR_V1 = "operator/v1"; + public static final String DOMAIN_V1 = "domain/v1"; + public static final String VOYAGER_LOAD_BALANCER_V1 = "voyager-load-balancer/v1"; + public static final String APACHE_LOAD_BALANCER_V1 = "apache-load-balancer/v1"; + public static final String TRAEFIK_LOAD_BALANCER_V1 = "traefik-load-balancer/v1"; +} diff --git a/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputsFileTest.java b/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputsFileTest.java index 0231e274469..2be900d73e8 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputsFileTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainInputsFileTest.java @@ -10,6 +10,7 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import static oracle.kubernetes.operator.VersionConstants.*; import static oracle.kubernetes.operator.create.CreateDomainInputs.*; import static oracle.kubernetes.operator.create.ExecCreateDomain.*; import static oracle.kubernetes.operator.create.ExecResultMatcher.*; @@ -43,6 +44,7 @@ public void defaultInputsFile_hasCorrectContents() throws Exception { assertThat( readDefaultInputsFile(), yamlEqualTo((new CreateDomainInputs()) + .version(CREATE_WEBLOGIC_DOMAIN_INPUTS_V1) .adminNodePort("30701") .adminPort("7001") .adminServerName("admin-server") diff --git a/operator/src/test/java/oracle/kubernetes/operator/create/CreateOperatorInputsFileTest.java b/operator/src/test/java/oracle/kubernetes/operator/create/CreateOperatorInputsFileTest.java index 152c21577c7..8ca22f6d196 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/create/CreateOperatorInputsFileTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/create/CreateOperatorInputsFileTest.java @@ -11,6 +11,7 @@ import java.nio.file.Path; import java.util.List; +import static oracle.kubernetes.operator.VersionConstants.*; import static oracle.kubernetes.operator.create.CreateOperatorInputs.*; import static oracle.kubernetes.operator.create.ExecCreateOperator.execCreateOperator; import static oracle.kubernetes.operator.create.ExecResultMatcher.succeedsAndPrints; @@ -46,6 +47,7 @@ public void defaultInputsFile_hasCorrectContents() throws Exception { assertThat( readDefaultInputsFile(), yamlEqualTo((new CreateOperatorInputs()) + .version(CREATE_WEBLOGIC_OPERATOR_INPUTS_V1) .elkIntegrationEnabled("false") .externalDebugHttpPort("30999") .externalOperatorCert("") diff --git a/operator/src/test/java/oracle/kubernetes/operator/create/CreateOperatorInputsValidationTest.java b/operator/src/test/java/oracle/kubernetes/operator/create/CreateOperatorInputsValidationTest.java index e7438fa5725..27e6a79426e 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/create/CreateOperatorInputsValidationTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/create/CreateOperatorInputsValidationTest.java @@ -7,11 +7,11 @@ import org.junit.Before; import org.junit.Test; +import static oracle.kubernetes.operator.create.CreateOperatorInputs.*; import static oracle.kubernetes.operator.create.ExecResultMatcher.errorRegexp; import static oracle.kubernetes.operator.create.ExecResultMatcher.failsAndPrints; import static org.hamcrest.MatcherAssert.assertThat; -import static oracle.kubernetes.operator.create.CreateOperatorInputs.*; /** * Tests that create-weblogic-operator.sh properly validates the parameters @@ -51,8 +51,6 @@ public void tearDown() throws Exception { private static final String PARAM_JAVA_LOGGING_LEVEL = "javaLoggingLevel"; private static final String PARAM_ELK_INTEGRATION_ENABLED = "elkIntegrationEnabled"; - private static final String VERSION_V1 = "create-weblogic-operator-inputs/v1"; - @Test public void createOperator_with_missingVersion_failsAndReturnsError() throws Exception { assertThat( From a99d7808a4aadd6c8ec2f94a4e957d7e287048bd Mon Sep 17 00:00:00 2001 From: Rosemary Marano Date: Tue, 1 May 2018 11:35:53 -0400 Subject: [PATCH 144/186] minor edits; removed limitations note --- site/scaling.md | 1 - 1 file changed, 1 deletion(-) diff --git a/site/scaling.md b/site/scaling.md index f3783452d80..093b25b672b 100644 --- a/site/scaling.md +++ b/site/scaling.md @@ -6,7 +6,6 @@ The operator provides the ability to scale WebLogic clusters by simply editing t Scaling up or scaling down a WebLogic cluster provides increased reliability of customer applications as well as optimization of resource usage. In Kubernetes cloud environments, scaling WebLogic clusters involves scaling the corresponding pods in which WebLogic Managed Server instances are running. Because the operator manages the life cycle of a WebLogic domain, the operator exposes a REST API that allows an authorized actor to request scaling of a WebLogic cluster. -**Note:** In the technology preview release, only WebLogic Server configured clusters are supported by the operator, and the operator will scale up only to the number of Managed Servers that are already defined. Support for WebLogic Server dynamic clusters, and for scaling configured clusters to more servers than are defined, is planned for a future release. The following URL format is used for describing the resources for scaling (scale up and scale down) a WebLogic cluster: From f8a39659494db0f9eaaf19dea2e70af5b9e55cdd Mon Sep 17 00:00:00 2001 From: Tom Moreau Date: Tue, 1 May 2018 11:41:11 -0400 Subject: [PATCH 145/186] Version the domain and load balancer resources --- .../create-weblogic-domain-job-template.yaml | 2 + .../domain-custom-resource-template.yaml | 1 + ...logic-domain-apache-security-template.yaml | 2 + .../weblogic-domain-apache-template.yaml | 3 ++ .../internal/weblogic-domain-pv-template.yaml | 1 + .../weblogic-domain-pvc-template.yaml | 1 + ...ogic-domain-traefik-security-template.yaml | 2 + .../weblogic-domain-traefik-template.yaml | 5 +++ .../operator/helpers/ConfigMapHelper.java | 2 + .../operator/helpers/PodHelper.java | 3 ++ .../create/ConfigMapHelperConfigTest.java | 2 + .../CreateDomainGeneratedFilesBaseTest.java | 38 ++++++++++++++----- .../operator/create/PodHelperConfigTest.java | 2 + 13 files changed, 54 insertions(+), 10 deletions(-) diff --git a/kubernetes/internal/create-weblogic-domain-job-template.yaml b/kubernetes/internal/create-weblogic-domain-job-template.yaml index 86efcdf8827..e262dc934fd 100644 --- a/kubernetes/internal/create-weblogic-domain-job-template.yaml +++ b/kubernetes/internal/create-weblogic-domain-job-template.yaml @@ -6,6 +6,7 @@ metadata: name: %DOMAIN_UID%-create-weblogic-domain-job-cm namespace: %NAMESPACE% labels: + weblogic.resourceVersion: domain/v1 weblogic.domainUID: %DOMAIN_UID% weblogic.domainName: %DOMAIN_NAME% data: @@ -266,6 +267,7 @@ spec: template: metadata: labels: + weblogic.resourceVersion: domain/v1 weblogic.domainUID: %DOMAIN_UID% weblogic.domainName: %DOMAIN_NAME% app: %DOMAIN_UID%-create-weblogic-domain-job diff --git a/kubernetes/internal/domain-custom-resource-template.yaml b/kubernetes/internal/domain-custom-resource-template.yaml index ffdb5d1e5af..79227bd4626 100644 --- a/kubernetes/internal/domain-custom-resource-template.yaml +++ b/kubernetes/internal/domain-custom-resource-template.yaml @@ -9,6 +9,7 @@ metadata: name: %DOMAIN_UID% namespace: %NAMESPACE% labels: + weblogic.resourceVersion: domain/v1 weblogic.domainUID: %DOMAIN_UID% weblogic.domainName: %DOMAIN_NAME% spec: diff --git a/kubernetes/internal/weblogic-domain-apache-security-template.yaml b/kubernetes/internal/weblogic-domain-apache-security-template.yaml index 30fa8200a42..761d4c5ce23 100644 --- a/kubernetes/internal/weblogic-domain-apache-security-template.yaml +++ b/kubernetes/internal/weblogic-domain-apache-security-template.yaml @@ -4,6 +4,7 @@ apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: %DOMAIN_UID%-apache-webtier labels: + weblogic.resourceVersion: apache-load-balancer/v1 weblogic.domainUID: %DOMAIN_UID% weblogic.domainName: %DOMAIN_NAME% rules: @@ -32,6 +33,7 @@ apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: %DOMAIN_UID%-apache-webtier labels: + weblogic.resourceVersion: apache-load-balancer/v1 weblogic.domainUID: %DOMAIN_UID% weblogic.domainName: %DOMAIN_NAME% roleRef: diff --git a/kubernetes/internal/weblogic-domain-apache-template.yaml b/kubernetes/internal/weblogic-domain-apache-template.yaml index dedfd79ea85..2b6b9a0d12d 100755 --- a/kubernetes/internal/weblogic-domain-apache-template.yaml +++ b/kubernetes/internal/weblogic-domain-apache-template.yaml @@ -5,6 +5,7 @@ metadata: name: %DOMAIN_UID%-apache-webtier namespace: %NAMESPACE% labels: + weblogic.resourceVersion: apache-load-balancer/v1 weblogic.domainUID: %DOMAIN_UID% weblogic.domainName: %DOMAIN_NAME% app: apache-webtier @@ -15,6 +16,7 @@ metadata: name: %DOMAIN_UID%-apache-webtier namespace: %NAMESPACE% labels: + weblogic.resourceVersion: apache-load-balancer/v1 weblogic.domainUID: %DOMAIN_UID% weblogic.domainName: %DOMAIN_NAME% app: apache-webtier @@ -78,6 +80,7 @@ metadata: name: %DOMAIN_UID%-apache-webtier namespace: %NAMESPACE% labels: + weblogic.resourceVersion: apache-load-balancer/v1 weblogic.domainUID: %DOMAIN_UID% weblogic.domainName: %DOMAIN_NAME% spec: diff --git a/kubernetes/internal/weblogic-domain-pv-template.yaml b/kubernetes/internal/weblogic-domain-pv-template.yaml index 9b1d8038f6a..8345d18362a 100644 --- a/kubernetes/internal/weblogic-domain-pv-template.yaml +++ b/kubernetes/internal/weblogic-domain-pv-template.yaml @@ -6,6 +6,7 @@ kind: PersistentVolume metadata: name: %DOMAIN_UID%-weblogic-domain-pv labels: + weblogic.resourceVersion: domain/v1 weblogic.domainUID: %DOMAIN_UID% weblogic.domainName: %DOMAIN_NAME% spec: diff --git a/kubernetes/internal/weblogic-domain-pvc-template.yaml b/kubernetes/internal/weblogic-domain-pvc-template.yaml index 7bf7855f16f..b73aa93b481 100644 --- a/kubernetes/internal/weblogic-domain-pvc-template.yaml +++ b/kubernetes/internal/weblogic-domain-pvc-template.yaml @@ -7,6 +7,7 @@ metadata: name: %DOMAIN_UID%-weblogic-domain-pvc namespace: %NAMESPACE% labels: + weblogic.resourceVersion: domain/v1 weblogic.domainUID: %DOMAIN_UID% weblogic.domainName: %DOMAIN_NAME% spec: diff --git a/kubernetes/internal/weblogic-domain-traefik-security-template.yaml b/kubernetes/internal/weblogic-domain-traefik-security-template.yaml index f42dc2e3cbe..7c9f72e26b9 100644 --- a/kubernetes/internal/weblogic-domain-traefik-security-template.yaml +++ b/kubernetes/internal/weblogic-domain-traefik-security-template.yaml @@ -4,6 +4,7 @@ apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: %DOMAIN_UID%-%CLUSTER_NAME_LC%-traefik labels: + weblogic.resourceVersion: traefik-load-balancer/v1 weblogic.domainUID: %DOMAIN_UID% weblogic.domainName: %DOMAIN_NAME% weblogic.clusterName: %CLUSTER_NAME% @@ -33,6 +34,7 @@ apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: %DOMAIN_UID%-%CLUSTER_NAME_LC%-traefik labels: + weblogic.resourceVersion: traefik-load-balancer/v1 weblogic.domainUID: %DOMAIN_UID% weblogic.domainName: %DOMAIN_NAME% weblogic.clusterName: %CLUSTER_NAME% diff --git a/kubernetes/internal/weblogic-domain-traefik-template.yaml b/kubernetes/internal/weblogic-domain-traefik-template.yaml index d307ddd7e81..d8a91a751c1 100644 --- a/kubernetes/internal/weblogic-domain-traefik-template.yaml +++ b/kubernetes/internal/weblogic-domain-traefik-template.yaml @@ -5,6 +5,7 @@ metadata: name: %DOMAIN_UID%-%CLUSTER_NAME_LC%-traefik namespace: %NAMESPACE% labels: + weblogic.resourceVersion: traefik-load-balancer/v1 weblogic.domainUID: %DOMAIN_UID% weblogic.domainName: %DOMAIN_NAME% weblogic.clusterName: %CLUSTER_NAME% @@ -15,6 +16,7 @@ metadata: name: %DOMAIN_UID%-%CLUSTER_NAME_LC%-traefik namespace: %NAMESPACE% labels: + weblogic.resourceVersion: traefik-load-balancer/v1 weblogic.domainUID: %DOMAIN_UID% weblogic.domainName: %DOMAIN_NAME% weblogic.clusterName: %CLUSTER_NAME% @@ -82,6 +84,7 @@ metadata: name: %DOMAIN_UID%-%CLUSTER_NAME_LC%-traefik-cm namespace: %NAMESPACE% labels: + weblogic.resourceVersion: traefik-load-balancer/v1 weblogic.domainUID: %DOMAIN_UID% weblogic.domainName: %DOMAIN_NAME% weblogic.clusterName: %CLUSTER_NAME% @@ -105,6 +108,7 @@ metadata: name: %DOMAIN_UID%-%CLUSTER_NAME_LC%-traefik namespace: %NAMESPACE% labels: + weblogic.resourceVersion: traefik-load-balancer/v1 weblogic.domainUID: %DOMAIN_UID% weblogic.domainName: %DOMAIN_NAME% weblogic.clusterName: %CLUSTER_NAME% @@ -125,6 +129,7 @@ metadata: name: %DOMAIN_UID%-%CLUSTER_NAME_LC%-traefik-dashboard namespace: %NAMESPACE% labels: + weblogic.resourceVersion: traefik-load-balancer/v1 weblogic.domainUID: %DOMAIN_UID% weblogic.domainName: %DOMAIN_NAME% weblogic.clusterName: %CLUSTER_NAME% diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java index a7e0d2fe95e..af8607984e0 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java @@ -24,6 +24,7 @@ import io.kubernetes.client.models.V1ObjectMeta; import oracle.kubernetes.operator.KubernetesConstants; import oracle.kubernetes.operator.LabelConstants; +import oracle.kubernetes.operator.VersionConstants; import oracle.kubernetes.operator.ProcessingConstants; import oracle.kubernetes.operator.logging.LoggingFacade; import oracle.kubernetes.operator.logging.LoggingFactory; @@ -145,6 +146,7 @@ protected V1ConfigMap computeDomainConfigMap() { AnnotationHelper.annotateWithFormat(metadata); Map labels = new HashMap<>(); + labels.put(LabelConstants.RESOURCE_VERSION_LABEL, VersionConstants.DOMAIN_V1); labels.put(LabelConstants.OPERATORNAME_LABEL, operatorNamespace); labels.put(LabelConstants.CREATEDBYOPERATOR_LABEL, "true"); metadata.setLabels(labels); diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java index d37faae90d1..d7e626d68f7 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java @@ -24,6 +24,7 @@ import oracle.kubernetes.operator.DomainStatusUpdater; import oracle.kubernetes.operator.KubernetesConstants; import oracle.kubernetes.operator.LabelConstants; +import oracle.kubernetes.operator.VersionConstants; import oracle.kubernetes.operator.PodWatcher; import oracle.kubernetes.operator.ProcessingConstants; import oracle.kubernetes.operator.TuningParameters; @@ -188,6 +189,7 @@ protected V1Pod computeAdminPodConfig(TuningParameters configMapHelper, Packet p AnnotationHelper.annotateForPrometheus(metadata, spec.getAsPort()); Map labels = new HashMap<>(); + labels.put(LabelConstants.RESOURCE_VERSION_LABEL, VersionConstants.DOMAIN_V1); labels.put(LabelConstants.DOMAINUID_LABEL, weblogicDomainUID); labels.put(LabelConstants.DOMAINNAME_LABEL, weblogicDomainName); labels.put(LabelConstants.SERVERNAME_LABEL, spec.getAsName()); @@ -615,6 +617,7 @@ protected V1Pod computeManagedPodConfig(TuningParameters configMapHelper, Packet AnnotationHelper.annotateForPrometheus(metadata, scan.getListenPort()); Map labels = new HashMap<>(); + labels.put(LabelConstants.RESOURCE_VERSION_LABEL, VersionConstants.DOMAIN_V1); labels.put(LabelConstants.DOMAINUID_LABEL, weblogicDomainUID); labels.put(LabelConstants.DOMAINNAME_LABEL, weblogicDomainName); labels.put(LabelConstants.SERVERNAME_LABEL, weblogicServerName); diff --git a/operator/src/test/java/oracle/kubernetes/operator/create/ConfigMapHelperConfigTest.java b/operator/src/test/java/oracle/kubernetes/operator/create/ConfigMapHelperConfigTest.java index 6d4aa54c124..2294d700350 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/create/ConfigMapHelperConfigTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/create/ConfigMapHelperConfigTest.java @@ -6,6 +6,7 @@ import io.kubernetes.client.models.V1ConfigMap; import static oracle.kubernetes.operator.KubernetesConstants.*; import static oracle.kubernetes.operator.LabelConstants.*; +import static oracle.kubernetes.operator.VersionConstants.*; import static oracle.kubernetes.operator.create.KubernetesArtifactUtils.*; import static oracle.kubernetes.operator.create.YamlUtils.*; import static oracle.kubernetes.operator.helpers.AnnotationHelper.*; @@ -55,6 +56,7 @@ private V1ConfigMap getDesiredDomainConfigMap() { .metadata(newObjectMeta() .name(DOMAIN_CONFIG_MAP_NAME) .namespace(DOMAIN_NAMESPACE) + .putLabelsItem(RESOURCE_VERSION_LABEL, DOMAIN_V1) .putLabelsItem(OPERATORNAME_LABEL, OPERATOR_NAMESPACE) .putLabelsItem(CREATEDBYOPERATOR_LABEL, "true") .putAnnotationsItem(FORMAT_ANNOTATION, FORMAT_VERSION)) diff --git a/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainGeneratedFilesBaseTest.java b/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainGeneratedFilesBaseTest.java index 1cb3465e049..6686b97b8ac 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainGeneratedFilesBaseTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainGeneratedFilesBaseTest.java @@ -9,6 +9,7 @@ import io.kubernetes.client.models.*; import static oracle.kubernetes.operator.LabelConstants.*; +import static oracle.kubernetes.operator.VersionConstants.*; import static oracle.kubernetes.operator.create.CreateDomainInputs.readInputsYamlFile; import static oracle.kubernetes.operator.create.KubernetesArtifactUtils.*; import static oracle.kubernetes.operator.create.YamlUtils.yamlEqualTo; @@ -156,6 +157,7 @@ protected V1Job getExpectedCreateWeblogicDomainJob() { .spec(newJobSpec() .template(newPodTemplateSpec() .metadata(newObjectMeta() + .putLabelsItem(RESOURCE_VERSION_LABEL, DOMAIN_V1) .putLabelsItem(DOMAINUID_LABEL, getInputs().getDomainUID()) .putLabelsItem(DOMAINNAME_LABEL, getInputs().getDomainName()) .putLabelsItem(APP_LABEL, getInputs().getDomainUID() + "-create-weblogic-domain-job")) @@ -236,6 +238,7 @@ protected V1ConfigMap getExpectedCreateWeblogicDomainJobConfigMap() { .metadata(newObjectMeta() .name(getInputs().getDomainUID() + "-create-weblogic-domain-job-cm") .namespace(getInputs().getNamespace()) + .putLabelsItem(RESOURCE_VERSION_LABEL, DOMAIN_V1) .putLabelsItem(DOMAINUID_LABEL, getInputs().getDomainUID()) .putLabelsItem(DOMAINNAME_LABEL, getInputs().getDomainName())) .putDataItem(PROPERTY_UTILITY_SH, "") @@ -355,6 +358,7 @@ protected Domain getExpectedDomain() { newObjectMeta() .name(getInputs().getDomainUID()) .namespace(getInputs().getNamespace()) + .putLabelsItem(RESOURCE_VERSION_LABEL, DOMAIN_V1) .putLabelsItem(DOMAINUID_LABEL, getInputs().getDomainUID()) .putLabelsItem(DOMAINNAME_LABEL, getInputs().getDomainName())) .withSpec(newDomainSpec() @@ -409,6 +413,7 @@ protected V1ServiceAccount getExpectedApacheServiceAccount() { .metadata(newObjectMeta() .name(getApacheName()) .namespace(getInputs().getNamespace()) + .putLabelsItem(RESOURCE_VERSION_LABEL, APACHE_LOAD_BALANCER_V1) .putLabelsItem(DOMAINUID_LABEL, getInputs().getDomainUID()) .putLabelsItem(DOMAINNAME_LABEL, getInputs().getDomainName()) .putLabelsItem(APP_LABEL, getApacheAppName())); @@ -425,6 +430,7 @@ protected V1ServiceAccount getExpectedTraefikServiceAccount() { .metadata(newObjectMeta() .name(getTraefikScope()) .namespace(getInputs().getNamespace()) + .putLabelsItem(RESOURCE_VERSION_LABEL, TRAEFIK_LOAD_BALANCER_V1) .putLabelsItem(DOMAINUID_LABEL, getInputs().getDomainUID()) .putLabelsItem(DOMAINNAME_LABEL, getInputs().getDomainName()) .putLabelsItem(CLUSTERNAME_LABEL, getInputs().getClusterName())); @@ -448,6 +454,7 @@ protected ExtensionsV1beta1Deployment getExpectedApacheDeployment() { .metadata(newObjectMeta() .name(getApacheName()) .namespace(getInputs().getNamespace()) + .putLabelsItem(RESOURCE_VERSION_LABEL, APACHE_LOAD_BALANCER_V1) .putLabelsItem(DOMAINUID_LABEL, getInputs().getDomainUID()) .putLabelsItem(DOMAINNAME_LABEL, getInputs().getDomainName()) .putLabelsItem(APP_LABEL, getApacheAppName())) @@ -514,6 +521,7 @@ protected ExtensionsV1beta1Deployment getExpectedTraefikDeployment() { .metadata(newObjectMeta() .name(getTraefikScope()) .namespace(getInputs().getNamespace()) + .putLabelsItem(RESOURCE_VERSION_LABEL, TRAEFIK_LOAD_BALANCER_V1) .putLabelsItem(DOMAINUID_LABEL, getInputs().getDomainUID()) .putLabelsItem(DOMAINNAME_LABEL, getInputs().getDomainName()) .putLabelsItem(CLUSTERNAME_LABEL, getInputs().getClusterName())) @@ -599,6 +607,7 @@ protected V1ConfigMap getExpectedTraefikConfigMap() { .metadata(newObjectMeta() .name(getTraefikScope() + "-cm") .namespace(getInputs().getNamespace()) + .putLabelsItem(RESOURCE_VERSION_LABEL, TRAEFIK_LOAD_BALANCER_V1) .putLabelsItem(DOMAINUID_LABEL, getInputs().getDomainUID()) .putLabelsItem(DOMAINNAME_LABEL, getInputs().getDomainName()) .putLabelsItem(CLUSTERNAME_LABEL, getInputs().getClusterName())) @@ -628,6 +637,7 @@ protected V1Service getExpectedApacheService() { .metadata(newObjectMeta() .name(getApacheName()) .namespace(getInputs().getNamespace()) + .putLabelsItem(RESOURCE_VERSION_LABEL, APACHE_LOAD_BALANCER_V1) .putLabelsItem(DOMAINUID_LABEL, getInputs().getDomainUID()) .putLabelsItem(DOMAINNAME_LABEL, getInputs().getDomainName())) .spec(newServiceSpec() @@ -651,6 +661,7 @@ protected V1Service getExpectedTraefikService() { .metadata(newObjectMeta() .name(getTraefikScope()) .namespace(getInputs().getNamespace()) + .putLabelsItem(RESOURCE_VERSION_LABEL, TRAEFIK_LOAD_BALANCER_V1) .putLabelsItem(DOMAINUID_LABEL, getInputs().getDomainUID()) .putLabelsItem(DOMAINNAME_LABEL, getInputs().getDomainName()) .putLabelsItem(CLUSTERNAME_LABEL, getInputs().getClusterName())) @@ -682,6 +693,7 @@ protected V1Service getExpectedTraefikDashboardService() { .metadata(newObjectMeta() .name(getTraefikScope() + "-dashboard") .namespace(getInputs().getNamespace()) + .putLabelsItem(RESOURCE_VERSION_LABEL, TRAEFIK_LOAD_BALANCER_V1) .putLabelsItem(DOMAINUID_LABEL, getInputs().getDomainUID()) .putLabelsItem(DOMAINNAME_LABEL, getInputs().getDomainName()) .putLabelsItem(CLUSTERNAME_LABEL, getInputs().getClusterName())) @@ -712,6 +724,7 @@ protected V1beta1ClusterRole getExpectedApacheClusterRole() { newClusterRole() .metadata(newObjectMeta() .name(getApacheName()) + .putLabelsItem(RESOURCE_VERSION_LABEL, APACHE_LOAD_BALANCER_V1) .putLabelsItem(DOMAINUID_LABEL, getInputs().getDomainUID()) .putLabelsItem(DOMAINNAME_LABEL, getInputs().getDomainName())) .addRulesItem(newPolicyRule() @@ -733,6 +746,7 @@ protected V1beta1ClusterRole getExpectedTraefikClusterRole() { newClusterRole() .metadata(newObjectMeta() .name(getTraefikScope()) + .putLabelsItem(RESOURCE_VERSION_LABEL, TRAEFIK_LOAD_BALANCER_V1) .putLabelsItem(DOMAINUID_LABEL, getInputs().getDomainUID()) .putLabelsItem(DOMAINNAME_LABEL, getInputs().getDomainName()) .putLabelsItem(CLUSTERNAME_LABEL, getInputs().getClusterName())) @@ -762,6 +776,7 @@ protected V1beta1ClusterRoleBinding getExpectedApacheClusterRoleBinding() { newClusterRoleBinding() .metadata(newObjectMeta() .name(getApacheName()) + .putLabelsItem(RESOURCE_VERSION_LABEL, APACHE_LOAD_BALANCER_V1) .putLabelsItem(DOMAINUID_LABEL, getInputs().getDomainUID()) .putLabelsItem(DOMAINNAME_LABEL, getInputs().getDomainName())) .addSubjectsItem(newSubject() @@ -782,6 +797,7 @@ protected V1beta1ClusterRoleBinding getExpectedTraefikDashboardClusterRoleBindin newClusterRoleBinding() .metadata(newObjectMeta() .name(getTraefikScope()) + .putLabelsItem(RESOURCE_VERSION_LABEL, TRAEFIK_LOAD_BALANCER_V1) .putLabelsItem(DOMAINUID_LABEL, getInputs().getDomainUID()) .putLabelsItem(DOMAINNAME_LABEL, getInputs().getDomainName()) .putLabelsItem(CLUSTERNAME_LABEL, getInputs().getClusterName())) @@ -807,16 +823,17 @@ protected V1PersistentVolume getActualWeblogicDomainPersistentVolume() { protected V1PersistentVolume getExpectedWeblogicDomainPersistentVolume() { return - newPersistentVolume() - .metadata(newObjectMeta() - .name(getInputs().getWeblogicDomainPersistentVolumeName()) - .putLabelsItem(DOMAINUID_LABEL, getInputs().getDomainUID()) - .putLabelsItem(DOMAINNAME_LABEL, getInputs().getDomainName())) - .spec(newPersistentVolumeSpec() - .storageClassName(getInputs().getWeblogicDomainStorageClass()) - .putCapacityItem("storage", Quantity.fromString(getInputs().getWeblogicDomainStorageSize())) - .addAccessModesItem("ReadWriteMany") - .persistentVolumeReclaimPolicy("Retain")); + newPersistentVolume() + .metadata(newObjectMeta() + .name(getInputs().getWeblogicDomainPersistentVolumeName()) + .putLabelsItem(RESOURCE_VERSION_LABEL, DOMAIN_V1) + .putLabelsItem(DOMAINUID_LABEL, getInputs().getDomainUID()) + .putLabelsItem(DOMAINNAME_LABEL, getInputs().getDomainName())) + .spec(newPersistentVolumeSpec() + .storageClassName(getInputs().getWeblogicDomainStorageClass()) + .putCapacityItem("storage", Quantity.fromString(getInputs().getWeblogicDomainStorageSize())) + .addAccessModesItem("ReadWriteMany") + .persistentVolumeReclaimPolicy("Retain")); } @Test @@ -836,6 +853,7 @@ protected V1PersistentVolumeClaim getExpectedWeblogicDomainPersistentVolumeClaim .metadata(newObjectMeta() .name(getInputs().getWeblogicDomainPersistentVolumeClaimName()) .namespace(getInputs().getNamespace()) + .putLabelsItem(RESOURCE_VERSION_LABEL, DOMAIN_V1) .putLabelsItem(DOMAINUID_LABEL, getInputs().getDomainUID()) .putLabelsItem(DOMAINNAME_LABEL, getInputs().getDomainName())) .spec(newPersistentVolumeClaimSpec() diff --git a/operator/src/test/java/oracle/kubernetes/operator/create/PodHelperConfigTest.java b/operator/src/test/java/oracle/kubernetes/operator/create/PodHelperConfigTest.java index a2ddbc1a458..ca54f994832 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/create/PodHelperConfigTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/create/PodHelperConfigTest.java @@ -11,6 +11,7 @@ import io.kubernetes.client.models.V1Pod; import static oracle.kubernetes.operator.KubernetesConstants.*; import static oracle.kubernetes.operator.LabelConstants.*; +import static oracle.kubernetes.operator.VersionConstants.*; import oracle.kubernetes.operator.ProcessingConstants; import oracle.kubernetes.operator.TuningParameters; import oracle.kubernetes.operator.TuningParameters.PodTuning; @@ -426,6 +427,7 @@ private V1Pod getDesiredBaseServerPodConfigForDefaults(String image, String imag .putAnnotationsItem("prometheus.io/port", "" + port) .putAnnotationsItem("prometheus.io/scrape", "true") .putAnnotationsItem("weblogic.oracle/operator-formatVersion", "1") + .putLabelsItem(RESOURCE_VERSION_LABEL, DOMAIN_V1) .putLabelsItem(CREATEDBYOPERATOR_LABEL, "true") .putLabelsItem(DOMAINNAME_LABEL, DOMAIN_NAME) .putLabelsItem(DOMAINUID_LABEL, DOMAIN_UID) From e91c5039852b48ef02ec30947c5cc9402ea418e8 Mon Sep 17 00:00:00 2001 From: Rosemary Marano Date: Tue, 1 May 2018 11:57:52 -0400 Subject: [PATCH 146/186] added v1.0 support, as per Monica --- site/recent-changes.md | 1 + 1 file changed, 1 insertion(+) diff --git a/site/recent-changes.md b/site/recent-changes.md index a2b91e096f7..2d5a1c7224f 100644 --- a/site/recent-changes.md +++ b/site/recent-changes.md @@ -6,3 +6,4 @@ This document tracks recent changes to the operator, especially ones that introd | --- | --- | --- | | March 20, 2018 | yes | See [Name Changes](name-changes.md). Several files and input parameters have been renamed. This affects how operators and domains are created. It also changes generated Kubernetes artifacts, therefore customers must recreate their operators and domains. | April 4, 2018 | yes | See [Name Changes](name-changes.md). Many Kubernetes artifact names and labels have changed. Also, the names of generated YAML files for creating a domain's PV and PVC have changed. Because of these changes, customers must recreate their operators and domains. +| May 7, 2018 | no | Added support for dynamic clusters, the Apache HTTP Server, the Voyager Ingress Controller, and for PV in NFS storage for multi-node environments. From 7b9e00a5f82d6c9cbec981d047c5d8e44c0b4a2a Mon Sep 17 00:00:00 2001 From: Tom Moreau Date: Tue, 1 May 2018 12:08:49 -0400 Subject: [PATCH 147/186] Version the operator related resources, also version the deployment templates so that the pods get versioned too --- .../internal/generate-security-policy.sh | 10 ++++++++++ .../weblogic-domain-apache-template.yaml | 1 + .../weblogic-domain-traefik-template.yaml | 1 + .../internal/weblogic-operator-template.yaml | 7 ++++++- .../CreateDomainGeneratedFilesBaseTest.java | 2 ++ .../CreateOperatorGeneratedFilesBaseTest.java | 18 ++++++++++++++++++ 6 files changed, 38 insertions(+), 1 deletion(-) diff --git a/kubernetes/internal/generate-security-policy.sh b/kubernetes/internal/generate-security-policy.sh index beb90170e82..47fb781139c 100755 --- a/kubernetes/internal/generate-security-policy.sh +++ b/kubernetes/internal/generate-security-policy.sh @@ -67,6 +67,7 @@ kind: Namespace metadata: name: ${NAMESPACE} labels: + weblogic.resourceVersion: operator/v1 weblogic.operatorName: ${NAMESPACE} --- # @@ -78,6 +79,7 @@ metadata: namespace: ${NAMESPACE} name: ${ACCOUNT_NAME} labels: + weblogic.resourceVersion: operator/v1 weblogic.operatorName: ${NAMESPACE} --- EOF @@ -91,6 +93,7 @@ apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: weblogic-operator-cluster-role labels: + weblogic.resourceVersion: operator/v1 weblogic.operatorName: ${NAMESPACE} rules: - apiGroups: [""] @@ -123,6 +126,7 @@ apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: weblogic-operator-cluster-role-nonresource labels: + weblogic.resourceVersion: operator/v1 weblogic.operatorName: ${NAMESPACE} rules: - nonResourceURLs: ["/version/*"] @@ -136,6 +140,7 @@ apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: ${NAMESPACE}-operator-rolebinding labels: + weblogic.resourceVersion: operator/v1 weblogic.operatorName: ${NAMESPACE} subjects: - kind: ServiceAccount @@ -152,6 +157,7 @@ apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: ${NAMESPACE}-operator-rolebinding-nonresource labels: + weblogic.resourceVersion: operator/v1 weblogic.operatorName: ${NAMESPACE} subjects: - kind: ServiceAccount @@ -168,6 +174,7 @@ apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: ${NAMESPACE}-operator-rolebinding-discovery labels: + weblogic.resourceVersion: operator/v1 weblogic.operatorName: ${NAMESPACE} subjects: - kind: ServiceAccount @@ -184,6 +191,7 @@ apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: ${NAMESPACE}-operator-rolebinding-auth-delegator labels: + weblogic.resourceVersion: operator/v1 weblogic.operatorName: ${NAMESPACE} subjects: - kind: ServiceAccount @@ -203,6 +211,7 @@ apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: weblogic-operator-namespace-role labels: + weblogic.resourceVersion: operator/v1 weblogic.operatorName: ${NAMESPACE} rules: - apiGroups: [""] @@ -245,6 +254,7 @@ metadata: name: weblogic-operator-rolebinding namespace: ${i} labels: + weblogic.resourceVersion: operator/v1 weblogic.operatorName: ${NAMESPACE} subjects: - kind: ServiceAccount diff --git a/kubernetes/internal/weblogic-domain-apache-template.yaml b/kubernetes/internal/weblogic-domain-apache-template.yaml index 2b6b9a0d12d..d2fe2a0d575 100755 --- a/kubernetes/internal/weblogic-domain-apache-template.yaml +++ b/kubernetes/internal/weblogic-domain-apache-template.yaml @@ -30,6 +30,7 @@ spec: template: metadata: labels: + weblogic.resourceVersion: apache-load-balancer/v1 weblogic.domainUID: %DOMAIN_UID% weblogic.domainName: %DOMAIN_NAME% app: apache-webtier diff --git a/kubernetes/internal/weblogic-domain-traefik-template.yaml b/kubernetes/internal/weblogic-domain-traefik-template.yaml index d8a91a751c1..d1013d34721 100644 --- a/kubernetes/internal/weblogic-domain-traefik-template.yaml +++ b/kubernetes/internal/weblogic-domain-traefik-template.yaml @@ -29,6 +29,7 @@ spec: template: metadata: labels: + weblogic.resourceVersion: traefik-load-balancer/v1 weblogic.domainUID: %DOMAIN_UID% weblogic.domainName: %DOMAIN_NAME% weblogic.clusterName: %CLUSTER_NAME% diff --git a/kubernetes/internal/weblogic-operator-template.yaml b/kubernetes/internal/weblogic-operator-template.yaml index ebd9500f1e4..8736051dd79 100644 --- a/kubernetes/internal/weblogic-operator-template.yaml +++ b/kubernetes/internal/weblogic-operator-template.yaml @@ -14,6 +14,7 @@ metadata: name: weblogic-operator-cm namespace: %NAMESPACE% labels: + weblogic.resourceVersion: operator/v1 weblogic.operatorName: %NAMESPACE% data: serviceaccount: %ACCOUNT_NAME% @@ -28,6 +29,7 @@ metadata: name: weblogic-operator-secrets namespace: %NAMESPACE% labels: + weblogic.resourceVersion: operator/v1 weblogic.operatorName: %NAMESPACE% type: Opaque data: @@ -42,12 +44,14 @@ metadata: # set the namespace that you want the operator deployed in here namespace: %NAMESPACE% labels: + weblogic.resourceVersion: operator/v1 weblogic.operatorName: %NAMESPACE% spec: replicas: 1 template: metadata: labels: + weblogic.resourceVersion: operator/v1 weblogic.operatorName: %NAMESPACE% app: weblogic-operator spec: @@ -129,6 +133,7 @@ spec: %EXTERNAL_OPERATOR_SERVICE_PREFIX% name: external-weblogic-operator-svc %EXTERNAL_OPERATOR_SERVICE_PREFIX% namespace: %NAMESPACE% %EXTERNAL_OPERATOR_SERVICE_PREFIX% labels: +%EXTERNAL_OPERATOR_SERVICE_PREFIX% weblogic.resourceVersion: operator/v1 %EXTERNAL_OPERATOR_SERVICE_PREFIX% weblogic.operatorName: %NAMESPACE% %EXTERNAL_OPERATOR_SERVICE_PREFIX%spec: %EXTERNAL_OPERATOR_SERVICE_PREFIX% type: NodePort @@ -151,6 +156,7 @@ metadata: name: internal-weblogic-operator-svc namespace: %NAMESPACE% labels: + weblogic.resourceVersion: operator/v1 weblogic.operatorName: %NAMESPACE% spec: type: ClusterIP @@ -159,4 +165,3 @@ spec: ports: - port: 8082 name: rest - diff --git a/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainGeneratedFilesBaseTest.java b/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainGeneratedFilesBaseTest.java index 6686b97b8ac..8d27b3e58a6 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainGeneratedFilesBaseTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainGeneratedFilesBaseTest.java @@ -466,6 +466,7 @@ protected ExtensionsV1beta1Deployment getExpectedApacheDeployment() { .putMatchLabelsItem(APP_LABEL, getApacheAppName())) .template(newPodTemplateSpec() .metadata(newObjectMeta() + .putLabelsItem(RESOURCE_VERSION_LABEL, APACHE_LOAD_BALANCER_V1) .putLabelsItem(DOMAINUID_LABEL, getInputs().getDomainUID()) .putLabelsItem(DOMAINNAME_LABEL, getInputs().getDomainName()) .putLabelsItem(APP_LABEL, getApacheAppName())) @@ -532,6 +533,7 @@ protected ExtensionsV1beta1Deployment getExpectedTraefikDeployment() { .putMatchLabelsItem(CLUSTERNAME_LABEL, getInputs().getClusterName())) .template(newPodTemplateSpec() .metadata(newObjectMeta() + .putLabelsItem(RESOURCE_VERSION_LABEL, TRAEFIK_LOAD_BALANCER_V1) .putLabelsItem(DOMAINUID_LABEL, getInputs().getDomainUID()) .putLabelsItem(DOMAINNAME_LABEL, getInputs().getDomainName()) .putLabelsItem(CLUSTERNAME_LABEL, getInputs().getClusterName())) diff --git a/operator/src/test/java/oracle/kubernetes/operator/create/CreateOperatorGeneratedFilesBaseTest.java b/operator/src/test/java/oracle/kubernetes/operator/create/CreateOperatorGeneratedFilesBaseTest.java index f7b367ca8f4..7c81b395282 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/create/CreateOperatorGeneratedFilesBaseTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/create/CreateOperatorGeneratedFilesBaseTest.java @@ -10,6 +10,7 @@ import static java.util.Arrays.asList; import static oracle.kubernetes.operator.LabelConstants.*; +import static oracle.kubernetes.operator.VersionConstants.*; import static oracle.kubernetes.operator.create.CreateOperatorInputs.readInputsYamlFile; import static oracle.kubernetes.operator.create.KubernetesArtifactUtils.*; import static oracle.kubernetes.operator.create.YamlUtils.yamlEqualTo; @@ -92,6 +93,7 @@ protected V1ConfigMap getExpectedWeblogicOperatorConfigMap() { .metadata(newObjectMeta() .name("weblogic-operator-cm") .namespace(getInputs().getNamespace()) + .putLabelsItem(RESOURCE_VERSION_LABEL, OPERATOR_V1) .putLabelsItem(OPERATORNAME_LABEL, getInputs().getNamespace())) .putDataItem("serviceaccount", getInputs().getServiceAccount()) .putDataItem("targetNamespaces", getInputs().getTargetNamespaces()) @@ -118,6 +120,7 @@ protected V1Secret getExpectedWeblogicOperatorSecrets() { .metadata(newObjectMeta() .name("weblogic-operator-secrets") .namespace(getInputs().getNamespace()) + .putLabelsItem(RESOURCE_VERSION_LABEL, OPERATOR_V1) .putLabelsItem(OPERATORNAME_LABEL, getInputs().getNamespace())) .type("Opaque") .putDataItem("externalOperatorKey", getExpectedExternalWeblogicOperatorKey().getBytes()) @@ -143,11 +146,13 @@ protected ExtensionsV1beta1Deployment getExpectedWeblogicOperatorDeployment() { .metadata(newObjectMeta() .name("weblogic-operator") .namespace(getInputs().getNamespace()) + .putLabelsItem(RESOURCE_VERSION_LABEL, OPERATOR_V1) .putLabelsItem(OPERATORNAME_LABEL, getInputs().getNamespace())) .spec(newDeploymentSpec() .replicas(1) .template(newPodTemplateSpec() .metadata(newObjectMeta() + .putLabelsItem(RESOURCE_VERSION_LABEL, OPERATOR_V1) .putLabelsItem(OPERATORNAME_LABEL, getInputs().getNamespace()) .putLabelsItem(APP_LABEL, "weblogic-operator")) .spec(newPodSpec() @@ -233,6 +238,7 @@ protected V1Service getExpectedExternalWeblogicOperatorService(boolean debugging .metadata(newObjectMeta() .name("external-weblogic-operator-svc") .namespace(getInputs().getNamespace()) + .putLabelsItem(RESOURCE_VERSION_LABEL, OPERATOR_V1) .putLabelsItem(OPERATORNAME_LABEL, getInputs().getNamespace())) .spec(spec); } @@ -254,6 +260,7 @@ protected V1Service getExpectedInternalWeblogicOperatorService() { .metadata(newObjectMeta() .name("internal-weblogic-operator-svc") .namespace(getInputs().getNamespace()) + .putLabelsItem(RESOURCE_VERSION_LABEL, OPERATOR_V1) .putLabelsItem(OPERATORNAME_LABEL, getInputs().getNamespace())) .spec(newServiceSpec() .type("ClusterIP") @@ -279,6 +286,7 @@ protected V1Namespace getExpectedWeblogicOperatorNamespace() { newNamespace() .metadata(newObjectMeta() .name(getInputs().getNamespace()) + .putLabelsItem(RESOURCE_VERSION_LABEL, OPERATOR_V1) .putLabelsItem(OPERATORNAME_LABEL, getInputs().getNamespace())); } @@ -299,6 +307,7 @@ protected V1ServiceAccount getExpectedWeblogicOperatorServiceAccount() { .metadata(newObjectMeta() .name(getInputs().getServiceAccount()) .namespace(getInputs().getNamespace()) + .putLabelsItem(RESOURCE_VERSION_LABEL, OPERATOR_V1) .putLabelsItem(OPERATORNAME_LABEL, getInputs().getNamespace())); } @@ -318,6 +327,7 @@ protected V1beta1ClusterRole getExpectedWeblogicOperatorClusterRole() { newClusterRole() .metadata(newObjectMeta() .name("weblogic-operator-cluster-role") + .putLabelsItem(RESOURCE_VERSION_LABEL, OPERATOR_V1) .putLabelsItem(OPERATORNAME_LABEL, getInputs().getNamespace())) .addRulesItem(newPolicyRule() .addApiGroupsItem("") @@ -369,6 +379,7 @@ protected V1beta1ClusterRole getExpectedWeblogicOperatorClusterRoleNonResource() newClusterRole() .metadata(newObjectMeta() .name("weblogic-operator-cluster-role-nonresource") + .putLabelsItem(RESOURCE_VERSION_LABEL, OPERATOR_V1) .putLabelsItem(OPERATORNAME_LABEL, getInputs().getNamespace())) .addRulesItem(newPolicyRule() .addNonResourceURLsItem("/version/*") @@ -383,6 +394,7 @@ public void generatesCorrect_operatorRoleBinding() throws Exception { newClusterRoleBinding() .metadata(newObjectMeta() .name(getInputs().getNamespace() + "-operator-rolebinding") + .putLabelsItem(RESOURCE_VERSION_LABEL, OPERATOR_V1) .putLabelsItem(OPERATORNAME_LABEL, getInputs().getNamespace())) .addSubjectsItem(newSubject() .kind("ServiceAccount") @@ -410,6 +422,7 @@ protected V1beta1ClusterRoleBinding getExpectedOperatorRoleBindingNonResource() newClusterRoleBinding() .metadata(newObjectMeta() .name(getInputs().getNamespace() + "-operator-rolebinding-nonresource") + .putLabelsItem(RESOURCE_VERSION_LABEL, OPERATOR_V1) .putLabelsItem(OPERATORNAME_LABEL, getInputs().getNamespace())) .addSubjectsItem(newSubject() .kind("ServiceAccount") @@ -437,6 +450,7 @@ protected V1beta1ClusterRoleBinding getExpectedOperatorRoleBindingDiscovery() { newClusterRoleBinding() .metadata(newObjectMeta() .name(getInputs().getNamespace() + "-operator-rolebinding-discovery") + .putLabelsItem(RESOURCE_VERSION_LABEL, OPERATOR_V1) .putLabelsItem(OPERATORNAME_LABEL, getInputs().getNamespace())) .addSubjectsItem(newSubject() .kind("ServiceAccount") @@ -464,6 +478,7 @@ protected V1beta1ClusterRoleBinding getExpectedOperatorRoleBindingAuthDelegator( newClusterRoleBinding() .metadata(newObjectMeta() .name(getInputs().getNamespace() + "-operator-rolebinding-auth-delegator") + .putLabelsItem(RESOURCE_VERSION_LABEL, OPERATOR_V1) .putLabelsItem(OPERATORNAME_LABEL, getInputs().getNamespace())) .addSubjectsItem(newSubject() .kind("ServiceAccount") @@ -491,6 +506,7 @@ protected V1beta1ClusterRole getExpectedWeblogicOperatorNamespaceRole() { newClusterRole() .metadata(newObjectMeta() .name("weblogic-operator-namespace-role") + .putLabelsItem(RESOURCE_VERSION_LABEL, OPERATOR_V1) .putLabelsItem(OPERATORNAME_LABEL, getInputs().getNamespace())) .addRulesItem(newPolicyRule() .addApiGroupsItem("") @@ -546,6 +562,7 @@ protected V1beta1RoleBinding getExpectedWeblogicOperatorRoleBinding(String names .metadata(newObjectMeta() .name("weblogic-operator-rolebinding") .namespace(namespace) + .putLabelsItem(RESOURCE_VERSION_LABEL, OPERATOR_V1) .putLabelsItem(OPERATORNAME_LABEL, getInputs().getNamespace())) .addSubjectsItem(newSubject() .kind("ServiceAccount") @@ -578,6 +595,7 @@ protected V1Service getExpectedExternalOperatorService(boolean debuggingEnabled, .metadata(newObjectMeta() .name("external-weblogic-operator-svc") .namespace(inputs.getNamespace()) + .putLabelsItem(RESOURCE_VERSION_LABEL, OPERATOR_V1) .putLabelsItem(OPERATORNAME_LABEL, getInputs().getNamespace())) .spec(spec); } From 7306a3c5f0edb11059e608f5bf032025e3afda3b Mon Sep 17 00:00:00 2001 From: Rosemary Marano Date: Tue, 1 May 2018 12:15:38 -0400 Subject: [PATCH 148/186] minor edits --- site/starting-domain.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/site/starting-domain.md b/site/starting-domain.md index 191dbd7df74..52b4589bbf6 100644 --- a/site/starting-domain.md +++ b/site/starting-domain.md @@ -1,6 +1,6 @@ # Starting a WebLogic domain -Startup of the domain is controlled by settings in the domain custom resource. The domain creation job, if used, will have created a domain custom resource YAML file. If the domain was created manually, this YAML file will also need to be created manually. +Startup of the domain is controlled by settings in the domain custom resource. If used, the domain creation job will have created a domain custom resource YAML file. If the domain was created manually, this YAML file will also need to be created manually. An example of the domain custom resource YAML file is shown below: @@ -96,10 +96,10 @@ The operator determines which servers to start using the following logic: * If `startupControl` is set to `SPECIFIED`, then: * The Administration Server will be started. - * Each server listed in a `serverStartup` section will be brought up to state that is specified in the `desiredState` in that section, `RUNNING` or `ADMIN`. - * For each cluster listed in a `clusterStartup` section, a number of servers in that cluster equal to the `replicas` setting will be brought up to state that is specified in the `desiredState` in that section, `RUNNING` or `ADMIN`. If `replicas` is not specified in `clusterStartup`, then the top-level `replicas` field in the domain custom resource will be used instead. + * Each server listed in a `serverStartup` section will be brought up to the state that is specified in the `desiredState` in that section, `RUNNING` or `ADMIN`. + * For each cluster listed in a `clusterStartup` section, a number of servers in that cluster, equal to the `replicas` setting, will be brought up to the state that is specified in the `desiredState` in that section, `RUNNING` or `ADMIN`. If `replicas` is not specified in `clusterStartup`, then the top-level `replicas` field in the domain custom resource will be used instead. * If `startupControl` is set to `AUTO`, then: * The operator will perform as if `startupControl` were set to `SPECIFIED`. - * For all clusters that do not have a `clusterStartup` section, a number of servers in that cluster equal to the top-level `replicas` setting will be brought up to `RUNNING` state. + * For all clusters that do not have a `clusterStartup` section, the number of servers in that cluster equal to the top-level `replicas` setting, will be brought up to the `RUNNING` state. From 18eb0e9704642439bb5d3b12533403ce4ba9b25e Mon Sep 17 00:00:00 2001 From: Rosemary Marano Date: Tue, 1 May 2018 12:24:14 -0400 Subject: [PATCH 149/186] minor edits --- site/traefik.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/site/traefik.md b/site/traefik.md index 996c8328ce6..80e74060953 100644 --- a/site/traefik.md +++ b/site/traefik.md @@ -1,6 +1,6 @@ # Load balancing with Traefik -If the `loadBalancer` option is set to `traefik` when running the `create-weblogic-domain.sh` script to create a WebLogic domain in Kubernetes, then the Traefik Ingress Controller will be installed into the *cluster* and an *ingress* will be created for each WebLogic *cluster* in the *domain*. +If the `loadBalancer` option is set to `traefik` when running the `create-weblogic-domain.sh` script to create a WebLogic domain in Kubernetes, then the Traefik Ingress Controller will be installed into the cluster and an Ingress will be created for each WebLogic cluster in the domain. More information about the Traefik Ingress controller can be found at: [https://docs.traefik.io/user-guide/kubernetes/](https://docs.traefik.io/user-guide/kubernetes/) @@ -13,7 +13,7 @@ loadBalancerWebPort: 30305 # Load balancer dashboard port loadBalancerDashboardPort: 30315 ``` -The operator will automatically update the Ingress to ensure that it contains a list of only those pods that are "ready". Here is an example of what the Ingress might look like for a WebLogic cluster called `cluster-1`, in a domain called `base_domain` with `domainUID domain1` that has three Managed Servers in the "ready" state: +The operator will automatically update the Ingress to ensure that it contains a list of only those pods that are "ready". Here is an example of what the Ingress might look like for a WebLogic cluster called `cluster-1`, in a domain called `base_domain`, with `domainUID domain1`, that has three Managed Servers in the "ready" state: ``` apiVersion: extensions/v1beta1 @@ -46,4 +46,4 @@ spec: path: / ``` -Notice that currently the only supported type of load balancing is using the root path ("`/`"). As such, there is one instance of Traefik for each WebLogic cluster. Please take a look at our [wish list](https://github.com/oracle/weblogic-kubernetes-operator/wiki/Wish-list) to get an idea of the load balancing improvements we would like to do. +Notice that currently the only supported type of load balancing is using the root path ("`/`"). As such, there is one instance of Traefik for each WebLogic cluster. From edba0fb5d8eff7519aa9574bee2042c0900ba55b Mon Sep 17 00:00:00 2001 From: Tom Barnes Date: Tue, 1 May 2018 09:44:30 -0700 Subject: [PATCH 150/186] Avoid exposing admin host and port by default via Apache. --- .../weblogic-domain-apache-template.yaml | 8 ++--- .../CreateDomainGeneratedFilesBaseTest.java | 12 ++++---- site/apache.md | 30 +++++++++++-------- 3 files changed, 27 insertions(+), 23 deletions(-) diff --git a/kubernetes/internal/weblogic-domain-apache-template.yaml b/kubernetes/internal/weblogic-domain-apache-template.yaml index dedfd79ea85..4e24c7dcd8f 100755 --- a/kubernetes/internal/weblogic-domain-apache-template.yaml +++ b/kubernetes/internal/weblogic-domain-apache-template.yaml @@ -50,10 +50,10 @@ spec: value: '%DOMAIN_UID%-cluster-%CLUSTER_NAME_LC%:%MANAGED_SERVER_PORT%' - name: LOCATION value: '%WEB_APP_PREPATH%' - - name: WEBLOGIC_HOST - value: '%DOMAIN_UID%-%ADMIN_SERVER_NAME%' - - name: WEBLOGIC_PORT - value: '%ADMIN_PORT%' + #- name: WEBLOGIC_HOST + # value: '%DOMAIN_UID%-%ADMIN_SERVER_NAME%' + #- name: WEBLOGIC_PORT + # value: '%ADMIN_PORT%' readinessProbe: tcpSocket: port: 80 diff --git a/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainGeneratedFilesBaseTest.java b/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainGeneratedFilesBaseTest.java index 1cb3465e049..8eda6d29e84 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainGeneratedFilesBaseTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/create/CreateDomainGeneratedFilesBaseTest.java @@ -475,12 +475,12 @@ protected ExtensionsV1beta1Deployment getExpectedApacheDeployment() { .addEnvItem(newEnvVar() .name("LOCATION") .value(getInputs().getLoadBalancerAppPrepath())) - .addEnvItem(newEnvVar() - .name("WEBLOGIC_HOST") - .value(getInputs().getDomainUID() + "-" + getInputs().getAdminServerName())) - .addEnvItem(newEnvVar() - .name("WEBLOGIC_PORT") - .value(getInputs().getAdminPort())) + // .addEnvItem(newEnvVar() + // .name("WEBLOGIC_HOST") + // .value(getInputs().getDomainUID() + "-" + getInputs().getAdminServerName())) + // .addEnvItem(newEnvVar() + // .name("WEBLOGIC_PORT") + // .value(getInputs().getAdminPort())) .readinessProbe(newProbe() .tcpSocket(newTCPSocketAction() .port(newIntOrString(80))) diff --git a/site/apache.md b/site/apache.md index 7b7f79a04c8..386aec66124 100644 --- a/site/apache.md +++ b/site/apache.md @@ -1,15 +1,15 @@ # Load Balancing with Apache Web Server -This page describes how to setup and start a Apache Web Server for load balancing inside a Kubernets cluster. The configuration and startup can either be automatic when you create a domain using the WebLogic Operator's `create-weblogic-domain.sh` script, or manually if you have an existing WebLogic domain configuration. +This page describes how to setup and start a Apache Web Server for load balancing inside a Kubernetes cluster. The configuration and startup can either be automatic when you create a domain using the WebLogic Operator's `create-weblogic-domain.sh` script, or manual if you have an existing WebLogic domain configuration. ## Build Docker Image for Apache Web Server -You need to build the Docker image for Apache Web Server that enbeds Oracle WebLogic Server Proxy Plugin. +You need to build the Docker image for Apache Web Server that embeds the Oracle WebLogic Server Proxy Plugin. 1. Download and build the Docker image for the Apache Web Server with 12.2.1.3.0 Oracle WebLogic Server Proxy Plugin. See the instructions in [Apache Web Server with Oracle WebLogic Server Proxy Plugin on Docker](https://github.com/oracle/docker-images/tree/master/OracleWebLogic/samples/12213-webtier-apache). - 2. tag your Docker image to `store/oracle/apache:12.2.1.3` using `docker tag` command. + 2. Tag your Docker image to `store/oracle/apache:12.2.1.3` using `docker tag` command. ``` @@ -19,7 +19,7 @@ You need to build the Docker image for Apache Web Server that enbeds Oracle WebL More information about the Apache plugin can be found at: [Apache Web Server with Oracle WebLogic Server Proxy Plugin on Docker](https://docs.oracle.com/middleware/1213/webtier/develop-plugin/apache.htm#PLGWL395). -Once you have access to the Docker image of the Apache Web Server, you can go ahead follow the instructions below to setup and start Kubernetes artifacts for Apache Web Server. +Once you have access to the Docker image of the Apache Web Server, you can go ahead follow the instructions below to setup and start Kubernetes resources for an Apache Web Server. ## Use Apache load balancer with a WebLogic domain created with the WebLogic Operator @@ -52,7 +52,9 @@ The user can access an application from outside of the Kubernetes cluster via ht ### Use the default plugin WL module configuration -By default, the Apache Docker image supports a simple WebLogic server proxy plugin configuration for a single WebLogic domain with an admin server and a cluster. The `create-weblogic-domain.sh` script automatically customizes the default behavior based on your domain configuration. The default setting only supports the type of load balancing that uses the root path ("/"). You can further customize the root path of the load balancer with `loadBalancerAppPrepath` property in the `create-weblogic-domain-inputs.yaml` file. +By default, the Apache Docker image supports a simple WebLogic server proxy plugin configuration for a single WebLogic domain with an admin server and a cluster. The `create-weblogic-domain.sh` script automatically customizes the default behavior based on your domain configuration by generating a customized Kubernetes resources yaml file for Apache named `weblogic-domain-apache.yaml`. The default setting only supports the type of load balancing that uses the root path ("/"). + +You can further customize the root path of the load balancer with `loadBalancerAppPrepath` property in the `create-weblogic-domain-inputs.yaml` file. ``` @@ -62,7 +64,9 @@ loadBalancerAppPrepath: /weblogic ``` -The user can then access an application from utsidie of the Kubernetes cluster via `http://:30305/weblogic/,` and the admin can access the admin console via `http://:30305/console`. +It is sometimes, but rarely, desirable to expose a WebLogic administrative host and port through a load balancer to a public network. If this is needed, then, once the `weblogic-domain-apache.yaml` file is generated, you can customize exposure of the WebLogic admin admin host and port by uncommenting the `WEBLOGIC_HOST` and `WEBLOGIC_PORT` environment variables in the file. If this files' resources have already been deployed (as happens automatically when running `create-weblogic-domain.sh`), one way to make the change is to delete the files' running Kubernetes resources via `kubectl delete -f weblogic-domain-apache.yaml`, and then deploy them again via `kubectl create -f weblogic-domain-apache.yaml`. + +The user can then access an application from outside of the Kubernetes cluster via `http://:30305/weblogic/,` and, if the admin server host and port environment variables are uncommented below, an admin can access the admin console via `http://:30305/console`. The generated Kubernetes yaml files look like the following given the domainUID "domain1". @@ -174,13 +178,13 @@ spec: value: '/weblogic' - - name: WEBLOGIC_HOST + #- name: WEBLOGIC_HOST - value: 'domain1-admin-server' + # value: 'domain1-admin-server' - - name: WEBLOGIC_PORT + #- name: WEBLOGIC_PORT - value: '7001' + # value: '7001' readinessProbe: @@ -349,7 +353,7 @@ subjects: ``` -Here are examples of the Kubernetes artifacts created by the WebLogic Operator: +Here are examples of the Kubernetes resources created by the WebLogic Operator: ``` @@ -406,7 +410,7 @@ loadBalancerVolumePath: ``` -After the `loadBalancerVolumePath` property is specified, the `create-weblogic-domain.sh` script will use the custom_mod_wl_apache.config file in `` directory to replace what is in the Docker image. +After the `loadBalancerVolumePath` property is specified, the `create-weblogic-domain.sh` script will use the custom_mod_wl_apache.conf file in `` directory to replace what is in the Docker image. The generated yaml files will look similar except with un-commented entries like bellow. @@ -440,7 +444,7 @@ The generated yaml files will look similar except with un-commented entries like ## Use the Apache load balancer with a manually created WebLogic Domain -If your WebLogic domain is not created by the WebLogic Operator, you need to manually create and start all Kubernetes' artifacts for Apache Web Server. +If your WebLogic domain is not created by the WebLogic Operator, you need to manually create and start all Kubernetes' resources for Apache Web Server. 1. Create your own custom_mod_wl_apache.conf file, and put it in a local dir, say ``. See the instructions in [Apache Web Server with Oracle WebLogic Server Proxy Plugin on Docker](https://docs.oracle.com/middleware/1213/webtier/develop-plugin/apache.htm#PLGWL395). From b0ae15c167418e1d20c7b857f327dab6a29381f2 Mon Sep 17 00:00:00 2001 From: Tom Moreau Date: Tue, 1 May 2018 14:05:04 -0400 Subject: [PATCH 151/186] Version the last of the artifacts, remove the old format annotations and use the version labels instead to determine when resources are obsolete --- .../operator/helpers/AnnotationHelper.java | 25 ------------- .../operator/helpers/ConfigMapHelper.java | 7 ++-- .../operator/helpers/IngressHelper.java | 5 +-- .../operator/helpers/PodHelper.java | 6 ++-- .../operator/helpers/ServiceHelper.java | 11 +++--- .../operator/helpers/VersionHelper.java | 35 +++++++++++++++++++ .../create/ConfigMapHelperConfigTest.java | 3 +- .../operator/create/PodHelperConfigTest.java | 1 - 8 files changed, 49 insertions(+), 44 deletions(-) create mode 100644 operator/src/main/java/oracle/kubernetes/operator/helpers/VersionHelper.java diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/AnnotationHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/AnnotationHelper.java index db882ee47d7..341e49bb3a3 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/AnnotationHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/AnnotationHelper.java @@ -3,8 +3,6 @@ package oracle.kubernetes.operator.helpers; -import java.util.Objects; - import io.kubernetes.client.models.V1ObjectMeta; /** @@ -12,19 +10,6 @@ * */ public class AnnotationHelper { - // Make these public so that the tests can use them: - public static final String FORMAT_ANNOTATION = "weblogic.oracle/operator-formatVersion"; - public static final String FORMAT_VERSION = "1"; - - /** - * Marks metadata object with an annotation saying that it was created for this format version - * @param meta Metadata object that will be included in a newly created resource, e.g. pod or service - */ - public static void annotateWithFormat(V1ObjectMeta meta) { - meta.putAnnotationsItem(FORMAT_ANNOTATION, FORMAT_VERSION); - } - - /** * Marks metadata with annotations that let Prometheus know how to retrieve metrics from * the wls-exporter web-app. The specified httpPort should be the listen port of the WebLogic server @@ -37,14 +22,4 @@ public static void annotateForPrometheus(V1ObjectMeta meta, int httpPort) { meta.putAnnotationsItem("prometheus.io/path", "/wls-exporter/metrics"); meta.putAnnotationsItem("prometheus.io/scrape", "true"); } - - /** - * Check the metadata object for the presence of an annotation matching the expected format version. - * @param meta The metadata object - * @return true, if the metadata includes an annotation matching the expected format version - */ - public static boolean checkFormatAnnotation(V1ObjectMeta meta) { - String metaResourceVersion = meta.getAnnotations().get(FORMAT_ANNOTATION); - return Objects.equals(FORMAT_VERSION, metaResourceVersion); - } } diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java index af8607984e0..d64183dd9f0 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java @@ -99,7 +99,8 @@ public NextAction onSuccess(Packet packet, V1ConfigMap result, int statusCode, } }); return doNext(create, packet); - } else if (AnnotationHelper.checkFormatAnnotation(result.getMetadata()) && result.getData().entrySet().containsAll(cm.getData().entrySet())) { + } else if (VersionHelper.matchesResourceVersion(result.getMetadata(), LabelConstants.RESOURCE_VERSION_LABEL) && + result.getData().entrySet().containsAll(cm.getData().entrySet())) { // existing config map has correct data LOGGER.fine(MessageKeys.CM_EXISTS, domainNamespace); packet.put(ProcessingConstants.SCRIPT_CONFIG_MAP, result); @@ -142,9 +143,7 @@ protected V1ConfigMap computeDomainConfigMap() { V1ObjectMeta metadata = new V1ObjectMeta(); metadata.setName(name); metadata.setNamespace(domainNamespace); - - AnnotationHelper.annotateWithFormat(metadata); - + Map labels = new HashMap<>(); labels.put(LabelConstants.RESOURCE_VERSION_LABEL, VersionConstants.DOMAIN_V1); labels.put(LabelConstants.OPERATORNAME_LABEL, operatorNamespace); diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/IngressHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/IngressHelper.java index b63ae86d18e..01f446b8a7c 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/IngressHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/IngressHelper.java @@ -20,6 +20,7 @@ import oracle.kubernetes.operator.KubernetesConstants; import oracle.kubernetes.operator.LabelConstants; import oracle.kubernetes.operator.ProcessingConstants; +import oracle.kubernetes.operator.VersionConstants; import oracle.kubernetes.weblogic.domain.v1.Domain; import oracle.kubernetes.weblogic.domain.v1.DomainSpec; import oracle.kubernetes.operator.work.ContainerResolver; @@ -78,9 +79,9 @@ public NextAction apply(Packet packet) { v1ObjectMeta.setNamespace(namespace); v1ObjectMeta.putAnnotationsItem(KubernetesConstants.CLASS_INGRESS, KubernetesConstants.CLASS_INGRESS_VALUE); - AnnotationHelper.annotateWithFormat(v1ObjectMeta); Map labels = new HashMap<>(); + labels.put(LabelConstants.RESOURCE_VERSION_LABEL, VersionConstants.DOMAIN_V1); labels.put(LabelConstants.DOMAINUID_LABEL, weblogicDomainUID); labels.put(LabelConstants.DOMAINNAME_LABEL, weblogicDomainName); labels.put(LabelConstants.CLUSTERNAME_LABEL, clusterName); @@ -141,7 +142,7 @@ public NextAction onSuccess(Packet packet, V1beta1Ingress result, int statusCode } }), packet); } else { - if (AnnotationHelper.checkFormatAnnotation(result.getMetadata()) && v1beta1Ingress.getSpec().equals(result.getSpec())) { + if (VersionHelper.matchesResourceVersion(result.getMetadata(), VersionConstants.DOMAIN_V1) && v1beta1Ingress.getSpec().equals(result.getSpec())) { return doNext(packet); } return doNext(factory.create().replaceIngressAsync(ingressName, meta.getNamespace(), diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java index d7e626d68f7..549eab127ad 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java @@ -184,8 +184,7 @@ protected V1Pod computeAdminPodConfig(TuningParameters configMapHelper, Packet p metadata.setName(podName); metadata.setNamespace(namespace); adminPod.setMetadata(metadata); - - AnnotationHelper.annotateWithFormat(metadata); + AnnotationHelper.annotateForPrometheus(metadata, spec.getAsPort()); Map labels = new HashMap<>(); @@ -408,7 +407,7 @@ private static boolean validateCurrentPod(V1Pod build, V1Pod current) { // returns fields, such as nodeName, even when export=true is specified. // Therefore, we'll just compare specific fields - if (!AnnotationHelper.checkFormatAnnotation(current.getMetadata())) { + if (!VersionHelper.matchesResourceVersion(current.getMetadata(), LabelConstants.RESOURCE_VERSION_LABEL)) { return false; } @@ -613,7 +612,6 @@ protected V1Pod computeManagedPodConfig(TuningParameters configMapHelper, Packet metadata.setNamespace(namespace); pod.setMetadata(metadata); - AnnotationHelper.annotateWithFormat(metadata); AnnotationHelper.annotateForPrometheus(metadata, scan.getListenPort()); Map labels = new HashMap<>(); diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/ServiceHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/ServiceHelper.java index 2e82ad02591..dbbaba02249 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/ServiceHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/ServiceHelper.java @@ -7,6 +7,7 @@ import io.kubernetes.client.models.*; import oracle.kubernetes.operator.LabelConstants; import oracle.kubernetes.operator.ProcessingConstants; +import oracle.kubernetes.operator.VersionConstants; import oracle.kubernetes.weblogic.domain.v1.Domain; import oracle.kubernetes.weblogic.domain.v1.DomainSpec; import oracle.kubernetes.operator.helpers.HealthCheckHelper.KubernetesVersion; @@ -72,10 +73,10 @@ public NextAction apply(Packet packet) { metadata.setName(name); metadata.setNamespace(namespace); - AnnotationHelper.annotateWithFormat(metadata); metadata.putAnnotationsItem("service.alpha.kubernetes.io/tolerate-unready-endpoints", "true"); Map labels = new HashMap<>(); + labels.put(LabelConstants.RESOURCE_VERSION_LABEL, VersionConstants.DOMAIN_V1); labels.put(LabelConstants.DOMAINUID_LABEL, weblogicDomainUID); labels.put(LabelConstants.DOMAINNAME_LABEL, weblogicDomainName); labels.put(LabelConstants.SERVERNAME_LABEL, serverName); @@ -252,9 +253,8 @@ public NextAction apply(Packet packet) { metadata.setName(name); metadata.setNamespace(namespace); - AnnotationHelper.annotateWithFormat(metadata); - Map labels = new HashMap<>(); + labels.put(LabelConstants.RESOURCE_VERSION_LABEL, VersionConstants.DOMAIN_V1); labels.put(LabelConstants.DOMAINUID_LABEL, weblogicDomainUID); labels.put(LabelConstants.DOMAINNAME_LABEL, weblogicDomainName); labels.put(LabelConstants.CLUSTERNAME_LABEL, clusterName); @@ -368,7 +368,7 @@ private static boolean validateCurrentService(V1Service build, V1Service current V1ServiceSpec buildSpec = build.getSpec(); V1ServiceSpec currentSpec = current.getSpec(); - if (!AnnotationHelper.checkFormatAnnotation(current.getMetadata())) { + if (!VersionHelper.matchesResourceVersion(current.getMetadata(), LabelConstants.RESOURCE_VERSION_LABEL)) { return false; } @@ -520,10 +520,9 @@ public NextAction apply(Packet packet) { V1ObjectMeta metadata = new V1ObjectMeta(); metadata.setName(name); metadata.setNamespace(namespace); - - AnnotationHelper.annotateWithFormat(metadata); Map labels = new HashMap<>(); + labels.put(LabelConstants.RESOURCE_VERSION_LABEL, VersionConstants.DOMAIN_V1); labels.put(LabelConstants.DOMAINUID_LABEL, weblogicDomainUID); labels.put(LabelConstants.DOMAINNAME_LABEL, weblogicDomainName); labels.put(LabelConstants.SERVERNAME_LABEL, serverName); diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/VersionHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/VersionHelper.java new file mode 100644 index 00000000000..874962ebbcd --- /dev/null +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/VersionHelper.java @@ -0,0 +1,35 @@ +// Copyright 2017, 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.helpers; + +import java.util.Map; + +import io.kubernetes.client.models.V1ObjectMeta; +import oracle.kubernetes.operator.LabelConstants; + +/** + * Helper methods for managing versions. + * + */ +public class VersionHelper { + /** + * Determines whether a resource matches a version + * @param meta Metadata + * @param resourceVersion resource version + */ + public static boolean matchesResourceVersion(V1ObjectMeta meta, String resourceVersion) { + if (meta == null) { + return false; + } + Map labels = meta.getLabels(); + if (labels == null) { + return false; + } + String val = labels.get(LabelConstants.RESOURCE_VERSION_LABEL); + if (val == null) { + return false; + } + return val.equals(resourceVersion); + } +} diff --git a/operator/src/test/java/oracle/kubernetes/operator/create/ConfigMapHelperConfigTest.java b/operator/src/test/java/oracle/kubernetes/operator/create/ConfigMapHelperConfigTest.java index 2294d700350..726af02ac27 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/create/ConfigMapHelperConfigTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/create/ConfigMapHelperConfigTest.java @@ -58,8 +58,7 @@ private V1ConfigMap getDesiredDomainConfigMap() { .namespace(DOMAIN_NAMESPACE) .putLabelsItem(RESOURCE_VERSION_LABEL, DOMAIN_V1) .putLabelsItem(OPERATORNAME_LABEL, OPERATOR_NAMESPACE) - .putLabelsItem(CREATEDBYOPERATOR_LABEL, "true") - .putAnnotationsItem(FORMAT_ANNOTATION, FORMAT_VERSION)) + .putLabelsItem(CREATEDBYOPERATOR_LABEL, "true")) .putDataItem(PROPERTY_LIVENESS_PROBE_SH, "") .putDataItem(PROPERTY_READINESS_PROBE_SH, "") .putDataItem(PROPERTY_READ_STATE_SH, "") diff --git a/operator/src/test/java/oracle/kubernetes/operator/create/PodHelperConfigTest.java b/operator/src/test/java/oracle/kubernetes/operator/create/PodHelperConfigTest.java index ca54f994832..5713fd7a225 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/create/PodHelperConfigTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/create/PodHelperConfigTest.java @@ -426,7 +426,6 @@ private V1Pod getDesiredBaseServerPodConfigForDefaults(String image, String imag .putAnnotationsItem("prometheus.io/path", "/wls-exporter/metrics") .putAnnotationsItem("prometheus.io/port", "" + port) .putAnnotationsItem("prometheus.io/scrape", "true") - .putAnnotationsItem("weblogic.oracle/operator-formatVersion", "1") .putLabelsItem(RESOURCE_VERSION_LABEL, DOMAIN_V1) .putLabelsItem(CREATEDBYOPERATOR_LABEL, "true") .putLabelsItem(DOMAINNAME_LABEL, DOMAIN_NAME) From 6043eb8b3595d9a2f42fa3416ede3babc9e16293 Mon Sep 17 00:00:00 2001 From: Tom Moreau Date: Tue, 1 May 2018 14:57:46 -0400 Subject: [PATCH 152/186] Switch version strings from e.g. operator/v1 to operator-v1 since metadata labels are not allowed to include / characters --- kubernetes/create-weblogic-domain-inputs.yaml | 2 +- .../create-weblogic-operator-inputs.yaml | 2 +- .../create-weblogic-domain-job-template.yaml | 4 ++-- kubernetes/internal/create-weblogic-domain.sh | 2 +- .../internal/create-weblogic-operator.sh | 2 +- .../domain-custom-resource-template.yaml | 2 +- .../internal/generate-security-policy.sh | 20 +++++++++---------- .../internal/voyager-ingress-template.yaml | 4 ++-- ...logic-domain-apache-security-template.yaml | 4 ++-- .../weblogic-domain-apache-template.yaml | 8 ++++---- .../internal/weblogic-domain-pv-template.yaml | 2 +- .../weblogic-domain-pvc-template.yaml | 2 +- ...ogic-domain-traefik-security-template.yaml | 4 ++-- .../weblogic-domain-traefik-template.yaml | 12 +++++------ .../internal/weblogic-operator-template.yaml | 12 +++++------ .../kubernetes/operator/VersionConstants.java | 14 ++++++------- 16 files changed, 48 insertions(+), 48 deletions(-) diff --git a/kubernetes/create-weblogic-domain-inputs.yaml b/kubernetes/create-weblogic-domain-inputs.yaml index 66a98bfd658..82d26617228 100644 --- a/kubernetes/create-weblogic-domain-inputs.yaml +++ b/kubernetes/create-weblogic-domain-inputs.yaml @@ -2,7 +2,7 @@ # Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. # The version of this inputs file. Do not modify. -version: create-weblogic-domain-inputs/v1 +version: create-weblogic-domain-inputs-v1 # Port number for admin server adminPort: 7001 diff --git a/kubernetes/create-weblogic-operator-inputs.yaml b/kubernetes/create-weblogic-operator-inputs.yaml index 8681bac66e1..1e7bba7cf8b 100644 --- a/kubernetes/create-weblogic-operator-inputs.yaml +++ b/kubernetes/create-weblogic-operator-inputs.yaml @@ -2,7 +2,7 @@ # Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. # The version of this inputs file. Do not modify. -version: create-weblogic-operator-inputs/v1 +version: create-weblogic-operator-inputs-v1 # The name of the service account that the operator will use to # make requests to the Kubernetes API server. diff --git a/kubernetes/internal/create-weblogic-domain-job-template.yaml b/kubernetes/internal/create-weblogic-domain-job-template.yaml index e262dc934fd..540f0133c2f 100644 --- a/kubernetes/internal/create-weblogic-domain-job-template.yaml +++ b/kubernetes/internal/create-weblogic-domain-job-template.yaml @@ -6,7 +6,7 @@ metadata: name: %DOMAIN_UID%-create-weblogic-domain-job-cm namespace: %NAMESPACE% labels: - weblogic.resourceVersion: domain/v1 + weblogic.resourceVersion: domain-v1 weblogic.domainUID: %DOMAIN_UID% weblogic.domainName: %DOMAIN_NAME% data: @@ -267,7 +267,7 @@ spec: template: metadata: labels: - weblogic.resourceVersion: domain/v1 + weblogic.resourceVersion: domain-v1 weblogic.domainUID: %DOMAIN_UID% weblogic.domainName: %DOMAIN_NAME% app: %DOMAIN_UID%-create-weblogic-domain-job diff --git a/kubernetes/internal/create-weblogic-domain.sh b/kubernetes/internal/create-weblogic-domain.sh index 8659101cffb..ef6da1a5b55 100755 --- a/kubernetes/internal/create-weblogic-domain.sh +++ b/kubernetes/internal/create-weblogic-domain.sh @@ -111,7 +111,7 @@ function initAndValidateOutputDir { # Function to validate the version of the inputs file # function validateVersion { - local requiredVersion='create-weblogic-domain-inputs/v1' + local requiredVersion='create-weblogic-domain-inputs-v1' if [ "${version}" != "${requiredVersion}" ]; then validationError "Invalid version: \"${version}\". Must be ${requiredVersion}." fi diff --git a/kubernetes/internal/create-weblogic-operator.sh b/kubernetes/internal/create-weblogic-operator.sh index f8577020e4e..baf0566576e 100755 --- a/kubernetes/internal/create-weblogic-operator.sh +++ b/kubernetes/internal/create-weblogic-operator.sh @@ -209,7 +209,7 @@ function validateImagePullPolicy { # Function to validate the version of the inputs file # function validateVersion { - local requiredVersion='create-weblogic-operator-inputs/v1' + local requiredVersion='create-weblogic-operator-inputs-v1' if [ "${version}" != "${requiredVersion}" ]; then validationError "Invalid version: \"${version}\". Must be ${requiredVersion}." fi diff --git a/kubernetes/internal/domain-custom-resource-template.yaml b/kubernetes/internal/domain-custom-resource-template.yaml index 79227bd4626..04f622292e8 100644 --- a/kubernetes/internal/domain-custom-resource-template.yaml +++ b/kubernetes/internal/domain-custom-resource-template.yaml @@ -9,7 +9,7 @@ metadata: name: %DOMAIN_UID% namespace: %NAMESPACE% labels: - weblogic.resourceVersion: domain/v1 + weblogic.resourceVersion: domain-v1 weblogic.domainUID: %DOMAIN_UID% weblogic.domainName: %DOMAIN_NAME% spec: diff --git a/kubernetes/internal/generate-security-policy.sh b/kubernetes/internal/generate-security-policy.sh index 47fb781139c..e9a9cc6ef8e 100755 --- a/kubernetes/internal/generate-security-policy.sh +++ b/kubernetes/internal/generate-security-policy.sh @@ -67,7 +67,7 @@ kind: Namespace metadata: name: ${NAMESPACE} labels: - weblogic.resourceVersion: operator/v1 + weblogic.resourceVersion: operator-v1 weblogic.operatorName: ${NAMESPACE} --- # @@ -79,7 +79,7 @@ metadata: namespace: ${NAMESPACE} name: ${ACCOUNT_NAME} labels: - weblogic.resourceVersion: operator/v1 + weblogic.resourceVersion: operator-v1 weblogic.operatorName: ${NAMESPACE} --- EOF @@ -93,7 +93,7 @@ apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: weblogic-operator-cluster-role labels: - weblogic.resourceVersion: operator/v1 + weblogic.resourceVersion: operator-v1 weblogic.operatorName: ${NAMESPACE} rules: - apiGroups: [""] @@ -126,7 +126,7 @@ apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: weblogic-operator-cluster-role-nonresource labels: - weblogic.resourceVersion: operator/v1 + weblogic.resourceVersion: operator-v1 weblogic.operatorName: ${NAMESPACE} rules: - nonResourceURLs: ["/version/*"] @@ -140,7 +140,7 @@ apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: ${NAMESPACE}-operator-rolebinding labels: - weblogic.resourceVersion: operator/v1 + weblogic.resourceVersion: operator-v1 weblogic.operatorName: ${NAMESPACE} subjects: - kind: ServiceAccount @@ -157,7 +157,7 @@ apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: ${NAMESPACE}-operator-rolebinding-nonresource labels: - weblogic.resourceVersion: operator/v1 + weblogic.resourceVersion: operator-v1 weblogic.operatorName: ${NAMESPACE} subjects: - kind: ServiceAccount @@ -174,7 +174,7 @@ apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: ${NAMESPACE}-operator-rolebinding-discovery labels: - weblogic.resourceVersion: operator/v1 + weblogic.resourceVersion: operator-v1 weblogic.operatorName: ${NAMESPACE} subjects: - kind: ServiceAccount @@ -191,7 +191,7 @@ apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: ${NAMESPACE}-operator-rolebinding-auth-delegator labels: - weblogic.resourceVersion: operator/v1 + weblogic.resourceVersion: operator-v1 weblogic.operatorName: ${NAMESPACE} subjects: - kind: ServiceAccount @@ -211,7 +211,7 @@ apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: weblogic-operator-namespace-role labels: - weblogic.resourceVersion: operator/v1 + weblogic.resourceVersion: operator-v1 weblogic.operatorName: ${NAMESPACE} rules: - apiGroups: [""] @@ -254,7 +254,7 @@ metadata: name: weblogic-operator-rolebinding namespace: ${i} labels: - weblogic.resourceVersion: operator/v1 + weblogic.resourceVersion: operator-v1 weblogic.operatorName: ${NAMESPACE} subjects: - kind: ServiceAccount diff --git a/kubernetes/internal/voyager-ingress-template.yaml b/kubernetes/internal/voyager-ingress-template.yaml index d63a136bb08..90ecd5499e9 100644 --- a/kubernetes/internal/voyager-ingress-template.yaml +++ b/kubernetes/internal/voyager-ingress-template.yaml @@ -4,7 +4,7 @@ metadata: name: %DOMAIN_UID%-voyager namespace: %NAMESPACE% labels: - weblogic.resourceVersion: voyager-load-balancer/v1 + weblogic.resourceVersion: voyager-load-balancer-v1 weblogic.domainUID: %DOMAIN_UID% weblogic.domainName: %DOMAIN_NAME% annotations: @@ -28,7 +28,7 @@ metadata: namespace: %NAMESPACE% labels: app: %DOMAIN_UID%-voyager-stats - weblogic.resourceVersion: voyager-load-balancer/v1 + weblogic.resourceVersion: voyager-load-balancer-v1 weblogic.domainUID: %DOMAIN_UID% weblogic.domainName: %DOMAIN_NAME% spec: diff --git a/kubernetes/internal/weblogic-domain-apache-security-template.yaml b/kubernetes/internal/weblogic-domain-apache-security-template.yaml index 761d4c5ce23..b24821ed687 100644 --- a/kubernetes/internal/weblogic-domain-apache-security-template.yaml +++ b/kubernetes/internal/weblogic-domain-apache-security-template.yaml @@ -4,7 +4,7 @@ apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: %DOMAIN_UID%-apache-webtier labels: - weblogic.resourceVersion: apache-load-balancer/v1 + weblogic.resourceVersion: apache-load-balancer-v1 weblogic.domainUID: %DOMAIN_UID% weblogic.domainName: %DOMAIN_NAME% rules: @@ -33,7 +33,7 @@ apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: %DOMAIN_UID%-apache-webtier labels: - weblogic.resourceVersion: apache-load-balancer/v1 + weblogic.resourceVersion: apache-load-balancer-v1 weblogic.domainUID: %DOMAIN_UID% weblogic.domainName: %DOMAIN_NAME% roleRef: diff --git a/kubernetes/internal/weblogic-domain-apache-template.yaml b/kubernetes/internal/weblogic-domain-apache-template.yaml index d2fe2a0d575..7ec8c56bd83 100755 --- a/kubernetes/internal/weblogic-domain-apache-template.yaml +++ b/kubernetes/internal/weblogic-domain-apache-template.yaml @@ -5,7 +5,7 @@ metadata: name: %DOMAIN_UID%-apache-webtier namespace: %NAMESPACE% labels: - weblogic.resourceVersion: apache-load-balancer/v1 + weblogic.resourceVersion: apache-load-balancer-v1 weblogic.domainUID: %DOMAIN_UID% weblogic.domainName: %DOMAIN_NAME% app: apache-webtier @@ -16,7 +16,7 @@ metadata: name: %DOMAIN_UID%-apache-webtier namespace: %NAMESPACE% labels: - weblogic.resourceVersion: apache-load-balancer/v1 + weblogic.resourceVersion: apache-load-balancer-v1 weblogic.domainUID: %DOMAIN_UID% weblogic.domainName: %DOMAIN_NAME% app: apache-webtier @@ -30,7 +30,7 @@ spec: template: metadata: labels: - weblogic.resourceVersion: apache-load-balancer/v1 + weblogic.resourceVersion: apache-load-balancer-v1 weblogic.domainUID: %DOMAIN_UID% weblogic.domainName: %DOMAIN_NAME% app: apache-webtier @@ -81,7 +81,7 @@ metadata: name: %DOMAIN_UID%-apache-webtier namespace: %NAMESPACE% labels: - weblogic.resourceVersion: apache-load-balancer/v1 + weblogic.resourceVersion: apache-load-balancer-v1 weblogic.domainUID: %DOMAIN_UID% weblogic.domainName: %DOMAIN_NAME% spec: diff --git a/kubernetes/internal/weblogic-domain-pv-template.yaml b/kubernetes/internal/weblogic-domain-pv-template.yaml index 8345d18362a..fe5a1a5676a 100644 --- a/kubernetes/internal/weblogic-domain-pv-template.yaml +++ b/kubernetes/internal/weblogic-domain-pv-template.yaml @@ -6,7 +6,7 @@ kind: PersistentVolume metadata: name: %DOMAIN_UID%-weblogic-domain-pv labels: - weblogic.resourceVersion: domain/v1 + weblogic.resourceVersion: domain-v1 weblogic.domainUID: %DOMAIN_UID% weblogic.domainName: %DOMAIN_NAME% spec: diff --git a/kubernetes/internal/weblogic-domain-pvc-template.yaml b/kubernetes/internal/weblogic-domain-pvc-template.yaml index b73aa93b481..85b061a28e5 100644 --- a/kubernetes/internal/weblogic-domain-pvc-template.yaml +++ b/kubernetes/internal/weblogic-domain-pvc-template.yaml @@ -7,7 +7,7 @@ metadata: name: %DOMAIN_UID%-weblogic-domain-pvc namespace: %NAMESPACE% labels: - weblogic.resourceVersion: domain/v1 + weblogic.resourceVersion: domain-v1 weblogic.domainUID: %DOMAIN_UID% weblogic.domainName: %DOMAIN_NAME% spec: diff --git a/kubernetes/internal/weblogic-domain-traefik-security-template.yaml b/kubernetes/internal/weblogic-domain-traefik-security-template.yaml index 7c9f72e26b9..7322c5ae9ba 100644 --- a/kubernetes/internal/weblogic-domain-traefik-security-template.yaml +++ b/kubernetes/internal/weblogic-domain-traefik-security-template.yaml @@ -4,7 +4,7 @@ apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: %DOMAIN_UID%-%CLUSTER_NAME_LC%-traefik labels: - weblogic.resourceVersion: traefik-load-balancer/v1 + weblogic.resourceVersion: traefik-load-balancer-v1 weblogic.domainUID: %DOMAIN_UID% weblogic.domainName: %DOMAIN_NAME% weblogic.clusterName: %CLUSTER_NAME% @@ -34,7 +34,7 @@ apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: %DOMAIN_UID%-%CLUSTER_NAME_LC%-traefik labels: - weblogic.resourceVersion: traefik-load-balancer/v1 + weblogic.resourceVersion: traefik-load-balancer-v1 weblogic.domainUID: %DOMAIN_UID% weblogic.domainName: %DOMAIN_NAME% weblogic.clusterName: %CLUSTER_NAME% diff --git a/kubernetes/internal/weblogic-domain-traefik-template.yaml b/kubernetes/internal/weblogic-domain-traefik-template.yaml index d1013d34721..c3ea39b458b 100644 --- a/kubernetes/internal/weblogic-domain-traefik-template.yaml +++ b/kubernetes/internal/weblogic-domain-traefik-template.yaml @@ -5,7 +5,7 @@ metadata: name: %DOMAIN_UID%-%CLUSTER_NAME_LC%-traefik namespace: %NAMESPACE% labels: - weblogic.resourceVersion: traefik-load-balancer/v1 + weblogic.resourceVersion: traefik-load-balancer-v1 weblogic.domainUID: %DOMAIN_UID% weblogic.domainName: %DOMAIN_NAME% weblogic.clusterName: %CLUSTER_NAME% @@ -16,7 +16,7 @@ metadata: name: %DOMAIN_UID%-%CLUSTER_NAME_LC%-traefik namespace: %NAMESPACE% labels: - weblogic.resourceVersion: traefik-load-balancer/v1 + weblogic.resourceVersion: traefik-load-balancer-v1 weblogic.domainUID: %DOMAIN_UID% weblogic.domainName: %DOMAIN_NAME% weblogic.clusterName: %CLUSTER_NAME% @@ -29,7 +29,7 @@ spec: template: metadata: labels: - weblogic.resourceVersion: traefik-load-balancer/v1 + weblogic.resourceVersion: traefik-load-balancer-v1 weblogic.domainUID: %DOMAIN_UID% weblogic.domainName: %DOMAIN_NAME% weblogic.clusterName: %CLUSTER_NAME% @@ -85,7 +85,7 @@ metadata: name: %DOMAIN_UID%-%CLUSTER_NAME_LC%-traefik-cm namespace: %NAMESPACE% labels: - weblogic.resourceVersion: traefik-load-balancer/v1 + weblogic.resourceVersion: traefik-load-balancer-v1 weblogic.domainUID: %DOMAIN_UID% weblogic.domainName: %DOMAIN_NAME% weblogic.clusterName: %CLUSTER_NAME% @@ -109,7 +109,7 @@ metadata: name: %DOMAIN_UID%-%CLUSTER_NAME_LC%-traefik namespace: %NAMESPACE% labels: - weblogic.resourceVersion: traefik-load-balancer/v1 + weblogic.resourceVersion: traefik-load-balancer-v1 weblogic.domainUID: %DOMAIN_UID% weblogic.domainName: %DOMAIN_NAME% weblogic.clusterName: %CLUSTER_NAME% @@ -130,7 +130,7 @@ metadata: name: %DOMAIN_UID%-%CLUSTER_NAME_LC%-traefik-dashboard namespace: %NAMESPACE% labels: - weblogic.resourceVersion: traefik-load-balancer/v1 + weblogic.resourceVersion: traefik-load-balancer-v1 weblogic.domainUID: %DOMAIN_UID% weblogic.domainName: %DOMAIN_NAME% weblogic.clusterName: %CLUSTER_NAME% diff --git a/kubernetes/internal/weblogic-operator-template.yaml b/kubernetes/internal/weblogic-operator-template.yaml index 8736051dd79..8babedf55e9 100644 --- a/kubernetes/internal/weblogic-operator-template.yaml +++ b/kubernetes/internal/weblogic-operator-template.yaml @@ -14,7 +14,7 @@ metadata: name: weblogic-operator-cm namespace: %NAMESPACE% labels: - weblogic.resourceVersion: operator/v1 + weblogic.resourceVersion: operator-v1 weblogic.operatorName: %NAMESPACE% data: serviceaccount: %ACCOUNT_NAME% @@ -29,7 +29,7 @@ metadata: name: weblogic-operator-secrets namespace: %NAMESPACE% labels: - weblogic.resourceVersion: operator/v1 + weblogic.resourceVersion: operator-v1 weblogic.operatorName: %NAMESPACE% type: Opaque data: @@ -44,14 +44,14 @@ metadata: # set the namespace that you want the operator deployed in here namespace: %NAMESPACE% labels: - weblogic.resourceVersion: operator/v1 + weblogic.resourceVersion: operator-v1 weblogic.operatorName: %NAMESPACE% spec: replicas: 1 template: metadata: labels: - weblogic.resourceVersion: operator/v1 + weblogic.resourceVersion: operator-v1 weblogic.operatorName: %NAMESPACE% app: weblogic-operator spec: @@ -133,7 +133,7 @@ spec: %EXTERNAL_OPERATOR_SERVICE_PREFIX% name: external-weblogic-operator-svc %EXTERNAL_OPERATOR_SERVICE_PREFIX% namespace: %NAMESPACE% %EXTERNAL_OPERATOR_SERVICE_PREFIX% labels: -%EXTERNAL_OPERATOR_SERVICE_PREFIX% weblogic.resourceVersion: operator/v1 +%EXTERNAL_OPERATOR_SERVICE_PREFIX% weblogic.resourceVersion: operator-v1 %EXTERNAL_OPERATOR_SERVICE_PREFIX% weblogic.operatorName: %NAMESPACE% %EXTERNAL_OPERATOR_SERVICE_PREFIX%spec: %EXTERNAL_OPERATOR_SERVICE_PREFIX% type: NodePort @@ -156,7 +156,7 @@ metadata: name: internal-weblogic-operator-svc namespace: %NAMESPACE% labels: - weblogic.resourceVersion: operator/v1 + weblogic.resourceVersion: operator-v1 weblogic.operatorName: %NAMESPACE% spec: type: ClusterIP diff --git a/operator/src/main/java/oracle/kubernetes/operator/VersionConstants.java b/operator/src/main/java/oracle/kubernetes/operator/VersionConstants.java index 45c631a90a3..62bb58e7282 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/VersionConstants.java +++ b/operator/src/main/java/oracle/kubernetes/operator/VersionConstants.java @@ -4,11 +4,11 @@ package oracle.kubernetes.operator; public interface VersionConstants { - public static final String CREATE_WEBLOGIC_OPERATOR_INPUTS_V1 = "create-weblogic-operator-inputs/v1"; - public static final String CREATE_WEBLOGIC_DOMAIN_INPUTS_V1 = "create-weblogic-domain-inputs/v1"; - public static final String OPERATOR_V1 = "operator/v1"; - public static final String DOMAIN_V1 = "domain/v1"; - public static final String VOYAGER_LOAD_BALANCER_V1 = "voyager-load-balancer/v1"; - public static final String APACHE_LOAD_BALANCER_V1 = "apache-load-balancer/v1"; - public static final String TRAEFIK_LOAD_BALANCER_V1 = "traefik-load-balancer/v1"; + public static final String CREATE_WEBLOGIC_OPERATOR_INPUTS_V1 = "create-weblogic-operator-inputs-v1"; + public static final String CREATE_WEBLOGIC_DOMAIN_INPUTS_V1 = "create-weblogic-domain-inputs-v1"; + public static final String OPERATOR_V1 = "operator-v1"; + public static final String DOMAIN_V1 = "domain-v1"; + public static final String VOYAGER_LOAD_BALANCER_V1 = "voyager-load-balancer-v1"; + public static final String APACHE_LOAD_BALANCER_V1 = "apache-load-balancer-v1"; + public static final String TRAEFIK_LOAD_BALANCER_V1 = "traefik-load-balancer-v1"; } From abd9843340b3a6e66a894aa2917f526410da126c Mon Sep 17 00:00:00 2001 From: Rosemary Marano Date: Tue, 1 May 2018 15:24:48 -0400 Subject: [PATCH 153/186] incorporated Tom's request --- site/name-changes.md | 187 ----------------------------------------- site/recent-changes.md | 4 +- 2 files changed, 2 insertions(+), 189 deletions(-) delete mode 100644 site/name-changes.md diff --git a/site/name-changes.md b/site/name-changes.md deleted file mode 100644 index 16fc30bce54..00000000000 --- a/site/name-changes.md +++ /dev/null @@ -1,187 +0,0 @@ -# Oracle WebLogic Server Kubernetes Operator Name Changes - -The initial version of the WebLogic Server Kubernetes Operator did not use consistent conventions for many customer visible names. - -We addressed this by: -* Standardizing the names of scripts and input files for creating operators and domains -* Standardizing the input parameter names and values -* Standardizing the names of the generated YAML files -* Requiring that the customer create and specify a directory that the operators' and domains' generated YAML files will be stored in - -This changes how operators and domains are created, and also changes the generated artifacts for operators and domains. - -We're not providing an upgrade tool or backward compatibility with the previous names. Instead, customers need to use the new script, inputs file, and parameter names to recreate their operators and domains. - -This document lists the customer visible naming changes. Also, the WebLogic Server Kubernetes Operator documentation has been updated. - -## Customer Visible Files - -### Files for Creating and Deleting Operators and Domains - -The following files are used to create the operator and to create and delete domains. - -| Previous File Name | New File Name | -| --- | --- | -| `kubernetes/create-domain-job.sh` | `kubernetes/create-weblogic-domain.sh` | -| `kubernetes/create-domain-job-inputs.yaml` | `kubernetes/create-weblogic-domain-inputs.yaml` | -| `kubernetes/create-operator-inputs.yaml` | `kubernetes/create-weblogic-operator-inputs.yaml` | -| `kubernetes/create-weblogic-operator.sh` | same | -| `kubernetes/delete-domain.sh` | `kubernetes/delete-weblogic-operator-resources.sh` | - -### Generated YAML Files for Operators and Domains - -The create scripts generate a number of YAML files that are used to configure the corresponding Kubernetes artifacts for the operator and the domains. -Typically, customers do not use these YAML files. However, customers can look at them. They can also change the operator and domain configuration by editing these files and reapplying them. - -#### Directory for the Generated YAML Files - -Previously, these files were placed in the `kubernetes` directory (for example, `kubernetes/weblogic-operator.yaml`). Now, they are placed in per-operator and per-domain directories (because a Kubernetes cluster can have more than one operator and an operator can manage more than one domain). - -The customer must create a directory that will parent the per-operator and per-domain directories, and use the `-o` option to pass the name of that directory to the create script, for example: - `mkdir /scratch/my-user-projects - create-weblogic-operator.sh -o /scratch/my-user-projects`. -The pathname can be either a full path name or a relative path name. If it's a relative pathname, then it's relative to the directory of the shell invoking the create script. - -The per-operator directory name is: - `/weblogic-operators/`. - -Similarly, the per-domain directory name is: - `/weblogic-domains/`. - -#### What If I Mess Up Creating a Domain or Operator And Want To Do It Again? - -* Remove the resources that were created for the domain: - * `kubernetes/delete-weblogic-domain-resources.sh yourDomainUID` -* Remove the resources that were created for the operator: - * `kubectl delete -f weblogic-operator.yaml` - * `kubectl delete -f weblogic-operator-security.yaml` -* Either remove the directory that was generated for that operator or domain, or remove the generated YAML files and the copy of the input file from it. -* Make whatever changes you need in your inputs file. -* Re-run the create script. - -If you run the create script without cleaning up the previously generated directory, the create script will tell you about the offending files and then exit without creating anything. - -#### Location of the Input YAML Files - -The create scripts support an `-i` option for specifying the location of the inputs file. Similar to the `-o` option, the path can be either a full path name or a relative path name. Relative path names are relative to the directory of the shell invoking the create script. - -If `-i` is not specified, `kubernetes/create-weblogic-operator.sh` uses `kubernetes/create-weblogic-operator-inputs.yaml`. - -Previously, `kubernetes/create-domain-job.sh` used `kubernetes/create-domain-job-inputs.yaml` as the input file if `-i` was not specified. This behavior has been changed. The customer must select a Kubernetes cluster-wide unique ID for the domain and set the `domainUid` property in the inputs file to that value. This means that the customer must always modify the inputs file. - -Also, we do not want the customer to have to change files in the operator's install directory. Because of this, the `-i` option MUST be specified when calling `kubernetes/create-weblogic-operator.sh`. The basic flow is: - -* Pick a user projects directory, for example, `/scratch/my-user-projects` -* `mkdir /scratch/my-user-projects` -* Pick a unique ID for the domain, for example, `foo.com` -* `cp kubernetes/create-weblogic-domain-inputs.yaml my-inputs.yaml` -* Set the domainUid in `my-inputs.yaml` to `foo.com` -* `kubernetes/create-weblogic-operator.sh -i my-inputs.yaml -o /scratch/my-user-projects` - -**Note:** `my-inputs.yaml` will be copied to `/scratch/my-user-projects/weblogic-domains/foo.com/create-weblogic-domain-inputs.yaml` - -#### File Names of the Generated YAML Files - -The names of several of the generated YAML files have changed. - -| Previous File Name | New File Name | -| --- | --- | -| `domain-custom-resource.yaml` | same | -| `domain-job.yaml` | `create-weblogic-domain-job.yaml` | -| `persistent-volume.yaml` | `weblogic-domain-pv.yaml` | -| `persistent-volume-claim.yaml` | `weblogic-domain-pvc.yaml` | -| `rbac.yaml` | `weblogic-operator-security.yaml` | -| `traefik-deployment.yaml` | `weblogic-domain-traefik-${clusterName, lower case}.yaml` | -| `traefik-rbac.yaml` | `weblogic-domain-traefik-security-${clusterName, lower case}.yaml` | -| `weblogic-operator.yaml` | same | - -## Input File Contents -Some of the contents of the inputs files have changed: -* Some properties have been renamed -* Some properties that are no longer needed have been removed -* Some of the legal property values have changed -* Some properties that were optional are now required - -### create-weblogic-operator-inputs.yaml - -#### Property Names - -| Previous Property Name | New Property Name | -| --- | --- | -| `image` | `weblogicOperatorImage` | -| `imagePullPolicy` | `weblogicOperatorImagePullPolicy` | -| `imagePullSecretName` | `weblogicOperatorImagePullSecretName` | - -#### Property Values - -| Previous Property Name | Property Name | Old Property Value | New Property Value | -| --- | --- | --- | --- | -| `externalRestOption` | `externalRestOption` | `none` | `NONE` | -| `externalRestOption` | `externalRestOption` | `custom-cert` | `CUSTOM_CERT` | -| `externalRestOption` | `externalRestOption` | `self-signed-cert` | `SELF_SIGNED_CERT` | - -### create-weblogic-domain-inputs.yaml - -#### Property Names - -| Previous Property Name | New Property Name | -| --- | --- | -| `createDomainScript` | This property has been removed | -| `domainUid` | `domainUID` | -| `imagePullSecretName` | `weblogicImagePullSecretName` | -| `loadBalancerAdminPort` | `loadBalancerDashboardPort` | -| `managedServerCount` | `configuredManagedServerCount` | -| `managedServerStartCount` | `initialManagedServerReplicas` | -| `nfsServer` | `weblogicDomainStorageNFSServer` | -| `persistencePath` | `weblogicDomainStoragePath` | -| `persistenceSize` | `weblogicDomainStorageSize` | -| `persistenceType` | `weblogicDomainStorageType` | -| `persistenceStorageClass` | This property has been removed | -| `persistenceVolumeClaimName` | This property has been removed | -| `persistenceVolumeName` | This property has been removed | -| `secretName` | `weblogicCredentialsSecretName` | - -#### Properties That Must be Customized -The following input properties, which used to have default values, now must be uncommented and customized. - -| Previous Property Name | New Property Name | Previous Default Value | Notes | -| --- | --- | --- | --- | -| `domainUid` | `domainUID` | `domain1` | Because the domain UID is supposed to be unique across the Kubernetes cluster, the customer must choose one. | -| `persistencePath` | `weblogicDomainStoragePath` | `/scratch/k8s_dir/persistentVolume001` | The customer must select a directory for the domain's storage. | -| `nfsServer` | `weblogicDomainStorageNFSServer` | `nfsServer` | If `weblogicDomainStorageType` is NFS, then the customer must specify the name or IP of the NFS server. | - -#### Property Values - -| Previous Property Name | New Property Name | Old Property Value | New Property Value | -| --- | --- | --- | --- | -| `loadBalancer` | `loadBalancer` | `none` | `NONE` | -| `loadBalancer` | `loadBalancer` | `traefik` | `TRAEFIK` | -| `persistenceType` | `weblogicDomainStorageType` | `hostPath` | `HOST_PATH` | -| `persistenceType` | `weblogicDomainStorageType` | `nfs` | `NFS` | - -## Kubernetes Artifact Names - -| Artifact Type | Previous Name | New Name | -| --- | --- | --- | -| persistent volume | `${domainUid}-${persistenceVolume}` or `${persistenceVolume}` | `${domainUID}-weblogic-domain-pv` | -| persistent volume claim | `${domainUid}-${persistenceVolumeClaim}` or `${persistenceVolumeClaim}` | `${domainUID}-weblogic-domain-pvc` | -| storage class name | `${domainUid}` or `${persistenceStorageClass}` | `${domainUID-weblogic-domain-storage-class` | -| job | `domain-${domainUid}-job` | `${domainUID}-create-weblogic-domain-job` | -| container | `domain-job` | `create-weblogic-domain-job` | -| container | `${d.domainUID}-${d.clusterName, lower case}-traefik` | `traefik` | -| config map | `operator-config-map` | `weblogic-operator-cm` | -| config map | `${domainUid}-${clusterName, lower case}-traefik` | `${domainUID}-${clusterName, lower case}-traefik-cm` | -| config map | `domain-${domainUid}-scripts` | `${domainUID}-create-weblogic-domain-job-cm` | -| config map | `weblogic-domain-config-map` | `weblogic-domain-cm` | -| secret | `operator-secrets` | `weblogic-operator-secrets` | -| port | `rest-https` | `rest` | -| service | `external-weblogic-operator-service` | `external-weblogic-operator-srv` | -| service | `internal-weblogic-operator-service` | `internal-weblogic-operator-srv` | -| volume & mount | `operator-config-volume` | `weblogic-operator-cm-volume` | -| volume & mount | `operator-secrets-volume` | `weblogic-operator-secrets-volume` | -| volume & mount | `config-map-scripts` | `create-weblogic-domain-job-cm-volume` | -| volume & mount | `pv-storage` | `weblogic-domain-storage-volume` | -| volume & mount | `secrets` | `weblogic-credentials-volume` | -| volume & mount | `scripts` | `weblogic-domain-cm-volume` | - -**Note:** The input properties for controlling the domain's persistent volume, persistent volume claim, and storage class names have been removed. These names are now derived from the domain UID. diff --git a/site/recent-changes.md b/site/recent-changes.md index 2d5a1c7224f..1279a1d1974 100644 --- a/site/recent-changes.md +++ b/site/recent-changes.md @@ -4,6 +4,6 @@ This document tracks recent changes to the operator, especially ones that introd | Date | Introduces backward incompatibilities | Change | | --- | --- | --- | -| March 20, 2018 | yes | See [Name Changes](name-changes.md). Several files and input parameters have been renamed. This affects how operators and domains are created. It also changes generated Kubernetes artifacts, therefore customers must recreate their operators and domains. -| April 4, 2018 | yes | See [Name Changes](name-changes.md). Many Kubernetes artifact names and labels have changed. Also, the names of generated YAML files for creating a domain's PV and PVC have changed. Because of these changes, customers must recreate their operators and domains. +| March 20, 2018 | yes | Several files and input parameters have been renamed. This affects how operators and domains are created. It also changes generated Kubernetes artifacts, therefore customers must recreate their operators and domains. +| April 4, 2018 | yes | Many Kubernetes artifact names and labels have changed. Also, the names of generated YAML files for creating a domain's PV and PVC have changed. Because of these changes, customers must recreate their operators and domains. | May 7, 2018 | no | Added support for dynamic clusters, the Apache HTTP Server, the Voyager Ingress Controller, and for PV in NFS storage for multi-node environments. From a7a062be897a3300ed0839cc52768cfc38bc5b17 Mon Sep 17 00:00:00 2001 From: Tom Barnes Date: Tue, 1 May 2018 12:39:18 -0700 Subject: [PATCH 154/186] Address on editor review comments. --- site/apache.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/site/apache.md b/site/apache.md index e1263d3b8f8..1648705827a 100644 --- a/site/apache.md +++ b/site/apache.md @@ -52,7 +52,7 @@ Users can access an application from outside of the Kubernetes cluster by using ### Use the default plugin WL module configuration -By default, the Apache Docker image supports a simple WebLogic Server proxy plugin configuration for a single WebLogic domain with an Administration Server and a cluster. The `create-weblogic-domain.sh` script automatically customizes the default behavior based on your domain configuration by generating a customized Kubernetes resources yaml file for Apache named `weblogic-domain-apache.yaml`. The default setting supports only the type of load balancing that uses the root path ("/"). You can further customize the root path of the load balancer with the `loadBalancerAppPrepath` property in the `create-weblogic-domain-inputs.yaml` file. +By default, the Apache Docker image supports a simple WebLogic Server proxy plugin configuration for a single WebLogic domain with an Administration Server and a cluster. The `create-weblogic-domain.sh` script automatically customizes the default behavior based on your domain configuration by generating a customized Kubernetes resources YAML file for Apache named `weblogic-domain-apache.yaml`. The default setting supports only the type of load balancing that uses the root path ("/"). You can further customize the root path of the load balancer with the `loadBalancerAppPrepath` property in the `create-weblogic-domain-inputs.yaml` file. ``` @@ -62,9 +62,9 @@ loadBalancerAppPrepath: /weblogic ``` -It is sometimes, but rarely, desirable to expose a WebLogic administrative host and port through a load balancer to a public network. If this is needed, then, once the `weblogic-domain-apache.yaml` file is generated, you can customize exposure of the WebLogic admin admin host and port by uncommenting the `WEBLOGIC_HOST` and `WEBLOGIC_PORT` environment variables in the file. If this files' resources have already been deployed (as happens automatically when running `create-weblogic-domain.sh`), one way to make the change is to delete the files' running Kubernetes resources via `kubectl delete -f weblogic-domain-apache.yaml`, and then deploy them again via `kubectl create -f weblogic-domain-apache.yaml`. +It is sometimes, but rarely, desirable to expose a WebLogic Administration Server host and port through a load balancer to a public network. If this is needed, then, after the `weblogic-domain-apache.yaml` file is generated, you can customize exposure of the WebLogic Administration Server host and port by uncommenting the `WEBLOGIC_HOST` and `WEBLOGIC_PORT` environment variables in the file. If this file's resources have already been deployed (as happens automatically when running `create-weblogic-domain.sh`), one way to make the change is to delete the file's running Kubernetes resources using `kubectl delete -f weblogic-domain-apache.yaml`, and then deploy them again via `kubectl create -f weblogic-domain-apache.yaml`. -Users can then access an application from outside of the Kubernetes cluster by using `http://:30305/weblogic/,` and, if the WebLogic administration server host and port environment variables are uncommented below, an adminstrator can access the Administration Console via `http://:30305/console`. +Users can then access an application from outside of the Kubernetes cluster by using `http://:30305/weblogic/,` and, if the WebLogic Administration Server host and port environment variables are uncommented below, an adminstrator can access the Administration Console using `http://:30305/console`. The generated Kubernetes YAML files look like the following, given the `domainUID`, "`domain1`". From 46b5482b765f9be462096141b95b7a0c3a16694a Mon Sep 17 00:00:00 2001 From: Tom Barnes Date: Tue, 1 May 2018 12:53:39 -0700 Subject: [PATCH 155/186] Fix integration test operator log capture after a failure (so that it's able to find the operator pod). --- src/integration-tests/bash/run.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/integration-tests/bash/run.sh b/src/integration-tests/bash/run.sh index 2c6d20cc295..00e87de8e3d 100755 --- a/src/integration-tests/bash/run.sh +++ b/src/integration-tests/bash/run.sh @@ -1392,7 +1392,7 @@ function call_operator_rest { trace "Calling some operator REST APIs via ${REST_ADDR}/${URL_TAIL}" - #pod=`kubectl get pod -n $OPERATOR_NS | grep $OPERATOR_NS | awk '{ print $1 }'` + #pod=`kubectl get pod -n $OPERATOR_NS --show-labels=true | grep $OPERATOR_NS | awk '{ print $1 }'` #kubectl logs $pod -n $OPERATOR_NS > "${OPERATOR_TMP_DIR}/operator.pre.rest.log" # turn off all of the https proxying so that curl will work @@ -1950,7 +1950,7 @@ function verify_service_and_pod_created { done if [ "${srv_count:=Error}" != "1" ]; then - local pod=`kubectl get pod -n $OPERATOR_NS | grep $OPERATOR_NS | awk '{ print $1 }'` + local pod=`kubectl get pod -n $OPERATOR_NS --show-labels=true | grep $OPERATOR_NS | awk '{ print $1 }'` local debuglog="${OPERATOR_TMP_DIR}/verify_domain_debugging.log" kubectl logs $pod -n $OPERATOR_NS > "${debuglog}" if [ -f ${debuglog} ] ; then @@ -1975,7 +1975,7 @@ function verify_service_and_pod_created { fi if [ "${srv_count:=Error}" != "1" ]; then - local pod=`kubectl get pod -n $OPERATOR_NS | grep $OPERATOR_NS | awk '{ print $1 }'` + local pod=`kubectl get pod -n $OPERATOR_NS --show-labels=true | grep $OPERATOR_NS | awk '{ print $1 }'` local debuglog="${OPERATOR_TMP_DIR}/verify_domain_debugging.log" kubectl logs $pod -n $OPERATOR_NS > "${debuglog}" if [ -f ${debuglog} ] ; then From a4ffffbec67374790fa82dc87c34e96a2ad1f25e Mon Sep 17 00:00:00 2001 From: Tom Barnes Date: Tue, 1 May 2018 12:54:56 -0700 Subject: [PATCH 156/186] Keep last 10 integration test archives (instead of last 5). --- src/integration-tests/bash/archive.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/integration-tests/bash/archive.sh b/src/integration-tests/bash/archive.sh index 27fecfb9ebc..e61b3e2753a 100755 --- a/src/integration-tests/bash/archive.sh +++ b/src/integration-tests/bash/archive.sh @@ -6,7 +6,7 @@ # archive.sh # - internal helper method called by run.sh # - archives directory ${1} into ${2}/IntSuite.TIMESTAMP.tar.gz -# - deletes all but the 5 newest archives +# - deletes all but the 10 newest archives # - this method doesn't have any configurable env vars # @@ -38,7 +38,7 @@ function archive { [ $? -eq 0 ] || fail "Could not archive, 'tar -czf $ARCHIVE $SOURCE_DIR' command failed: `cat $OUTFILE`" rm -f $OUTFILE - find $ARCHIVE_DIR -maxdepth 1 -name "IntSuite*tar.gz" | sort -r | awk '{ if (NR>5) print $NF }' | xargs rm -f + find $ARCHIVE_DIR -maxdepth 1 -name "IntSuite*tar.gz" | sort -r | awk '{ if (NR>10) print $NF }' | xargs rm -f trace Archived to \'$ARCHIVE\'. } From 0e12e0a663a3a5cef3cdd81276f16bbd9c2c36c2 Mon Sep 17 00:00:00 2001 From: Rosemary Marano Date: Tue, 1 May 2018 15:57:09 -0400 Subject: [PATCH 157/186] minor text edits --- site/wlst.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/site/wlst.md b/site/wlst.md index 5f54d2358a3..7bf22e7c814 100644 --- a/site/wlst.md +++ b/site/wlst.md @@ -2,7 +2,7 @@ Note that a video demonstration of using WLST to create a data source is available [here](https://youtu.be/eY-KXEk8rI4). -WebLogic Scripting Tool (WLST) can be used to manage a domain running in Kubernetes. If the domain was configured to expose a T3 channel using the `exposeAdminT3Channel` setting when creating the domain, then the matching T3 service can be used to connect. For example, if the `domainUID` is `domain1`, and the Administration Server name is `admin-server`, then the service would be called: +You can use the WebLogic Scripting Tool (WLST) to manage a domain running in Kubernetes. If the domain was configured to expose a T3 channel using the `exposeAdminT3Channel` setting when creating the domain, then the matching T3 service can be used to connect. For example, if the `domainUID` is `domain1`, and the Administration Server name is `admin-server`, then the service would be called: ``` domain1-admin-server-extchannel-t3channel @@ -15,7 +15,7 @@ $ kubectl get service domain1-admin-server-extchannel-t3channel -n domain1 -o js 30012 ``` -In this example, the `nodePort` is `30012`. If the Kubernetes server’s address were `kubernetes001`, then WLST can connect to `t3://kubernetes001:30012` as shown below: +In this example, the `nodePort` is `30012`. If the Kubernetes server’s address was `kubernetes001`, then WLST can connect to `t3://kubernetes001:30012` as shown below: ``` $ ~/wls/oracle_common/common/bin/wlst.sh From 453be30317447e1488be686d59f0830c52c2b969 Mon Sep 17 00:00:00 2001 From: Rosemary Marano Date: Tue, 1 May 2018 16:41:33 -0400 Subject: [PATCH 158/186] incorporated Monica's edit --- site/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/installation.md b/site/installation.md index 0ae945228f4..640700bb171 100644 --- a/site/installation.md +++ b/site/installation.md @@ -69,7 +69,7 @@ The operator is deployed with the provided installation script (`create-weblogic | Parameter | Definition | Default | | --- | --- | --- | | `elkIntegrationEnabled` | Determines whether the Elastic Stack integration will be enabled. If set to `true`, then Elasticsearch, Logstash, and Kibana will be installed, and Logstash will be configured to export the operator’s logs to Elasticsearch. | `false` | -| `externalDebugHttpPort` | The port number of the operator's debugging port outside of the Kubernetes cluster. | `3099`9 | +| `externalDebugHttpPort` | The port number of the operator's debugging port outside of the Kubernetes cluster. | `30999` | | `externalOperatorCert` | A base64 encoded string containing the X.509 certificate that the operator will present to clients accessing its REST endpoints. This value is only used when `externalRestOption` is set to `CUSTOM_CERT`. | | | `externalOperatorKey` | A base64 encoded string containing the private key of the operator's X.509 certificate. This value is used only when `externalRestOption` is set to `CUSTOM_CERT`. | | | `externalRestHttpsPort`| The `NodePort` number that should be allocated on which the operator REST server should listen for HTTPS requests. | `31001` | From 002d4c5a476a2cec62c9067075e975588e547d42 Mon Sep 17 00:00:00 2001 From: Rosemary Marano Date: Tue, 1 May 2018 17:19:14 -0400 Subject: [PATCH 159/186] incorporated Lenny's comments --- site/scaling.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/site/scaling.md b/site/scaling.md index 093b25b672b..5848a91a7a9 100644 --- a/site/scaling.md +++ b/site/scaling.md @@ -1,5 +1,7 @@ # Scaling a WebLogic cluster +WebLogic Server supports two types of clustering configurations, configured and dynamic clustering. Configured clusters are created by manually configuring each individual Managed Server instance. In dynamic clusters, the Managed Server configurations are generated from a single, shared template.  With dynamic clusters, when additional server capacity is needed, new server instances can be added to the cluster without having to manually configure them individually. Also, unlike configured clusters, scaling up of dynamic clusters is not restricted to the set of servers defined in the cluster but can be increased based on runtime demands. For more information on how to create, configure, and use dynamic clusters in WebLogic Server, see [Dynamic Clusters](https://docs.oracle.com/middleware/1221/wls/CLUST/dynamic_clusters.htm#CLUST678). + The operator provides the ability to scale WebLogic clusters by simply editing the replicas setting, as you would do with most other Kubernetes resources that support scaling. ## Initiating a scaling operation using the REST API From cd4fb6343d5a77ca3c1ed71d56a2f92c48b751a8 Mon Sep 17 00:00:00 2001 From: Tom Moreau Date: Tue, 1 May 2018 19:24:01 -0400 Subject: [PATCH 160/186] Add VersionHelper unit tests --- .../operator/helpers/VersionHelperTest.java | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 operator/src/test/java/oracle/kubernetes/operator/helpers/VersionHelperTest.java diff --git a/operator/src/test/java/oracle/kubernetes/operator/helpers/VersionHelperTest.java b/operator/src/test/java/oracle/kubernetes/operator/helpers/VersionHelperTest.java new file mode 100644 index 00000000000..1cc1dd7c190 --- /dev/null +++ b/operator/src/test/java/oracle/kubernetes/operator/helpers/VersionHelperTest.java @@ -0,0 +1,42 @@ +// Copyright 2018, Oracle Corporation and/or its affiliates. All rights reserved. +// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.helpers; + +import io.kubernetes.client.models.V1ObjectMeta; +import static oracle.kubernetes.operator.LabelConstants.*; +import static oracle.kubernetes.operator.create.KubernetesArtifactUtils.*; +import static oracle.kubernetes.operator.helpers.VersionHelper.*; +import static org.hamcrest.MatcherAssert.*; +import static org.hamcrest.Matchers.*; +import org.junit.Test; + +public class VersionHelperTest { + + private static final String V1 = "v1"; + + @Test + public void null_metadata_returns_false() throws Exception { + assertThat(matchesResourceVersion(null, V1), equalTo(false)); + } + + @Test + public void null_labels_returns_false() throws Exception { + assertThat(matchesResourceVersion(newObjectMeta().labels(null), V1), equalTo(false)); + } + + @Test + public void null_version_returns_false() throws Exception { + assertThat(matchesResourceVersion(newObjectMeta().putLabelsItem(RESOURCE_VERSION_LABEL, null), V1), equalTo(false)); + } + + @Test + public void different_version_returns_false() throws Exception { + assertThat(matchesResourceVersion(newObjectMeta().putLabelsItem(RESOURCE_VERSION_LABEL, "v2"), V1), equalTo(false)); + } + + @Test + public void same_version_returns_true() throws Exception { + assertThat(matchesResourceVersion(newObjectMeta().putLabelsItem(RESOURCE_VERSION_LABEL, V1), V1), equalTo(true)); + } +} From 2c4823bd5534cc0ddda9caa67a36868873f2c7cc Mon Sep 17 00:00:00 2001 From: Rosemary Marano Date: Wed, 2 May 2018 07:12:51 -0400 Subject: [PATCH 161/186] changed admin UI to Web UI --- site/traefik.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/traefik.md b/site/traefik.md index 80e74060953..fda4e5e40ee 100644 --- a/site/traefik.md +++ b/site/traefik.md @@ -4,7 +4,7 @@ If the `loadBalancer` option is set to `traefik` when running the `create-weblog More information about the Traefik Ingress controller can be found at: [https://docs.traefik.io/user-guide/kubernetes/](https://docs.traefik.io/user-guide/kubernetes/) -Traefik will expose two `NodePorts` that allow access to the Ingress itself and to the Traefik admin UI. The ports are controlled by these settings in the domain inputs YAML file: +Traefik will expose two `NodePorts` that allow access to the Ingress itself and to the Traefik Web UI. The ports are controlled by these settings in the domain inputs YAML file: ``` # Load balancer web port From ae22707d407e540c8f694339563de97947ab5b8f Mon Sep 17 00:00:00 2001 From: Rosemary Marano Date: Wed, 2 May 2018 08:04:52 -0400 Subject: [PATCH 162/186] second pass edits --- site/installation.md | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/site/installation.md b/site/installation.md index cb1c58453f0..cb7c04efb00 100644 --- a/site/installation.md +++ b/site/installation.md @@ -15,38 +15,38 @@ Note that there is a short video demonstration of the installation process avail You can build, test, and publish the Docker image for the operator directly from Wercker using the ```wercker.yml``` from this repository. -If you haven't done so already, navigate to [wercker.com](https://www.wercker.com) and create an account. Once you are logged in, -on the [app.wercker.com] (https://app.wercker.com) page press, "Create your first application." +If you haven't done so already, navigate to [wercker.com](https://www.wercker.com) and create an account. After you are logged in, +on the [app.wercker.com](https://app.wercker.com) page, click "Create your first application." -Select GitHub (the default, if you are new to Wercker). If you haven't done so already, press the "Connect" button within the -larger GitHub button and follow the prompts to provide a login for GitHub. This connects your Wercker and GitHub accounts so -that Wercker pipelines will later be able to clone this repository. Press, "Next." +Select GitHub (the default, if you are new to Wercker). If you haven't done so already, click the "Connect" button within the +larger GitHub button, and follow the prompts to provide a login for GitHub. This connects your Wercker and GitHub accounts so +that Wercker pipelines will later be able to clone this repository. Click "Next." -Select the repository from GitHub. This will be "oracle / weblogic-kubernetes-operator" or a different value if you -forked this repository. Press, "Next." +Select the repository from GitHub. This will be `oracle / weblogic-kubernetes-operator` or a different value if you +forked this repository. Click "Next." -Configure Wercker's access to the GitHub repository. The default choice, "wercker will check out the code without using an SSH key", -is typically sufficient. Press, "Next." +Configure Wercker's access to the GitHub repository. The default choice, "wercker will check out the code without using an SSH key", +is typically sufficient. Click "Next." -Verify the settings so far on the review page and press, "Create." +Verify the settings so far on the review page and click "Create." -Since this GitHub repository already has a ```wercker.yml``` file, you can skip directly to the "Environment" tab. +Because this GitHub repository already has a ```wercker.yml``` file, you can skip directly to the "Environment" tab. -Please provide the following key/value pairs on the environment page. Remember that these values will be +Please provide the following key/value pairs on the Environment page. Remember that these values will be visible to anyone to whom you give access to the Wercker application, therefore, select "Protected" for any values that should remain hidden, including all passwords. | Key | Value | OCIR Sample | | --- | --- | --- | -| DOCKER_USERNAME | Username for the Docker store for pulling serverjre image | | -| DOCKER_PASSWORD | Password for the Docker store | | -| REPO_REGISTRY| Registry address | https://phx.ocir.io/v2 | -| REPO_REPOSITORY | Repository value | phx.ocir.io//weblogic-kubernetes-operator | -| REPO_USERNAME | Username for registry | / | -| REPO_PASSWORD | Password for registry | Use generated SWIFT password | -| IMAGE_TAG_OPERATOR | Image tag, such as 0.2 or latest | | - -Select the "Runs" tab. Skip to the bottom and press, "Trigger your first build now." +| `DOCKER_USERNAME` | Username for the Docker store for pulling server JRE image | | +| `DOCKER_PASSWORD` | Password for the Docker store | | +| `REPO_REGISTRY`| Registry address | `https://phx.ocir.io/v2` | +| `REPO_REPOSITORY` | Repository value | `phx.ocir.io//weblogic-kubernetes-operator` | +| `REPO_USERNAME` | Username for registry | `/` | +| `REPO_PASSWORD` | Password for registry | `Use generated SWIFT password` | +| `IMAGE_TAG_OPERATOR` | Image tag, such as 0.2 or latest | | + +Select the "Runs" tab. Scroll to the bottom and click "Trigger your first build now." When the run completes successfully, the Docker image for the operator will be built and published to your repository. From fa4976fa441ab516610788cddbb53c5e44a899ea Mon Sep 17 00:00:00 2001 From: Rosemary Marano Date: Wed, 2 May 2018 08:53:24 -0400 Subject: [PATCH 163/186] fix nested list --- site/scaling.md | 1 - 1 file changed, 1 deletion(-) diff --git a/site/scaling.md b/site/scaling.md index 5848a91a7a9..d90cdd17a31 100644 --- a/site/scaling.md +++ b/site/scaling.md @@ -47,7 +47,6 @@ When the operator receives a scaling request, it will: * Validate that the WebLogic cluster, identified by `clusterName`, exists. * Verify that the specified WebLogic cluster has a sufficient number of configured servers to satisfy the scaling request. * Initiate scaling by setting the `replicas` property within the corresponding domain custom resource, which can be done in either: - * A `clusterStartup` entry, if defined within its cluster list * At the domain level, if not defined in a `clusterStartup` entry and the `startupControl` property is set to `AUTO` From f2bcb9c1c71bae36a44517e008a9e70698272222 Mon Sep 17 00:00:00 2001 From: Russell Gold Date: Wed, 2 May 2018 09:06:00 -0400 Subject: [PATCH 164/186] remove wdt changes --- Dockerfile | 1 - .../operator/KubernetesConstants.java | 1 - .../operator/ProcessingConstants.java | 1 - .../operator/helpers/DomainHomeHelper.java | 200 ------------------ pom.xml | 17 -- 5 files changed, 220 deletions(-) delete mode 100644 operator/src/main/java/oracle/kubernetes/operator/helpers/DomainHomeHelper.java diff --git a/Dockerfile b/Dockerfile index 61555a75d81..36b6c070e02 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,6 @@ ENV PATH=$PATH:/operator COPY src/scripts/* /operator/ COPY operator/target/weblogic-kubernetes-operator-0.2.jar /operator/weblogic-kubernetes-operator.jar -COPY target/weblogic-deploy.zip /operator/weblogic-deploy.zip COPY operator/target/lib/*.jar /operator/lib/ HEALTHCHECK --interval=1m --timeout=10s \ diff --git a/operator/src/main/java/oracle/kubernetes/operator/KubernetesConstants.java b/operator/src/main/java/oracle/kubernetes/operator/KubernetesConstants.java index 89b384655d5..0e12b245032 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/KubernetesConstants.java +++ b/operator/src/main/java/oracle/kubernetes/operator/KubernetesConstants.java @@ -26,6 +26,5 @@ public interface KubernetesConstants { public static final String CONTAINER_NAME = "weblogic-server"; public static final String DOMAIN_CONFIG_MAP_NAME = "weblogic-domain-cm"; - public static final String DOMAIN_HOME_CONFIG_MAP_NAME = "weblogic-domain-home-cm"; } diff --git a/operator/src/main/java/oracle/kubernetes/operator/ProcessingConstants.java b/operator/src/main/java/oracle/kubernetes/operator/ProcessingConstants.java index e6649b54079..fec5c18378a 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/ProcessingConstants.java +++ b/operator/src/main/java/oracle/kubernetes/operator/ProcessingConstants.java @@ -31,7 +31,6 @@ public interface ProcessingConstants { public static final String EXPLICIT_RESTART_CLUSTERS = "explicitRestartClusters"; public static final String SCRIPT_CONFIG_MAP = "scriptConfigMap"; - public static final String DOMAIN_HOME_CONFIG_MAP = "domainHomeConfigMap"; public static final String SERVER_STATE_MAP = "serverStateMap"; public static final String SERVER_HEALTH_MAP = "serverHealthMap"; diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/DomainHomeHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/DomainHomeHelper.java deleted file mode 100644 index 897cf6dac75..00000000000 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/DomainHomeHelper.java +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright 2018, Oracle Corporation and/or its affiliates. All rights reserved. -// Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. - -package oracle.kubernetes.operator.helpers; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import io.kubernetes.client.ApiException; -import io.kubernetes.client.models.V1ConfigMapVolumeSource; -import io.kubernetes.client.models.V1Container; -import io.kubernetes.client.models.V1EnvVar; -import io.kubernetes.client.models.V1Job; -import io.kubernetes.client.models.V1JobSpec; -import io.kubernetes.client.models.V1ObjectMeta; -import io.kubernetes.client.models.V1PersistentVolumeClaimVolumeSource; -import io.kubernetes.client.models.V1PodSpec; -import io.kubernetes.client.models.V1PodTemplateSpec; -import io.kubernetes.client.models.V1SecretVolumeSource; -import io.kubernetes.client.models.V1Status; -import io.kubernetes.client.models.V1Volume; -import io.kubernetes.client.models.V1VolumeMount; -import oracle.kubernetes.operator.KubernetesConstants; -import oracle.kubernetes.operator.LabelConstants; -import oracle.kubernetes.operator.logging.LoggingFacade; -import oracle.kubernetes.operator.logging.LoggingFactory; -import oracle.kubernetes.operator.work.Container; -import oracle.kubernetes.operator.work.ContainerResolver; -import oracle.kubernetes.operator.work.NextAction; -import oracle.kubernetes.operator.work.Packet; -import oracle.kubernetes.operator.work.Step; -import oracle.kubernetes.weblogic.domain.v1.Domain; -import oracle.kubernetes.weblogic.domain.v1.DomainSpec; - -public class DomainHomeHelper { - private static final LoggingFacade LOGGER = LoggingFactory.getLogger("Operator", "Operator"); - - private DomainHomeHelper() {} - - public static Step createDomainHomeStep(Step next) { - return new DomainHomeStep(next); - } - - public static class DomainHomeStep extends Step { - public DomainHomeStep(Step next) { - super(next); - } - - @Override - public NextAction apply(Packet packet) { - Container c = ContainerResolver.getInstance().getContainer(); - CallBuilderFactory factory = c.getSPI(CallBuilderFactory.class); - - DomainPresenceInfo info = packet.getSPI(DomainPresenceInfo.class); - Domain dom = info.getDomain(); - V1ObjectMeta meta = dom.getMetadata(); - String namespace = meta.getNamespace(); - - V1Job job = computeDomainHomeJob(info); - Step conflictStep = factory.create().deleteJobAsync(job.getMetadata().getName(), namespace, new ResponseStep(next) { - @Override - public NextAction onFailure(Packet packet, ApiException e, int statusCode, - Map> responseHeaders) { - if (statusCode == CallBuilder.NOT_FOUND) { - return onSuccess(packet, null, statusCode, responseHeaders); - } - return super.onFailure(packet, e, statusCode, responseHeaders); - } - - @Override - public NextAction onSuccess(Packet packet, V1Status result, int statusCode, - Map> responseHeaders) { - return doNext(DomainHomeStep.this, packet); - } - }); - - Step create = factory.create().createJobAsync(namespace, computeDomainHomeJob(info), new ResponseStep(next) { - @Override - public NextAction onFailure(Packet packet, ApiException e, int statusCode, - Map> responseHeaders) { - return super.onFailure(conflictStep, packet, e, statusCode, responseHeaders); - } - - @Override - public NextAction onSuccess(Packet packet, V1Job result, int statusCode, - Map> responseHeaders) { - // TODO: wait for the job to finish or fail - // update domain status condition - return null; - } - }); - - return doNext(create, packet); - } - - protected V1Job computeDomainHomeJob(DomainPresenceInfo info) { - Domain dom = info.getDomain(); - V1ObjectMeta meta = dom.getMetadata(); - DomainSpec spec = dom.getSpec(); - String namespace = meta.getNamespace(); - - String weblogicDomainUID = spec.getDomainUID(); - String weblogicDomainName = spec.getDomainName(); - - String jobName = weblogicDomainUID + "-create-weblogic-domain-job"; - - String imageName = spec.getImage(); - if (imageName == null || imageName.length() == 0) { - imageName = KubernetesConstants.DEFAULT_IMAGE; - } - String imagePullPolicy = spec.getImagePullPolicy(); - if (imagePullPolicy == null || imagePullPolicy.length() == 0) { - imagePullPolicy = (imageName.endsWith(KubernetesConstants.LATEST_IMAGE_SUFFIX)) ? KubernetesConstants.ALWAYS_IMAGEPULLPOLICY : KubernetesConstants.IFNOTPRESENT_IMAGEPULLPOLICY; - } - - V1Job job = new V1Job(); - - V1ObjectMeta metadata = new V1ObjectMeta(); - metadata.setName(jobName); - metadata.setNamespace(namespace); - job.setMetadata(metadata); - - AnnotationHelper.annotateWithFormat(metadata); - - Map labels = new HashMap<>(); - labels.put(LabelConstants.DOMAINUID_LABEL, weblogicDomainUID); - labels.put(LabelConstants.DOMAINNAME_LABEL, weblogicDomainName); - labels.put(LabelConstants.SERVERNAME_LABEL, spec.getAsName()); - labels.put(LabelConstants.CREATEDBYOPERATOR_LABEL, "true"); - metadata.setLabels(labels); - - V1JobSpec jobSpec = new V1JobSpec(); - V1PodTemplateSpec templateSpec = new V1PodTemplateSpec(); - V1PodSpec podSpec = new V1PodSpec(); - podSpec.setRestartPolicy("Never"); - - V1Container container = new V1Container(); - container.setName(weblogicDomainUID + "-create-weblogic-domain-job"); - container.setImage(imageName); - container.setImagePullPolicy(imagePullPolicy); - - V1VolumeMount volumeMount = new V1VolumeMount(); - volumeMount.setName("create-weblogic-domain-job-cm-volume"); - volumeMount.setMountPath("/u01/weblogic"); - container.addVolumeMountsItem(volumeMount); - - V1VolumeMount volumeMountShared = new V1VolumeMount(); - volumeMountShared.setName("weblogic-domain-storage-volume"); - volumeMountShared.setMountPath("/shared"); - container.addVolumeMountsItem(volumeMountShared); - - V1VolumeMount volumeMountSecrets = new V1VolumeMount(); - volumeMountSecrets.setName("weblogic-credentials-volume"); - volumeMountSecrets.setMountPath("/weblogic-operator/secrets"); - container.addVolumeMountsItem(volumeMountSecrets); - - container.setCommand(Arrays.asList("/bin/sh")); - container.addArgsItem("/u01/weblogic/create-domain-job.sh"); - - V1EnvVar envItem = new V1EnvVar(); - envItem.setName("SHARED_PATH"); - envItem.setValue("/shared"); - container.addEnvItem(envItem); - - podSpec.addContainersItem(container); - - V1Volume volumeDomainConfigMap = new V1Volume(); - volumeDomainConfigMap.setName("create-weblogic-domain-job-cm-volume"); - V1ConfigMapVolumeSource cm = new V1ConfigMapVolumeSource(); - cm.setName(KubernetesConstants.DOMAIN_HOME_CONFIG_MAP_NAME); - cm.setDefaultMode(0555); // read and execute - volumeDomainConfigMap.setConfigMap(cm); - podSpec.addVolumesItem(volumeDomainConfigMap); - - if (!info.getClaims().getItems().isEmpty()) { - V1Volume volume = new V1Volume(); - volume.setName("weblogic-domain-storage-volume"); - V1PersistentVolumeClaimVolumeSource pvClaimSource = new V1PersistentVolumeClaimVolumeSource(); - pvClaimSource.setClaimName(info.getClaims().getItems().iterator().next().getMetadata().getName()); - volume.setPersistentVolumeClaim(pvClaimSource); - podSpec.addVolumesItem(volume); - } - - V1Volume volumeSecret = new V1Volume(); - volumeSecret.setName("weblogic-credentials-volume"); - V1SecretVolumeSource secret = new V1SecretVolumeSource(); - secret.setSecretName(spec.getAdminSecret().getName()); - volumeSecret.setSecret(secret); - podSpec.addVolumesItem(volumeSecret); - - templateSpec.setSpec(podSpec); - - jobSpec.setTemplate(templateSpec); - job.setSpec(jobSpec); - return job; - } - } -} diff --git a/pom.xml b/pom.xml index ef6ecf9f01d..5874b7dddce 100644 --- a/pom.xml +++ b/pom.xml @@ -49,23 +49,6 @@ - - com.googlecode.maven-download-plugin - download-maven-plugin - 1.4.0 - - - install-wdt - - wget - - - https://github.com/oracle/weblogic-deploy-tooling/releases/download/weblogic-deploy-tooling-0.8/weblogic-deploy.zip - ${project.build.directory} - - - - org.apache.maven.plugins maven-release-plugin From 594a559865084b97322cfe9207065d2f66a7f9c6 Mon Sep 17 00:00:00 2001 From: Russell Gold Date: Wed, 2 May 2018 10:37:55 -0400 Subject: [PATCH 165/186] remove wdt changes --- wercker.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/wercker.yml b/wercker.yml index b95b10a21cb..1a2722b5add 100644 --- a/wercker.yml +++ b/wercker.yml @@ -45,7 +45,6 @@ build: cp -R src/scripts/* /operator/ cp operator/target/weblogic-kubernetes-operator-0.2.jar /operator/weblogic-kubernetes-operator.jar cp operator/target/lib/*.jar /operator/lib/ - cp target/weblogic-deploy.zip /operator/weblogic-deploy.zip export IMAGE_TAG_OPERATOR="${IMAGE_TAG_OPERATOR:-${WERCKER_GIT_BRANCH//[_\/]/-}}" if [ "$IMAGE_TAG_OPERATOR" = "master" ]; then export IMAGE_TAG_OPERATOR="latest" From c927e05ac300b8e8563db620fc19f4a40f5b0ccd Mon Sep 17 00:00:00 2001 From: Rosemary Marano Date: Wed, 2 May 2018 11:35:14 -0400 Subject: [PATCH 166/186] re-order sections --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index a51befa06ad..e45d1c779a2 100644 --- a/README.md +++ b/README.md @@ -154,10 +154,6 @@ The operator provides the ability to scale up or down WebLogic clusters. There Please refer to [Scaling a WebLogic cluster](site/scaling.md) for more information. -## Shutting down a domain - -Please refer to [Shutting down a domain](site/shutdown-domain.md) for information about how to shut down a domain running in Kubernetes. - ## Load balancing with an Ingress controller or a web server @@ -166,6 +162,10 @@ You can choose a load balancer provider for your WebLogic domains running in a K [comment]: # (Exporting operator logs to ELK. The operator provides an option to export its log files to the ELK stack. Please refer to [ELK integration]site/elk.md for information about this capability.) +## Shutting down a domain + +Please refer to [Shutting down a domain](site/shutdown-domain.md) for information about how to shut down a domain running in Kubernetes. + ## Removing a domain To permanently remove the Kubernetes resources for a domain from a Kubernetes cluster, run the [Delete WebLogic domain resources](kubernetes/delete-weblogic-domain-resources.sh) script. This script will delete a specific domain, or all domains, and all the Kubernetes resources associated with a set of given domains. The script will also attempt a clean shutdown of a domain’s WebLogic pods before deleting its resources.  You can run the script in a test mode to show what would be shutdown and deleted without actually performing the shutdowns and deletions.   For script help, use its `-h` option. From 761623e6de103dfde90a999e8f2e9b29dc467061 Mon Sep 17 00:00:00 2001 From: Tom Moreau Date: Wed, 2 May 2018 15:41:31 -0400 Subject: [PATCH 167/186] Fix a bug where we were checking against the wrong version string when deciding to recreate resources because they're out of date --- .../oracle/kubernetes/operator/helpers/ConfigMapHelper.java | 2 +- .../main/java/oracle/kubernetes/operator/helpers/PodHelper.java | 2 +- .../java/oracle/kubernetes/operator/helpers/ServiceHelper.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java index d64183dd9f0..0e80a37ede6 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java @@ -99,7 +99,7 @@ public NextAction onSuccess(Packet packet, V1ConfigMap result, int statusCode, } }); return doNext(create, packet); - } else if (VersionHelper.matchesResourceVersion(result.getMetadata(), LabelConstants.RESOURCE_VERSION_LABEL) && + } else if (VersionHelper.matchesResourceVersion(result.getMetadata(), VersionConstants.DOMAIN_V1) && result.getData().entrySet().containsAll(cm.getData().entrySet())) { // existing config map has correct data LOGGER.fine(MessageKeys.CM_EXISTS, domainNamespace); diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java index 549eab127ad..6c8b2bf22a0 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java @@ -407,7 +407,7 @@ private static boolean validateCurrentPod(V1Pod build, V1Pod current) { // returns fields, such as nodeName, even when export=true is specified. // Therefore, we'll just compare specific fields - if (!VersionHelper.matchesResourceVersion(current.getMetadata(), LabelConstants.RESOURCE_VERSION_LABEL)) { + if (!VersionHelper.matchesResourceVersion(current.getMetadata(), VersionConstants.DOMAIN_V1)) { return false; } diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/ServiceHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/ServiceHelper.java index dbbaba02249..d0c1f8ee92c 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/ServiceHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/ServiceHelper.java @@ -368,7 +368,7 @@ private static boolean validateCurrentService(V1Service build, V1Service current V1ServiceSpec buildSpec = build.getSpec(); V1ServiceSpec currentSpec = current.getSpec(); - if (!VersionHelper.matchesResourceVersion(current.getMetadata(), LabelConstants.RESOURCE_VERSION_LABEL)) { + if (!VersionHelper.matchesResourceVersion(current.getMetadata(), VersionConstants.DOMAIN_V1)) { return false; } From 149d18bc35cd6d4825a92b37c9c74318bff884b4 Mon Sep 17 00:00:00 2001 From: Ryan Eberhard Date: Thu, 3 May 2018 10:32:00 -0400 Subject: [PATCH 168/186] Increase socket timeout for k8s calls --- .../operator/HealthCheckHelperTest.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/operator/src/test/java/oracle/kubernetes/operator/HealthCheckHelperTest.java b/operator/src/test/java/oracle/kubernetes/operator/HealthCheckHelperTest.java index 3e1c72cab74..2e5c7c5c85a 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/HealthCheckHelperTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/HealthCheckHelperTest.java @@ -51,6 +51,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.concurrent.TimeUnit; import java.util.logging.*; public class HealthCheckHelperTest { @@ -66,6 +67,9 @@ public class HealthCheckHelperTest { private List savedHandlers = new ArrayList<>(); private UserProjects userProjects; + + private ApiClient apiClient; + private CoreV1Api core; @Before public void setUp() throws Exception { @@ -83,6 +87,13 @@ public void setUp() throws Exception { unitHealthCheckHelper = new HealthCheckHelper(UNIT_NAMESPACE, Collections.singleton(UNIT_NAMESPACE)); userProjects = UserProjects.createUserProjectsDirectory(); + + apiClient = Config.defaultClient(); + core = new CoreV1Api(apiClient); + + // Ensure that client doesn't time out before call or watch + apiClient.getHttpClient().setReadTimeout(5, TimeUnit.MINUTES); + } @After @@ -101,8 +112,6 @@ public void testAccountNoPrivs() throws Exception { Assume.assumeTrue(TestUtils.isKubernetesAvailable()); // Create service account - ApiClient apiClient= Config.defaultClient(); - CoreV1Api core = new CoreV1Api(apiClient); V1ServiceAccount alice = new V1ServiceAccount(); alice.setMetadata(new V1ObjectMeta().name("alice")); try { @@ -158,8 +167,6 @@ public void testAccountPrivs() throws Exception { Assume.assumeTrue(TestUtils.isKubernetesAvailable()); // Create service account - ApiClient apiClient= Config.defaultClient(); - CoreV1Api core = new CoreV1Api(apiClient); V1ServiceAccount theo = new V1ServiceAccount(); theo.setMetadata(new V1ObjectMeta().name("theo")); try { From 498aab28c0d644bb0c2dbb22ea07deb4ebc45648 Mon Sep 17 00:00:00 2001 From: Rosemary Marano Date: Thu, 3 May 2018 14:10:44 -0400 Subject: [PATCH 169/186] re-order sections --- README.md | 55 ++++++++++++++++++++++++++----------------------------- 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index e45d1c779a2..fa1f4377aa1 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ In this documentation, several important terms are used and are intended to have Before using the operator, it is highly recommended that you read the [design philosophy](site/design.md) to develop an understanding of the operator's design, and the [architectural overview](site/architecture.md) to understand its architecture, including how WebLogic domains are deployed in Kubernetes using the operator. It is also worth reading the details of the [Kubernetes RBAC definitions](site/rbac.md) required by the operator. -# Exposing applications outside the Kubernetes cluster +## Exposing applications outside the Kubernetes cluster The operator can configure services to expose WebLogic applications and features outside of the Kubernetes cluster. Care should be taken when exposing anything externally to ensure that the appropriate security considerations are taken into account. In this regard, there is no significant difference between a WebLogic domain running in a Kubernetes cluster and a domain running in a traditional data center. The same kinds of considerations should be taken into account, for example: * Only expose those protocols and ports that need to be exposed. @@ -58,7 +58,7 @@ While it is natural to expose web applications outside the cluster, exposing adm Oracle recommends careful consideration before deciding to expose any administrative interfaces externally. -# Requirements +## Requirements The Oracle WebLogic Server Kubernetes Operator has the following requirements: @@ -69,7 +69,7 @@ The Oracle WebLogic Server Kubernetes Operator has the following requirements: **Note:** Minikube and the embedded Kubernetes in Docker for Mac and Docker for Windows are not "supported" platforms right now, but we have done some basic testing and everything appears to work in these environments. They are probably suitable for "trying out" the operator, but if you run into issues, we would ask that you try to reproduce them on a supported environment before reporting them. Also, Calico networking appears to work in the limited testing we have done so far. -# Restrictions +## Restrictions The following features are not certified or supported in this release: @@ -83,14 +83,6 @@ The following features are not certified or supported in this release: Please consult My Oracle Support [Doc ID 2349228.1](https://support.oracle.com/rs?type=doc&id=2349228.1) for up-to-date information about the features of WebLogic Server that are supported in Kubernetes environments. -# API documentation - -Documentation for APIs is provided here: - -* [Javadoc](https://oracle.github.io/weblogic-kubernetes-operator/apidocs/index.html) for the operator. - -* [Swagger](https://oracle.github.io/weblogic-kubernetes-operator/swagger/index.html) documentation for the operator's REST interface. - # User guide ## Prefer to see it rather than read about it? @@ -110,7 +102,7 @@ Like what you see? Read on for all the nitty-gritty details... Before installing the Oracle WebLogic Server Kubernetes Operator, ensure that the requirements listed above are met. If you need help setting up a Kubernetes environment, please check our [cheat sheets](site/k8s_setup.md). -The overall process of installing and configuring the operator and using it to manage WebLogic domains consists of the following steps. The provided scripts will perform most of these steps, but some must be performed manually: +The overall process of installing and configuring the operator, and using it to manage WebLogic domains, consists of the following steps: * Registering for access to the Oracle Container Registry * Setting up secrets to access the Oracle Container Registry @@ -121,27 +113,27 @@ The overall process of installing and configuring the operator and using it to m * Customizing the domain parameters file * Creating a WebLogic domain -All of the [installation steps are explained in detail here](site/installation.md). Example files are provided in the `kubernetes` directory in this repository. +The provided scripts will perform most of these steps, but some must be performed manually. All of the [installation steps are explained in detail here](site/installation.md). Example files are provided in the `kubernetes` directory in this repository. [comment]: # (If you need an Oracle database in your Kubernetes cluster, e.g. because your web application needs a place to keep its data, please see [this page] site/database.md for information about how to run the Oracle database in Kubernetes.) -## Using the operator's REST services - -The operator provides a REST API that you can use to obtain information about the configuration and to initiate scaling actions. Please refer to [Using the operator's REST services](site/rest.md) for details about how to use the REST APIs. - ## Creating a WebLogic domain with the operator -Please refer to [Creating a WebLogic domain with the operator](site/creating-domain.md) for information about how to create a WebLogic domain with the operator. +For information about how to create a WebLogic domain with the operator, see [Creating a WebLogic domain with the operator](site/creating-domain.md). [comment]: # ( Manually creating a WebLogic domain. If preferred, a domain can be created manually, i.e. without using the scripts provided with the operator. As long as the domain follows the guidelines, it can still be managed by the operator. Please refer to [Manually creating a WebLogic domain] site/manually-creating-domain.md for details. A good example of when manual domain creation may be preferred is when a user already has a set of existing WLST scripts that are used to create domains and they wish to reuse those same WLST scripts in Kubernetes, perhaps with some small modifications. ) +## Using WLST + +When creating a domain, there is an option to expose a T3 channel outside of the Kubernetes cluster to allow remote WLST access. For more information about how to use WLST with a domain running in Kubernetes, see [Using WLST](site/wlst.md) . + ## Starting up the domain -The operator will automatically start up domains that it is aware of, based on the configuration in the domain custom resource. Please refer to [Startup up a WebLogic domain](site/starting-domain.md) for details. +The operator will automatically start up domains that it is aware of, based on the configuration in the domain custom resource. For details, see [Startup up a WebLogic domain](site/starting-domain.md). -## Using WLST +## Using the operator's REST services -When creating a domain, there is an option to expose a T3 channel outside of the Kubernetes cluster to allow remote WLST access. Please refer to [Using WLST](site/wlst.md) for more information about how to use WLST with a domain running in Kubernetes. +The operator provides a REST API that you can use to obtain information about the configuration and to initiate scaling actions. For details about how to use the REST APIs, see [Using the operator's REST services](site/rest.md) ## Scaling a cluster @@ -152,8 +144,7 @@ The operator provides the ability to scale up or down WebLogic clusters. There * Using a WLDF policy rule and script action to call the operator's REST `scale` API. * Using a Prometheus alert action to call the operator's REST `scale` API. -Please refer to [Scaling a WebLogic cluster](site/scaling.md) for more information. - +For more information, see [Scaling a WebLogic cluster](site/scaling.md). ## Load balancing with an Ingress controller or a web server @@ -164,7 +155,7 @@ You can choose a load balancer provider for your WebLogic domains running in a K ## Shutting down a domain -Please refer to [Shutting down a domain](site/shutdown-domain.md) for information about how to shut down a domain running in Kubernetes. +For information about how to shut down a domain running in Kubernetes, see [Shutting down a domain](site/shutdown-domain.md) . ## Removing a domain @@ -201,17 +192,23 @@ Replace `NAMESPACE` with the namespace that the operator is running in. To remove more than one operator, repeat these steps for each operator namespace. +# Developer guide -# Recent changes +Developers interested in this project are encouraged to read the [Developer guide](site/developer.md) to learn how to build the project, run tests, and so on. The Developer guide also provides details about the structure of the code, coding standards, and the Asynchronous Call facility used in the code to manage calls to the Kuberentes API. -See [Recent changes](site/recent-changes.md) for recent changes to the operator, including any backward incompatible changes. +Please take a look at our [wish list](https://github.com/oracle/weblogic-kubernetes-operator/wiki/Wish-list) to get an idea of the kind of features we would like to add to the operator. Maybe you will see something you would like to contribute to! +# API documentation -# Developer guide +Documentation for APIs is provided here: -Developers interested in this project are encouraged to read the [Developer guide](site/developer.md) to learn how to build the project, run tests, and so on. The Developer guide also provides details about the structure of the code, coding standards, and the Asynchronous Call facility used in the code to manage calls to the Kuberentes API. +* [Javadoc](https://oracle.github.io/weblogic-kubernetes-operator/apidocs/index.html) for the operator. -Please take a look at our [wish list](https://github.com/oracle/weblogic-kubernetes-operator/wiki/Wish-list) to get an idea of the kind of features we would like to add to the operator. Maybe you will see something you would like to contribute to! +* [Swagger](https://oracle.github.io/weblogic-kubernetes-operator/swagger/index.html) documentation for the operator's REST interface. + +# Recent changes + +See [Recent changes](site/recent-changes.md) for recent changes to the operator, including any backward incompatible changes. # Contributing to the operator From 3fe6a8c79488d198a651bebd9bfd9185f54b694a Mon Sep 17 00:00:00 2001 From: anthony_lai Date: Thu, 3 May 2018 15:47:06 -0700 Subject: [PATCH 170/186] test for jenkins job --- .../kubernetes/operator/helpers/ConfigMapHelper.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java index 0e80a37ede6..d153e4e9b63 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java @@ -175,7 +175,13 @@ private synchronized Map loadScripts() { } } catch (IOException e) { LOGGER.warning(MessageKeys.EXCEPTION, e); - throw new RuntimeException(e); + e.printStackTrace(); // xyz- + LOGGER.warning(MessageKeys.EXCEPTION, new IOException("xyz- uri is " + uri)); + try (FileSystem fileSystem = FileSystems.getFileSystem(uri)) { + return walkScriptsPath(fileSystem.getPath(SCRIPTS)); + } catch(IOException ioe) { + throw new RuntimeException(e); + } } } From af44484d3deb8ecdbaf8a0a554a9fa16b26e043c Mon Sep 17 00:00:00 2001 From: Lily He Date: Fri, 4 May 2018 19:54:53 +0800 Subject: [PATCH 171/186] initial version --- site/images/haproxy-stats.png | Bin 0 -> 384215 bytes site/voyager.md | 130 ++++++++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+) create mode 100644 site/images/haproxy-stats.png create mode 100644 site/voyager.md diff --git a/site/images/haproxy-stats.png b/site/images/haproxy-stats.png new file mode 100644 index 0000000000000000000000000000000000000000..24a78c2130ef0451e1691bb7445e3cd45d2bda9c GIT binary patch literal 384215 zcmeFYXH-@o*UM3t^0hv>&}P0_U@^!>aOlxUB9Z@VINiHi5^lu#K6EHQuy#*0|SEq zfq{Xwh>v^UGChKsiGlHm)K*63qk@bK-A7kvD_aLk42%z9@c=ws&3>{B0}Xrv_9qYC zW(!II-7BB|l$0i*r!Sfv%=v0h{k-4OCz?~k3Mmj z&5-l9Q-8OecIy$L)6Axw5kaiL5;3JWL$XsCeI(Dr*QK$N-WJ;`kDCW#s$}7uVZ?Ql z5#E@a=VJW&aeH`jARw7{Lu6oE2BN+L=Zvg{&tcG2J+XG_+86yEgBfUF6PzQBfzSTY znJ4$-n{KQp_Th9=7~dP8TlZJhKez5XewVBwZAjM@fDv2FmhgrUgY>Pl)l(6|9`zrt zwd5`rnBPn3&&^JCUw)^R6y@OLr2VAL)0IH6(%pY$FcdkK#HKU-CSiwx#Px{@+mPk( zl|}wCZW^C;qm!@e$*cUNhsOj`a_*11D4&>YyIGuT_{A_2C~3{2 zYP!F;3}aw*K!?q6WTngSxMhCE?Sea^*?tU>&*zR^|2e|O8F{kR`*uB7_Uq#A8`(5V zQKt+1u3WEu&J(Yg?4*~XEJnf?Cy zv&KllllI}~!Cn21TnN$Q55`HURC?vC@Bgf zr=!~X%20I2QxY%XC{^H4`(gVBR)%z1D12SKk(>r6GoVsv{h+b;JFTXNh0Z4~?(Z+; z9?|PZotVl@H&upV+^y^~m_dW*6q@kmTrbFCj(&}*dMh`|?lLUmD6EFDh5xY=2L0H@ z+g9d!>i*k?K75}C0(~qPT4r>Fu}F0iy7(Elg3#f!byAqb0im1Qm-#~qMY$$_gF)Jz zxawPR+F;{u{chRc4CZHp*wOkoFBp$n_}4qcPvn&xC_J%SzNL3pm@-%5c66zlN9huJ zYJ|(Pu7#C7le3D5`#kvN^TjKHbB}ZX^Gr}uYJQSQv+=t0!I1PG^hpwD8sob4@x$a> z7r||wP8^4p&&u+?cwQKJzAb)m@6K4n9R3DO(C9Y>hNxmT4?H#660_sD5My-3e>&InSkEL`l5hcOcqeo_ z6sC`L@f9U0bcqB46)Q>iF?>4$Twd@6lB}{d;#YhTa3Nd_^%x^G4t#L*_z^w6w6qOA z^_bM7r@URbpWiP&rG5P(OnN<5>os-1H1>Dwm2@R1%YtyP@tiNgMT^WVG0ZOh1q;S7|if21h^WKn6Zmv&(-*HXjU; z#N!TpYyM~mzovV!N|Fs%sgsBMsQ@u+_dpfpKE^b5`zcWwA-q?7{e4&8ZnS*EQEg+BMu2)WzH7`HA>*?0bg2ScebGc`u$tXnkOzcVs}kN@wzj z1uD??4OqxjzY_YECPx-a{$sG;x4*EDdeeH7o{^uCmr*VM1!MF$TfVlUNPNn#3RWZw zk^1CYvggiC4y?%Pps^7%id_e*ReD zV3zEPyn}*_?B#q&_L$8!pM*YTd2Ibp9VHQY-}mS&X+V`G0bs3Y2vDq14A}eVQ(Tf~ zR{#SGtJLY%86v;KJf;Hp8TsAnB>5BOMeA7VHtf>vf?)0*c*qw>Q6wqMcM4o2F+*pY zWRGp{!k1vPKl-k4Os032=}b~M6_Nyr3;42JsxVnQ`e$)PvZSU(G;m2*%`SIsb+rGdl9o1ttY)&GbX%2T-qusyn-DD~dY(a3VqH#NC}`juY8uksf|#k=OkWn%5g}hVJ57CkOC5wouujHWnN7lH^Gf$N4^$| z^X0`qqH})5K0%jmgpLs|5uYN2Bk&UVluQOO2Pp?rb>6djmA)(;)^VIJs*B)79u)A%kDWs!S9B>?A(D{4jV%r%jdn zgX91~zz_I?8F4|)2KNTL230Q=lrD<&AW;mT<^j#An5wtmnLCp0OmzP;F|iyFJ!1^s zAz@>3Vct(0-EiC=K3P6k+Uh)0K+SDWY}s!Po|GT^?_00;9xrU6hE@h77C@PIgBWNG z^^U4Q)R&AeZdg!kGpTH;e5vx^l)uA%$N!Fzj+CjEhDqZ}8%bM-W)QskCL2;t0Dm+? zY#lliLj1l_b|~r{twK~}Oj)GgOG|3TaJ!W6s*y>JPU}H7H8xW2GLOC!#ORTqa!oVT z)15acbSYHDl`x4#xqVM1@p_vF|&L_zbXUEDG=9wS)g zK#?MoM{rBO#EC_sjyvjC!moN(M;F(lO`%EpXjl)V7KO-E9jCqadILz*ay;o&3Nid{*7a{CbpW(dhIY`pS;}Y%=7cf zdI95=5*gdx<=eBu=4bKG)D?C#!1)G=Yl9bq`iUa!E4A15Zwt@tlKu63=5|Q;NdX|w zi>CFRwfJOK13**7e055re&y9UW6r0>Gr=xyF z{fsgqRVR%~RTeB?K5P03@F``^3cOqRug!5+oRtdqeWMtU_*(cjSm?vr2@hdlQL>RZVmBL1a6^vFOZ_)GT zcnD`vQ(sfGRL}2fnbp|MQQG!`J0RL+b8Y+8f&Al_{381PkpX43G{u?P5~CgVyLQTs z4iYq3;3PH-!mll;TEyz@x|`<-57E8nNMt)Wb|AA+OWx zywh@W22MlB3)kaC#yh1#PG1x8&)ug3$E61Vs){D#Qs>XMLgy`mhrRkn25@kTk+Z|8 zjlEswXh2D4!^VD878SX3$V#gx>y-`~(Y4?uTS9#U@n`5RTZb@p1J!XS;<dg>v(&W{zIX`h?9Rjt>6cLvgBOjkR z@7{HX zYrL2HclY}zF$NoVcNbw!PA@Mn4liB~XIE=ZZXqEdPA(o!9v=343U)UiCwDV%b|<%2 z|Hb70=kwmu&D_=2#ogA~iS8f1W}lor+{G9e{t@(_-+ztM-PY=VNOE%f_q6UO$obDd zoZK8-od4l{-&ORVR$&?EdwXT+=626toLlr?l>bZH|B&-v^lGlQmiHt5XFUI*|DW6b zclwVu&hF0lT648ES8#H-biEh$@7_HBmiBjwfAbRM{Ac3-lOq00%l~Sakqg z^Vssktt9)4btsAt>;}Qav-8`pgN%(lGzj>sMT^OTI42G>A-cnMWi=))euubAiFL~I zaizDs-rDQqf4rBj#;7Tibp{iQdQXs>qYX$)Bu`0he$vK?=yEDSHkqq%M^-x?iCzEHiZWiZ|x`o+%)I{B`0zA!F%J?oWK{e=V za*v6`EUfBSL?af=PwP0&OuIily*R!q%*g!abbyfG!lK|R)xXZ$8((#0cAIkUvdZn? zrI;v^it%-*BzcBb!=4~)t*V%wHQ;@XrF@;+2D$$2@5K4aT1nXg72?}2){O!uN~ANE z3YXlcBbGv;(4Aq8^it-;m8{xrRfn&oXS%r1Sd8(Jz&wM^7 zj{kEU3I9<+1fPSlv!#2{LZ{lzQm6T;GQ(+Cy}wTd*|uLcKVgLZd@`d#@>|NJC4*OT zsE@8j7s}{cSXcZ2@mogQ6Q5`)U!?UxsZHb}Rcmm=jCSAK-)w(5U7mP6h(H|_x|=rm z=s6p^hO}05a<;k;dI?(N#olX!(HEN(lLfM4n% z@l{4~KxC%oys-0+i~yU|tyBBDJ_X10wx(+nX&o<>%cTz5Bynor z6bv*Yq}YoSKoM8{l=Y(X9te_lq4k~>OF*zTT29bA8(3(}U6&=5efa*HvvXZ8NU-N$h@ZvK=?)9?2?Z7}pJKzBHqDKB{H zWWe&}Q@pen73;<{GK&)o_A%h0YOECAJ)tFuW!Vnn@6Ka?hJh9xhkR}dQhk)6b61d1 zd318u^(;s4h1e#8DW{g@Fi8ZT+3HJwPr4J@B{cDTi%6o#opGqCD!mYa0G12<4oXPH=MtpJl!MkuUa=%IUM4v zBc?uST^{zIimdlttq^=q7y~Yqd|dGKFU}t^Hl@et@_E`KmrRy6AxB14ofOInAnY5K zkwz{BeK^(9^Zec*jN)S}@`1)_`Jvib9~X=gP}V zh^O3{>tg#>%4y9!MRjSg(l)$%&V%shv1OC@b<0)R+AZXj1vpt2<4XurzG@@@+r;uF?COZxD{6!$MndL)^3jEjgTwfSB1^wDs`ry$qP*6X8=g{tr6#Ozb& z=FtkeVfW_d**mF2fcj7OA1VqE^r1 z8X#WCRsTX(A(~auIWN~1ZSfrg^9!bmhnXj{LDXA#6 z<5rs%etc}Nt<~T_c{F|8jD#jm??y4|lSap{oTFw$XyPK@zT&qv^kPiTUONXH_+3i7DJ)w|ZyH9;NeipA}A1UPzLi3pVzW?V0kD%vmQty}nBe-rn8 zB|lM!s1dqUWpTEy;y84vAHIThqa_*!Ua5--Hx#K_6u<@-0bxl^c!=z`gv5Smoj z;;W$)iGYe%og6WG7$2y;;)kEUQ^8ItT0zW|tk_ly5`7koe!f5M+kR6@#uvMc)VF#x zHKcY0(={Y%YB^5VNKPra90PkePctmgWsbQyPxW#9=Jd?{EcyxV^YtbUgEJbpQ_FZ`y*O@1N+wFZrl31A_{She{%N}x_@EQf>>s@AReR5 zfaHlv7BHx5<`Yq6lP|xS_;&0Gjm^K>ULYCR9rcD<8a&at_q-ypqwOZ1g|4=2ssgtU zXG+XFI^}D096c`cKEwQOeSy2iE$u{VcJcvX1oQT!8qSA@YgPVCKcY7e8;F6*sxzlG za^J3RmbLtQRh0g}c&Cy+-kGUzA83W;&PQ_hMY@Lan`;2bysc{6IqBkU+*=Nea$1XT%l$UZ7XsTxG%KctZ(a>@8y*{9#3P0$To-5@*F82aV1Z&Fe*-JS z9Q$!WJL3$H!V&5u=0s)GxNtR?W;i}Ros04EPz~a%?fB5?!%k=u3OBNGsBPxJn+!fN z*Db(G!e?t|+F|Tb-Su0KG998q$Ohetn>Vo-DYU^sXfNTh4D9IY(Bc)aVCPH(-01RC zeK63|$t!2wkNJI62*!UnOQ<~e`OK2?YQd-;wfSo0^1%5wE$88yv1r_{L!;K4DBoRk z8EMS(60xgA9<5?11bhvX)z+*CP*p!K7uZZIo9~hMA>6DvdYY5C4H5R@W(+aBwy7lA)G-_+-$D^a*l2AwVWd{b_+DeOpI?t%Eznx?*N>~F+i ziQP>~Gc~!EOqJ^93|NFR1}ow9gm&KZ+iRxt3z@+P*WVL*08BcvVI@*CW? zW|D% z*mM~0rQPT46>Ln(I}=cB&k5pa+GnslCjj}CHq?VPjiAu#pEC2+!fH)akm(9&c&$W3 ztsI87H1O-ZQBt028wZz%U?*ik6JgH$-5eQ0)YN0-n^)N~3RY}mA~7YDSuNJSF<(DH z$2*yF=ay$0qr-8b3^Mr%rtT?$nO7{01Zg5`)cyH(t*PU8U52$CH3!4|BPc(07$ZCj zY^E66-idT@bEOVJu8@c^VvW;3&UY~CFTT%2)0bu1O4N!2J<9oZqt8=Nu~S^G8n<7K zSTdep{}eIZC{m>n`k1xY?w>uSl06g3!cu?Q2LvLEc4!jB(!vL5Sh0!_JDw571nlj2!(*? z1qpakeW&EkvW>rM5o_Txl=+eB^8k+dt)z~gGtglviO9oR^PAtRz(Ox=)U>CzU5<|R zY-%!_}@;VrKL8s-&9mIJjkq=B%D>kdAO#=f!P<=re!% z1NZ6~6FPAjm7U6GnEA)zBzz5;=%sX7*rulEQU~waM7Drp)*-m{S1-BSINDI_uIznP z@TvmW(!gtDWvATFY9__pK?wRmNdH6xiJ$?gULBiJv~YK*?5J0{LAV|JI}lDeBgmGJnMO~OPhQ{a#q{6j)5?{NVayn zVGd4qZ*RMTRn;;OHC`zd3)$>#*DvgKGJvmC_n;bMnv=5HQaZ2Vsb}*g?ykW4tvNk& z+VS%0%TndXH@4x-aqPe3(8Rn+L7r~#UNp4hr(hKhpq6viZX0i;X343>iXJ^rbt zmgbcrZgurx$Mo(zG}ymUwS|(Oqee%UEivu=oWSQ+(*^!~4|dSX9X>e|mgGr#>;*C8 zn|s469AoR1ZSOL))H4V~1@G#&_s2!q)&C1s+IZ9vZH8A$tBXoRCoR7CUwkgMBa-Uwl}7kMIw z)Me2XF&>cvVM>#+$BFhIEcpkcskM*S4z((4KCU$EXC>UiW-i1_{#;WM-a8E*51$k( zk_tZslWotf_D!oH+k4Au_mTY#vc*lk9H38nm+eEg0N8_xQcJDJmNp)CytHw?KC-f( zt{!Qm`V-CdL|pmI1ax0rzQu+P0oQg|>u%=>6M?z7&|*YPd(cROM8-`0dE<%VoM#?F_>)XKtOQ$k0$JLOGa zNJbOhDjy%AGq@B3pXqPtB?c#8{j-k;$3fq8l)h7{QftUl2co9JHsh^b;9y5!e=ct>yjE zIr=WnDWg^kjTLfCsGKkh`c|Oo2{Zb0HgeYGt6y8QJ*uV=ww;v4(mR@|E^~Io|F)p7 zV4xZ7YdS_h(NcTPTFN8+Gg-1iZD$TUFDZ{ekVr!=iQG9`NN1>F{QKVCbg7?`n~Jk2 z-aY`YVlJ(Q(M7;p*ex(1Lx$jf$chvkG~)Db4Y`>zpPZX1z;*wm)ke!2 z)9hXr_;(na%56>Lr-%d;WEidPYns>Cu=VX%Sl2J*8r01Sc%S@DzKtjPjs5DXcI-0U_ z|4wd%y+5_+*E=)?EZT8PX`gy;8-n)MjxLrrJS9_9R8a3(T`s`WXeneWcIj7K$bOUm z9CU1!{5na{iLgNK@@}43Y>qkQ>UBg7t05-iX?_ zivj~4zDE6l0*u@0;=hR4^DeQKH_&8wloqs$D1gD2@!OkAV2PdO{6w9|i8yDZ7e3v& zrMH%k{XW(nR6?TGU=-J#IA(xX0mrV!Z6PW1moA9xE}$)17d$?4ciScKLHM%O#qsj3 zKhi6r9X&2mqWLipgvEmrdOy~dQn#k(?Q5j1#1;HEguwp9XLOn3IK0*uVs*e^YPgSn zy9GITUOOq142fRRXtt}BiPX9!f!D3b+EyuOlxf$b`PN2kYF|%V1MGf>yYM6O=nDj! zSD|mqGj8$3wsM$jRuNx!6v>??Z}yz+{J(fq%4I5;^DnOHUbv=upsFdqn2GG%%APv- zj+{Ei=k#V4(`>)n`?)-D#9n{Nc4yNtoD$d_>IKgweQ-)2_o||u%OVUtxB12YCCnOg zlT64Ry}?<)WwNyTeJZNEOWbCHF@qFjWv6twXzEq5|_B?I1kPr*4c`fSwt5k9MnOa z((6kR&G)7#m4;EiG?&2d<{)%_dwt9khw5y6P+&$M=%vC!Om<$x5#RlO7+m(jfO>xV zEHY33U{=BZCMtDxx37kCp+zCC_X>zn4&TWth|$aHP>r>VH~%#fgK%4!ltFFF8V8g5 zqzEot3jN`pz%sG3+ZHLX*6!6Z)5(}jHNV|m4Dr@hO-jM7SZ=o~4&WaP=#4vo=-U1e z-%MFc^8a!Wkx4XTZf2^&Qs~yOmwj+(Yb`^tSW}eatiFjl36F%KDY~q@O2~ZGB@X8~ z4>L1ck4~w}fcC7VBft8X*4I-?Ea5te2eCP|wKeH?P68lbHbb*aQ12<0uOIIoWuKU+ z-9P4%CUM+oPEX)*6BV)@)S!^<)h4zruRyF;gMOe$6-Bd!HcOo7J?m+y-8NZL0Jv+7 z#(l)%Mj;exmr6Y-A+pNgQak*Dza{3NI1I$|gG&{b@RPH-63)8i?sn)jVmusYA-Bey zQ3RQomU8!bDbr_rxQ!(!L4WT=WFAd!;CDEl<5Bf+yXp;x=~xC1t!x`Og<@33ZkkyUo=vKm%vgt(|75iLlNmC1p79YPyKL3Q*C1?D z?r?~E)2r2z$VF;t80S{rjC&3wPb}f~T0oAO0?&Ls4Q3^y{vb`>VHfJ*i0dQ3!`0i= z)bNIlb^)PZMn9~_3R4|?Z%=TzjrqA6t+UrXXkkB!$;3X)YK?hT`REB;J`FY1TOe90 z#}}3G8c7G^oF6|w;Cb|cpA(IRsM;goq0{S%TQ-t-3SiG6TJ~cNpA}$aj)E1-OEj4f z$EblRamOK?f~m@)Mt+J)nMPI%gFCa<0W{YCn{^5U{tA$X zn&U0P4!Kko7BQ@9i~dS{*4H*7R`vSqU4xvnEeDU6GLM6DZx#}g)fj4oOZYl3RkCqi zBVNCC5`RFe052C0uve}ZREp_VGF^B!eMWwpjix^LhfkvFPIlt>@*Jyf{X+daA~~Ux z{Q!7~I6Y_8&HkdGdbiV1lm&rqMxTAOVx)X|QJb0$r zZaGb$Lf6l1LNxIr^va*dJ{4JSWjnrv^^5F9R}mJ_(V>y%d_7($J5{)_#}?X*(61RS zu;z{f?d{8^fRxCis$K-l)N616v@$ch^>2Mc=@QPK8@b6lBb$n%M&9wNTCUt>QQHf` z+iZOvbBdzg?i_~l`PjHf<_?zVF`ZstRh0!V{qeS3oYdP(Pc5Yz;zn+L6n2X{zb;7? zo?cfq1z>@&V!6!%tMAw~DX4Q^20IwIxa<5v7=Zz94krT;Wx&*K-6WHvBe`JdESFWl zw(g9Bn7zT=D&oZR&Xi1RB{XPe+UefYQtQM$@f=YF#LzenwVfv2Ryh^g4yFa22qZO} zk>H9DFF64g?)SEQ-KqrQK>G_Lv(4&eMe~tz^36lrTtWn><4Kat?%7Bat?(lvxEeKh zWDYS^aH+7xzEsH2iw0#_jj)d9(zmGrwWHg6JXdy*lL27K&lMP2jYU+ehmvH?8*VtpNciJy1b+2miYG zsy2NVS5EypNwqm7%>L(V3)EDw6!hFGd*(hCC=rJ(DX7%)9typXLoJ+#*o~Z8&)_5`XgT}%o|10SjvsK|789A7_yM#WI<(2(5$A4#+o!y$DIV@ZGMcld zxE*hf2-#6M9Iw3qcaOhF1v3IsQmOZaj-sIwZgmw@@fBSIiL%|kUVAr%ww3dnnrvqAVI+p1MDlxf+fQweEIcnn5`OWU$1jQS|+S%e$@L9&yO58Zs!}5$s zCAe9Ltg$22>hajuB_(()`h;}K2S=?NZW4lrCyopD5+t8lV~83yENBvt(c~DK zE9Ta0(X;mSF5EcM0~jTe)f%n+jHzR?Q8-eTKF1J4UgRsDw?BFbY6@?k&%1ls)K!mH zkVdh<2ise&5|M}@-6L#SeL}*>lP9HsEtqY zcO}y82Aft4-W2(^lW&i_Eq>E^YA|LG_XcP&yjEE1c3~W_c=xtjZk_@FE&CCKGlGM;&X9rBdj?tsR#5B z4~J;W`?}O{xQ2kTv`Cif1{R0Mzm{?zmR&jNdYtZlszhu9t9=f#WN#zAFRgu7#F>~$%D50@bM-bxmI)DhHE$ZYD_n1I21&a1720hM(-;qtvMWdqH}~DZ7um+K(m5l2n^v} zp)pY&Lc=fh3)C$9?w;vj<8*N|0)@3t_%64#h5?QAw9j!5aLBxCG9uU-bRfj^d+!jGyri2Te9eVVi%_-^U(tIv-PBL5f`?i7o%FX&T0glP+% z`|7uC?|u%YrM!qsK3yBVmAP#a_O}*y%jJHAKU+G)&ON+n;s*Bc!(~ZtQ@?wsQ@Cyp zsY<;!{_+PQ=Z~H}+R*-&{r3+z8Bi+$*YjNXh20v?Evqy;Zz}FY)`aebXIZ1eHb; z=wX>mPx#lk1UBV;FyCo}3Ge4~qP9G%OrbD;-~PS!%N>cTWr6R5*_BD<%}mzc=BHRt z*eXCjmnh>P{epb^4|l6(Qi410&h+!VmzvhCkV*u*b#L;-{d;02umqFo-4NnFRCPeG zkGd_=c3GWx>#RjDZ{o9!Ra7GTRPy0?q1E@r6xyBa&%DgY5;Fr{VlSni z0iIQzN@z^>yGfSKu``}aYqV+4mj)EkV@1{r{Xqv{(odlt)OC-U?ryyppRMAseetKm z@H#7hM@*{4^vvNA2V2m-Yr<~xQ)d=ysf>q8b{Qt0ZEuj{;U+fQ+i2uLiURP=miJpV zfhl5!S-ZbHcjgYzW-L)(sO!6QNr>)f^-M(Z%LJc(epPvk&?b|~*LFzuSMn0n<2&JF z>gZ>3X$4H1Q)QZd^%wWqgQODCTH-V(=L0wQtu?h8Ht9qwDWtem zjhCIj=8)|?RMk2d^yKXG`lo_HrSiqdRAvKU1(H$iw5#c+s|VP!Hpvk$H_BsU9bI|( z(qBx$$Z4!p#%3hwpiK)rVr)URSnbr=KsopOY6~bs$SP~-EHTbPubL!{!@79}4UJc>!jnV?egD2e^7ImCNt5;PeI^#>pFq^5{ zOATQ?A`r%I_Alfn&9ZAsB#8*y_lmvusSS)k;UcMFalf0V+86NAT zZ|UiDVQi!K17kquWj8AxQz|;o?Weh_Sr%HPybP<#V9rV7f99u3ZML5+nXPA;{_3ovK z=;-&hbSLsTKhj>~6F?!1%^SWCcx(l&{;lkIGA>ZFC{!kXeG6_s%+yM6BeDYCRKhY2 z$1Qgk(rg^R+(MJi+13jo8%MC|+8n3@Hy3P75IH$#G42L^067V4+k{%skcq$CRF{p<&kOkwXkMc zn}a24NWsYiDxi-|asnPmB^fL?99oW(SI*b@e0)6L-~>AQ+R=2nuI)P1+AjJt=Z`&iPQUAok z9e!+RehVlzcm>0fMI@!6Nt*#=6NEk{%spzhEv>KAY_1ip7HM@g6z zPrD5Od&}IVowtZIvk>`Au2}|`vT7RRmCoE+G)J{imV}}6X@mOKbowDp-Z8*S9e%-s zSh@~ghIr*7+zj^i8G!|A)ieU27=8cq@&)s%3rvryUB}xqelVTQ5ZPFB1)C}^+26AM z83gZXYOO<@QZKVp6ZfX>zqhF&_E4(#c%tpW(o&qOe=Z+NzceXlG$NG$DpZLrC^jYY zhlcpeJ)M?)K-tVZRuKQ|o^*FT(#+WS3WauxiU3>oGMyIV>FWF&m-;WsFXyUe&bLA$ z%?nG`5SR_sDNZ?)G~2-%(R&`lu+x}l)VzbKdgZJpp7l=2g36$~y`PnwV88y}k#-$h z`Y*Qida0ygH~yUr&Iul(K@<#LaCJ-Ti{asq0YYROue8ay{QTr5u{zc@IL4EPEVpDN zrWRv9u^Sp^1>7boTVAF(&!7|ME=u6`As@Ta_nvsyd@5)1FfpjnZBQDj{P`m~WAFT$ zZ^`~G9CC0U`A=P#*D4$oDcXWw-IfasDj$BI2;FVAZ+XjoLRKLRjV%tj2TlOCe8*q8 zia|AZKZcU&i!$p8ZNpuiQ;}?xUaAWxSzFWz$ah8TEio*)CH}R_&&U2WS~^q-LQ1Nx_5eTw26UboXI2&tl!!nRC4YUd|!hX`=32xc>?oxMJlIDUu0;`RDNW4URm}0JyVOB}=`GFfA?*QqA*zC<6FTb9y2Yy`XBw|2mg}koV>R zcPBW_clAfm`r0}% zbSJS1y!$2v3pzsLC!el;;t@B}r)9ZdmbjOb=E}(+p+Twnj24o{J9}Is=cz+n)=2R!2H(!r{fa-ov;wAw@lg(DH@e7+a^yD@ zv>y>Kfxw!Q1#jtmbtMDSWxhz2YVgfJl1jDb#YV_Sc%_QAXgjF?1Tzu_2|P}+1!w!` zevZVfX3KOBXlh0hXO_-lBQA|qNSzm7F9d4xTI^&|%6MjI+$N-O=1ALrgkQF%&jVbW zo*C6NN=W#7C{l`QpcZRoI7jODaQd~oBkH_ic-t$AOTu?dQ9~|9dcBKO<46+SF_MA? zZgCHiHw%~f{?N>VhT(%TpYqTTNtF-ku1pF4ko=B-B-nwOnU689&5EQrWXraHcFF@|?J9c~dJC|wX1ls8${&fD!-U(dfBOR8EUpL_#7?$=CnpM+;YYMPzV1Pdp`j~bvCkQ! zB}U<*H$7Q_Z6#3~^RN(!#iG`;2FF_+mQj;L>hVo8da?clzfupf_8FK!R{Ah8uB!a0 zVz->#(;IjRtn3r!-*$^3ouPyF@I$>;WpMHvpi=#iSLAGjqSU2Y*-QXQ?My{N6Jv75 zQ>!F$c1ug0JN5Rkbp}9m7Spltgd?#~0`ioclvZ5~`VOUBvnF-dQ+IXii9)S44C<94 z+hOGaPm$iQJgDQ|Cez!Li_3C?vTukzS?>#N;ere2;z@kj4YG%ibcEy}YcP~No0ym-v&wkrVw!K zEDg}0R%I3CBfGM-m``=p`28yZKo_ee;E#49@&M6TcZ3gjAStidkZ?ArA|4@ZfHqP)&Lo1Zwr6BU)`oko}!KWi?xP8+&H%5JK$uQ^9)~S zsJF7bmZMTf3B~ENa-U4yc?+<=LO5ZbEAvyXiHWi;bCbNcM(+fKQ#=!#5|3!qS~^ko z6+4UdaPCK+K;opoo=5L5TOK!Ej82Z>cK8dm z!b;?fGt~^ekcr2kYw#2nVVN@A>4OVEDZ|5Ar2)mUBiqEiF+p5xE6vcoH<`3%lpklf zoyTBPm4-bY#U3taj`w)~pd@+So&MT|XMH3zx5}%yb6)<*n_VY~HkGN-OsUrLOcVl8 zmYFGfW*_UAk1j3tUJ7jn<;Z$D(d#|(+m=J!19nHMIWbWSFLvvg!#V4x?*YktVzP*I z;c&tJ5fuq2kIBlP>ll^8e0qCS`5aYsm*9eNI<&?oM2Kkk1|w>i!D=R+_M7uxzL3kJ z!TvOlXJ~{f+?_ClXY|2Rs=Ci3tR2Q1Z-dmpH1X2sHI{ZFjJ7o}GAF=~;*BtpiiTu! z-!tX&p4fSzp>3y05M)em;uU2{MSRHOV*jIZ2eIpe-U!rQNTk&jS~}G=t_5Cblrmoy zIkFzP|3J~(Gi)n1SR`T7*19map0ny1)0rk9HBmRZlc}H({^f3$fx6h}xj}N1wD9aN zD(ZNcp?-n8#(B{8iobjv2<_zG>>Vc#^f6%{Suwh@F%M%p0~lVuqq@ zqn3P;Lluh7)KSuu8Z_*aJX4dsOczcQAq-0nGrU{G@-KmVN?J1`{4HUurIty~vV_`Y zg3fw~$izjt+(UTb??lg!b_?(MPgl|g&WmcM?Ltqd*Jk7{uWdY?Qcol|HBx`7g%JFI z?7d}Fn{Bf;Tqsh!xJ!}ZP~5GwP~4paD;nILQYc!Ww79!NaM$AQPH-o<6XfN7-hDso zd)~G0wY$FG`(M^da^w`UO2~c_J&C>ZAGSV!NN(2afUBgF)dr4?G`c~z%XKjP z4hfUqasMvs+Ip@%fhvA@Bat{6!4Ce=D^IN0k|ro>KJ}_yw1_=U`(iWIr*&bT_bxx( zquU6iPf*cPmSs=<3-KRFePubg#V29F;C)v8Jj%1?qlb5n_G>!qQrF9{oJEKot;#Z6 z$45%;yr}m`t)!b7vtZPMk39<0VvPmXdaTNRxOw8Z6%!}Qsib#UWK`wdTkTf8n|}@Z zoCFhZbk*@u`0;i6^(WPFuf^(CJEz4jkt0vbiwOKZH@_x6&g;lsY@>!dYL-lQY7R30 zGBdD)KeiTCWC#BYavslkgjm;FXM}I)767F9Oks~A?y}v8D@^Qe`Jx7}8eSGwhk?O!Y|{^z9o$yUGTP{$ww%Of|5o=)6o7c<>bA zt~D_=sCyIZN;V=$TiR(|JsMR%W4>>#mR93i;n>EYnclJHDJPOMEKo+>RdhXcYI&T) zYc2XY=(Ahv6Je=tLFmSL%#V}|#emW2TM^aih64esSy34q-MXSz+vi1ORpzE%e0z0n zC8`pqGES7Qu0AE$*8tP46orFF|1VWeV5H=gd$&lBJjD*>Qm^3kG4#3ciM4dd^AXRH zG$oxX`b#jm+!ri;_1L6MvKDkNMVAX~;KylW1 zb|+FXK3KPy2h$&~#m_VH(3gi5RQ%UMWOjY`L}%Rra`0j{Ga(;`Lc-XEN(KOHzZ3SDx#y zNgdyN@m5~D2MKexPnyY_nzGi@Q=DLJRu&;w_3q%dNq?UeM!#9#EuS|SPs3ZX%vi6i zvbtjMaEaDneA>7aeB`*~qB%=gaeEDxGc8O~rD@L`P3S`F50%*VRu@=to+Q2JMUCCy z`$^us+jGwpP!Y(dC1zwfCpGFyIjMV6V!6_9?s%Kw0BE&+A8JLB z7*m$-EDl_ty7&2lWo;VR58%nydis0w3g z51+>9?uLr$atFv|cin8e;F=o1c9YTL@q)HmwNA-#Lc(G$d*e5Y|I!>a9#9W5AI{cq z`>jydMMT}sWyjvlsPoQ6dQxq_-lKZ?gys`N#(ZN=#g2v_@p$$ z!@)=%-|H%2n)bEUE;u0ZAg&+NJn36#Rs+7V zY%tZ435`T>E7Pb+XH@_j9NF6DnO7mY?tCXcy1XaQ@Q@P3PQCl6Thw&X@6(vTYaFit z_CVCVzcz;OaBm8-56<4lm%hyL&c!5~?9Xjz61-C-XS-SEb+7_wyM-70JqrM@_pG+9 zZQ1O}x~f_mJN%qRB{eu94fF8puwxvlf9sc|l@SbL|LUy3E3q|`j(fM;Fg;T|5dXF)zx(xMuY#W2P-Q8X z1hmv9GG3Cg0{E{)swmz3U&cs8#?fl9GWA1!wS^H>h1ZNtg z2b^2J_})5Wc5@ZrX*0qmVt!p$=)vm&(v@&A%T#+4y?dLkgLqS;<=LB;sfRS2ZoA!@ z3SuDu0<sG@z5$GbXLC-w3_FXus~{KI~ZLf$xljE@iC%EBY#f z;h$`C6p2;mX_EXxpPD{69o@8Vf0O^1G%{&71?H*+{U(Ba3Cv0izqs2X+%=2uEijd0 z$C&B~@uVYh_loKpVSaiR#)Q~DX6Mr~U86L$XO77)T3l=yQKPDP?iQ)Yaf6I@Yw^z; zu9Gr@gjVvR0amS0u}T*O`}QrQR7OEoD4MTFqNgRK6CS{Y+;Wl z{5dTLDK2XmFdqK#1?$?ow?|Y$jZaB6eawcs$N?RP?>UW&+TYQEZC#%KlJ;%os(L06 zbpiN@$rpheD`of%M?jccOJ(Jd26itdrABXY5VMdo5k$%n%dO^D>xWhf-C-`7*5Q?% z+ikl^Ey2i1K;~f)iuN#v*v%9`6larDa9sv-j zg}H-W{7>q%HfkVDX{v%{k@whMvW*UsS{PYVw@WJyaMP*)2St@9Qv zw9&sReGPk+*kAAY1t+Z>D+gUZ&0bB+GzJTow4U3!F*ImtW)cC}8WW%*U{It(V=xsz zSvEENWYG9Ci7$E}(h61?b=mcy$;LamcHD+fo6;cpE44P+5&z-D!^@rB5iK??&GcFc z5h`*qK!|e3wpEQR#*Y_QYo>;#>eDl~b9R(Ny>rS?Yi(xDf}W2& zl$njCc?epfU)x^`k=Xj>Y~=wU!bSN#D{+~TG71-mSHdkq4yETJ90GuNWGS9_pW;!% znG%hjRrT;%K5jt0An*5`yfNLy5k8#fN_u8J*5}^5|Gog31HZs?ntW1~df+~0rn#lt zAzVc({kVSF()8?y@4j^q;*wFkMYm_E$UALGc%b+qolVN%@L=tVe*RX~iYqj+9i1Y- zTP@zw0YOOMqQ*}}EBs4kWee%WiF$Y%2K-s;WvP)YTl)3%F^O868`-{_a+Jt@;j-r* z4k5eF=O6^`7PLl;u)!nJu_kqKL=%De+NR2?xGCt`NY)V4@4&dCdOIu52k9VbXgJdU zCn(P+gJ&`)aMPWOw$$f!zS#B1r0)@@@t;v56pf?~rwNsm?Lo{^J&>3R=YDB^7RU-2#V?%7|Hu z#bd+6cbLM{4^VfpV@~Ak_lc8y1D97{xOe))0NamNPbLwo5AtIoZF_{3VjYZ2LtCT3 z;@%;#+4PIF#syQX-(}||y@7ufc|Lb#%>S#%^P55Uo}Ph?X&|9eC<>L+&Dy8H21GhL zqjToU9(ydTh3I!&hf-LxV=E(5j*Et_g?;x#djgA;w9_nxyBC>Ogyr?n^xMk)A@=N1 z0k}day%PPol#vS7Z0y6hImSPL;nza`>0N9Gy4&9F`H%FyZ(j8xtlo2%V#4tc3t3$R zA_NblP8lq(Y?ME8YflSnVEJ*=770yw3`{qIcIOE^C?+V&49qH;JyrH&Ks=l$2}vg0 zdl5&|C?$CWWm8UQnA)X<4J^6g!~_l7#>I5;5O*W_uSz8YMZg*Lct!N8w$`rg+{fIq zfKlDOAgwZWE0;I5YJU-WYmHvsC7$!eCA!6mNxXIWHX_AZH$exG{H2tyCvaY7GzQ`p zs+T=3>1P*rgWG6&iTavtD{W`f3k8fo* zGr4U&rr+2wQpE~OtIp*!>IT2GoDGN96rX&skmA`k@$jX^pvcp!0^t##!sq7`7Y`-5 z5SK_R99NCs-5jl$z#p%~5_@Sxg6mrJmK%lw9i?};PQD%R(B{f~NYsc0oDK$r)LzoJ zUTzHzrKKpQKSvE5*qz6MugBv(o|o9uw6cLsZkspX3QTKr;c2x?0jWl2^3~B=^x@2W z@trrc~t=X}Nwm}`-ozyF)fVNdfAw;Zfyk^ckr_cvv z3&R8)PI^< zvYtQ=PgjJS8UhaFY%^6(u)FZm%erG7v$Ng4CY>wtfGHFoo_GABT`nR$+e`SrU@-@% zT}(!jen_Dn56w1h!hXJC4e3u^C2h461}}GxaqrGL?NWYqS=e+wk94TrXf-^J2Odz& zOKBEzdF|hdp=0iGT*kEOn>eIYvW+1Uv~=m29_YDxCOn(%uBi+fDKhvEl}G;CDB4Je z-wyoVx}I7GUyM&GvQOZ08|NgqcBV8Wx$Kp$MT%rWC#BN~kWFz$Xc-*P%_0Ri_r};W zJ2mtCl0@UJM(DLb7I9AYDK>BS4lj%GuMf(`XL@sPs&bPa9m`EDfYY7pL1mU%gOn{r z!VGVm6)wFCf%o0gWkHhi6m#_Q!0lYvW?C9Tp2n=0eNm~D0Fp}e_-;vG07_CWpd@|< zSSsIpE>cd>F^QrBx!hgw!&*F7<{ll5%mzFqQsvyC$@u=d6eCJTk4wu@Z(!b>x!!G6BSKc*DK%OAGK3(^5ciXKgu=R1c@%V~bY z3Rv-vtW2%AmB^MoC4j+KouG;mMjBdU|HSpju z_V87QnrBrSG#X$nk?V}Qbs8jRH1(uY%Hnr;<6bsAXo_%U=a(wg4+SZ%?7!BIvz12X`LNwH%lCn{E*`>1|{NQAwARnwqoi zzWW8PJ2Tt+<)a)wc`()}j+`;1n0^U6S1;>2NqM2f2Q$PK|t9P-v zB2et}$=+bsm1l?aYXexb#?20$;L-~)Exgb2PkfXyy&-n>><4o?Ap{4&oGkV|RZH`> z1FNn;E0JxivH;$^|HH!S1%XI$9p|BmJ+-pz6n>laae{B>LOVr5ih?e;(-0FXv%+m{ z18u>_2twT2P*6#ToXjksiehXCf7S{LIV6u#o9@|TB!MNNgxs7UT^<mlQQZ*p(u3T7;YxAg05Hp+wgY*}H?n|YeCLm;#_Z^+( zdofe~=2^EQ3;jiBk>+h*k=upW7(+!QeWRARbP=+%BIoqGmpa+83Zj(Q+I7elkeAqZEY<-7;&>q zdx{fm%hli|XUX|zH9eTSz>?3`0T*rukNXk+&)Z>An#irwV*3z3E^ z35bhA(fWexn5nCT{x<{AhGM?fvMS4HlvP9?$;E2vTV;;~&cl7(#CoT#R)rIckA6c4 zA2iHyxTn|U-43;Sjc6|JwheYi824ly@oCAy{%UNK=gEG#O+BuTkj|{<6n9~xpE3#7 zLmCi+-BU@%BA}9OP_O9+N}iaLw~j6EMZM6#stddmF!SoDS}VEQo@7?uc{BY-~ync3fav-RHlazMuN^E`S?L z?w&iH4h%Uot^zOx9}cZ5WN5>es&_7*mG>(ynJP>O)8g5ifg$H>^@JpdT9mJ28%0Y? z@K#HO^|4}U#h2spYqId%HNxq-25?Y~iya;T#%d#~AHk1*5>G=S+n?_IwsBKS`nE1! z^JiwYd|YU{&IoH|)hrCPi4g1Q#!hZHkjVw)6N@m;m_1!L%nX_H}q=M2M~9P)OPUbY8(kK)?~gZ zyfLNg&Kvo*PzC$ZD&(wBpUhi`eiSyTV@|&hjWlP>1^g^@OL!5IpmpQi;%kg44TteD z16pupy^=5Dv9d2NV`>+mkX&f9FCcsHGt1Qmg22~pYwm5+A;#-d+6?GELuBcOEqv81 zoqD`?ZOj1a)s#rS_JUd+itenp+};^GwW7)k(XX`f*Z`N=^4;V#jC0x&>v{POTQ8pKq^q7|jp9 z+LAjP>1G|YclKRXP}dx5Zy(-EA%5#+O3NN{iJ4&3AiOL&tY8^W!3J9Z)7p4#E_S!| zK;&b`qumF8!5f^xyJ_jaaJ(AJ1N`XpvkSZgjlakzy%A~VHGZCTJ2q>OSnEA2LC#A~ zo18aQy=A)H6uaO80<*QK#gZKz$NlskPOk#cCu<kSCrpKm#Ip9iCea zbZ)b4S9V(|9H4(EGEs;N&celJ?Ladnk*E9ab-3k?011cx;YU8`tkSEDL_uE{@cEk^ zE)+Wj<{F&$rZY4Cw)qtK_lYn25J{-{@>e=GU7k^VvQ z*_T=Mdk{M>%&%l5@Ml0Hn@%|ZPBkx`%wzg&dp<}1-V+oV|G#hkzxDEed*i>}{h;zC z?J6roM)Q(&?8D(*waWWPiCBW589HEL_K3#QpyxrQ5hP}b$ywxowAugX!s1ZGT}$?> zo)~2;cvI7S=Yjj;8*Y`VZH4_z0=(yuS%0s>|M&_4?R7OFw1e!=zwy^oao|3Wt#y2B z@;|KQ|I66D!lANY4+;1u`uQI$VoWbS283Go6aPa&&cAohfBc+3)pM-*%AfF0snmab z_}^Eia)p29hezU4{_XjH^oalaVE!^`xgiBloq#J%XC0_8{$6jRCJkKbuoZ&d*C z?Q(%$xj<#xSfEvnmQ{_ub+=DPn0hs0U>QMk6U(dcNj~!b-qaFkD_E5_cGROMdHXb5_I2@|- z7kIpk7k~J2)e76*z+zNA;`nOQaAn7Xzq$z>pYSt6T1YDPw@^2X?#=%3lV68dFAWT0 zwF1|}6_QXN$D>0~8yk^jnrC479_vfZm90HfIAMqO6=@VFr+fYHY=YnDd?^z*c{w1* z?69VfF~GbC1}L$!Uy$R4-Xs7UgPlxvUOCTJ)dLY=HNdpAb)C)D75VC-Rlz^Jckwzj z{p%KYmmK`d+}j&th^V!`!QOig(S*}8yd@s4B>y+DX(-1z&Ci1+RfcnbAIsG3KiZ=e4-a-o`w50rEATi5 z3gxdx#5$ZEE=|b5L(W;)ZlF#qURl~rN)D2z1KXiT7#HE~7Ns&m2w=#oTU{keN_Qb# zCT_s*J(F+zE;$H@JM3A_Stvl+Q_`2@cMtj*FsmBaZT}aF9_8ZKQW*~It9|Y+WC53xnx?8k^n6->k=#i}#ttpF74YSw zOUFE;&~h|PvB}>>fTwRjAhP7L_f%IOFRsOh>sZEysf>rg{;rC>;w)4+*ei-;ws%c* zDH9i=aZe}3tzLFHQKHrUfe`o57^|fjj=hw}($3k{W`f~<>*2Gz5PumHJ#zZP7FF&3 zH;vN45VYQ`mF18^6Hh(x7Uv&6TQ)q~gH5yOF|9A36QLvI+t0+{9eUrdy={q6R5vS$_73FJW*5%mrMzPqGs6&b{U(0xJ*&Rq z-Y6_in2fWweSG5ReOT6>#?L6hcjYqHYGtkSsMHM09O?V1OV5{2ba1@YBeXqNM zY)^eKIx6BsS^V8(9u$MKg1E(ZQAg`^0Aj^M_n zr6ec!X+uM-S6x4L)7Z4SUjhhsj1srxYJ>U@tpgNGjeYw{xScL# ziJr2>Y|cGZEYD3dnP>3ulTdPWos#)}yNuE9W52`mqgbX*F`MTmOq_)%?sk`AD2%?> zRSeJz|7-+c9ENpUi=z}F8#PR z5}b<_)B}?lkr0?se1dOs$*YYgv#4uSM!2p$X#nl~#{Z)jSVjj&pV!M{bna)}ffC$u z-BIxIA)23)s6zs1QP2{Y1;v5-uw#8gdQ5nj*2+zYpEGD8P(y4ToLWY3H@=~or<^qH zb)LAxtf?aC6^lV8G?5N+eX92OE6(gy+G|x@nxnbAXFn>+ZArNxr%S*mgtUH@(~1%g zKGM)?vix0!6YHg-lAs1d3b>I z&1;8&X9nApQo)7n#_ww$!2u|Gao+oJ?P(1P>GxZBd8HIH4=dN(NT;cPb^iVo6CQ+#@_tZ zgDCfyP^O|x58l@ndU)p~$s`3-H4fR=$|gEtadb}bH0ASdoPo_^Swb7e3 zzoU@+c=C}~6gZkjQBYjaRp{gK30AI}UoxZcZShRGi9e}>DOgo&8I!z83jkI6DC#QL zIo&=-XtpsvuhmYr!sdFsAW$OI3L>%R1&IDEv6CgbYs9X<>g(UZz*iU0m+c<|;LR!MjH!UcICL;eeCMuqgLn_xCys90Vw zDKtWA+!ioh;fk8yrTw9y=Hn47RQEXDt7Zw1^t=JS2w%Vl8tp^fC1;fepr4nXK8rqH zY_QIO`B2Sjnqci`jj|ag*F~VxlDngn5xo7igmV+5EB#+C*D8=xfLcU0al}L*JX(y-?H zq*@i$1w&prZ6A6NpG8|5`{9+I_Bx+OFB~iIO!tX7!3oUgGW3;7eKZweOE#Z4Kb3Nq zr$Ryx7r#Gw7l?Q{#rDC(CU}7{XtFpK0Yw-7WzUGVT=72hfW!Tw{^uOR4QRivde}*< zlAY-k=p?w`RB(+s7$$D)MYUv~bOkLN1E1N}=t~=pk@S|pLF3%`UD-ho zyq)r=kaypS7Aa92{FAczYGc+`(?=Ygl}kULfAp@?qwnXC^rB^uG5X@i?OSX1>S~1* zYeF_^dTH$Y&lGPmh!4elu$+qaGDXcE| zJrOV8L;K+B-%RgQFyOD$3Bs)f8nGHEOU++ggIIoJ4i|2VaGcY>Ts@qBx%}AEHftIi zdn{PQm6VP}VAJqY%>;UIDkuMYV|3f-7WMX+=SsR?x8w4UA}xb`4GK}1QVf}3#Cx65 z_r@HI9f@MB*Hh_cRgEay)`}f*?58Wta;uZPP~gW%k8mFo$8;FHlFgOB48o*5^=p3E zgeBY`sy47eHeIkaxy4qon& z-di|C(d8s(7nv~d%AQ^*N+I(;>-G7xxL@EG+jsX4i?jl{RYs+LXHVvnOf{B*zU6~j zqpm?h6#McVeIC(N`Z5oi$K3Y^@lRA{tdzk5ToJ)SkV6ZXFnxm(hE9&uF*#_xa(-uV zT1)fAqKTi31=Q|$rqb2MgV^V6<$ND{cKG;;#=TGwAg|^ z^`lb#W@gv$W+#Tn!~W0^hbblX^~)Tka=j`GKjFlN7oFWStn4k;kPxV)qbM8!fv%@tD#LF#H)2Z;= zxOvYeoTS$8ta!>Uo&QWbBE((rqpgkbG)AS+Aq(zV!s&k~#Csj`W_;wH?M-2|jZ16- z$#viin+ysOp{4MrVvP^5MkMcZzFUSal;S)P($!FJa8$_a@U!O{Cb3qvKY}RjHS~rI z)mk1!`pf4p>>ggHL85^nZOlmdn6ue+QNdORrfMR0ieyW?JY@+;+$}F$2)pnc1m92{ zz@u_4c2Z+t%;+t|fx=FD*xSUDXz}tL0!=l(Sq$D?C{>7>zS0ECVZV8EJ{WOmUGkw! zadAE9CXspCmoL`j{M=wY8|zZ?qb~Ans7mJyqyjFPGUfXUTSOh|TcE@P$nu6Rr>zp> zV1}RA`L#sRaFk8Qc0GgeZ4ePLr#gN0IAf&l{s_2OM6CeV6E27zNeEzJA_=o_*G2AC zj2dt0mE+_*oq!NetVU`k${I3*Y=Un3yMKarXmsc$wbBh}+4a7&>;fDx*%g=axE5kv z(5%W&Ge_+$`teHKX-7N*MjUx+OYZ2G0mt_~xp=xaYYr?=C(XM%hgs=@v#B_JF*Vd> zlt?$+hP`JP+F$`sq96Q>2p;J<4cCc`peWOp^U>fidFEd1%4m(IKK(uEV2BGFF#MNy zre4`Grtj?3PHI=U7~~r+Qd@Era4vMkeG+3U*i5K@~v6}HwDOYa}{$x&t>wCHsC>b zzkfyTvretSiKKEb8!L>uA?VN2$1)k1Lb z^$_<&a-2`hjwJH?>dv225}RebT6becZu!RoHMVa0D`md3DUX{@@$g>z6d*s6xXty5 z5*2+7&04cBB4-^cpH518GFg#Ndd@~qp#aLYQ=Q|`^tuzf^@n--wyrq9ZNW5y8Ys4~ z9$0b}XcuxPJ z&+a04#Xz{iPsb(oPNKEZWl?%|DZ34a5{S?#mI!^>z7wb=ryidbxVJJI*{DI8Fq#6} z#9F24(et4j&|>VZY3h{Sm6t~#dgYkHUsO11$Uz`M;O`#kItwJc60a@Nov09rWpPV< zN3P*e;RBJ9c#ul=5I2-vD${Y%!@kBZiyXg!HJf(eWyFjPUv0|eKFZWcngvg#+KwOz zC^n#_D<+P9VheV&jA!ct8N3Fq82^r=e<~Czh2}hcnqsIT-W%jI&Q{c0ICxhZ16vaH z=JOK`3I{1Z6!ka^kJ-a`tex+yT^a`HaYY}vDRBv;LDmDk`-MUPU!-}$xyEO&lOXeC zz|21U3ak{*I}5QrEC%;+_umzzcq#EM7gt^_mL2zL#N=S?1_6&g zNKCLBcF0H4JQJ%dL3Jy=-3+gt)d(+}SrVB3$-FIe2BkT&ts%YQYdzr~g6>#&r1xIN zWW>>zj@ug1z##4|WMbA}#y71MVqqWG>n+eDao>o(thvLG>-stS`%9X!bHjpni3f|` zLSRK(;ulh6kD3p9BqT{==E`2L@_9F?c+F+8@JYRYG%^wMDTI9ZVxjfW9nRk%MoJ0p zi@i58U>poo+mZhi3Pp6QQDZ#coINxAycrR#30HOX7;}6l#V5?OH_upaqRDa>_9gRH zBi0GWFRez7epf5t*X>($LMNDu+f&Tgmh@BHHotHz-gn|E<9FhLnIlJo-7{EQG(6qa44sBYOT>X6yRivxSR}pW_gv2_w5#|+hmu@!5Kh)#_3ey@v$}lRD)!2 z#MHYx&#UGv`!X>sk*Hfv81ng3;c!!>A|Z)^|Ip@%MAd;a0Yy5GGwb_W-UAUS5l#U@^9~mGj!Cc!?l&O4 z7|x@}C~iS-RF{Ij!BFjg8;LLJe|~5Dez7fk2n6sXRNFXOqV<~TgZHZC(J+1>C;f2> zm1hCWJ;DMHfn~n71N~ioVwr?r(0gJ|c}L%CF>g5hVSMf_(3u(gWDN`zl?9a99k52tZc9pa}Z zsHw7~`O)C3%+SnzyOPq@=YAWB(Uo1rNlSBaYiR#e8?ZE?IL@I)9Jz;3fMaWcMpK9` zf3t1pQN%MtY3#>^i)Dp?3BSI6I>krf?!B%UTG%H8v`XHt*MlMBdB<&2IqyZKxAMK@ z>aB+53@QF559jQT>8m`ak8wnGQ2a~_s|^flY#+bts;}2~$r5)IsU=9b?4oyfHKCH{ zsca9c9~ucC8$i6hy2|@zGT)19^V`}Oq)_&1jG+41tZp+P-8La2afU{{`=UF;<&@}p z=52{@+0fCe^RT5M+4i^9T3&g560Z~m7H|vMbBrIOV08#~!&@4u{6S(PkO2L2)KWJG z@)YdZ4huPV`G)P<109OpQ zt1ez9QkUmjHPIWJ2O702i(TNY)rD=uzcY@hfAsNeTuUr0>N`bFUnUYAA!B3u=;si4 z*donS5*CX}SR7p+a9Y zu1#gYC}{Gs{J(kHI$6DlfDB?k4$DiW&<1rY5ZUjBdnV>HrLY|84Sbe4I2=nx+#9U?7G<1q$0T$6RvDAh_t<%8?%tn!Ofx}+T1pIxb>J@&&gGo<-PZD#7zNI3J3<7A zipq#6%^i#r#ZUBR$dR03i!t*91f&QwkaOw+xQr(C&-rbz-EyHRls4%;=(#k`y-hH$ z8L!-p@&K=d5O2_sMQScQGoGrczt9u}ZUxm?k&Qo9Klm)De&kDJc4=CK$Z$H}Ub2$x zLfsV9fNz154@cKsUoq-TG>zGtj)EiPdhAx3peGdHQ(tE`9v`A|HfIqBFV9Bok+AhQ zo6O*@tX@Rz)YE;4ucN~lq;wi<&B=6cccoQyGPzOwGnr`i`xn|Hvz4~Z{B2{2?))2z zUtV9mSa~+_42Os6I5sYC>&+{1-c0)9n+^mBNla2hrC(OzZG6La`rZ9Xy%w(dtvd#B zD9S{KQ&Gj~t-MFbTQI>9;dg z!{iHN6wxyP9L{N${<1T^+w`ey<=2TBIYPmT>}IDR4QG_h%y&kWtHo4nWTrP>x#FPV0e z^mSU&={5`GPIn(8qMMb=bq!LwtvT16-WJGIF(AxSRe6~*!c}%@!Ev_ci=8v&epGF% z^|To6r2+VPs?<38j!xf(5Z;Ehsc9&70di=oFE?Lx0Jh83gq$NR9w(VW?^_-jn$CGy zUi*vm_v2qOUavOiY|JGTPL(~7q?F*kn-%@wWuqY41%=r9#;yhU#jF?Jk{JA!0^v*= z*a5G;`LCqIpWYOc(4>%*2E$bT3~6i7WZ(}+bGH(jm9~L@7>|$Yc;mMZ~1}m=L*d^2U)31$4g!BtKjtH;n?y0&cP5d}tn5Y@Qv-tGx z_gBx~?A*%uWD^V>RvKE@VAT|4nR+?PSfYqsbFM83CDZtV{%j6A88(t;8_DE$)04%~mz3z+tmAybZDgIwPCKMPKg4p&B+E^h|%(}?&L z^^^7JW{qBg|W!shx zZI_B%M%h|h7nAbc(hOk$wdimK!Rw6U6cnT*2}<;I{J(q;CZWF~iA>S}CWFNtixt8k z5jhMjx<5m(PbE&y5B(R`RiVH(WKyx1r^w7H&kFIKhSdxmI^sn4y(9Do*P58MsBS$o zIbSSCWfZz#(g&gMXX{mZns5!A%l`0cewAOv>~Q3cIAS`LvHsOxJW4~tp`Qf1@mT{Q>7Crl z{4(e8v+yNp3CWg)3c~0WSY+XBvJw=6B{jSs$5dyK za546-*S|BvV#tej;n8rB2e479>0!Z!2`CoFO}Y` zdq*}+i4C1x4m2Dm{j^isLvZn*ZzHlU0XJSt(%ubLUH1&w%@@N7 z84Uh9qHOw%n&wNZcnR3qS)n}Fl%i?Bz?`xBDb|EKs=1&5VJ+=z?of{$9`y^2B13@J zbHlPr{ORqCjC9VWHR$%IY&^mOF0*1S9~CFO+OdZ3m%A|8lwr0Gx0STcOfcGyWX7kB zx{{=@3!EwH5w?)&98yhGDSdS;o#?nr1^lg>?J9Kdd;K)q7afjYj=s`Mq1Rxcx)!AJ z#PU-s(Hq?maahIo4AReOnUr>D5X$%IrbEuE*x1OhT0C*zcJ<}+L-!w3A*@&i;@^Mr z`Igj%b}SdW9>_{-bSru@2<7qlRN^8E%+=T{kkl^CU~!pP2T_(VblIkY@ZN|6%p$yH zEs-yol6{(6)V(ye2>Om^W86bZEPD-%7iMM(%aTtaZ6p1IXrUb#D7n&xOa4t2M z^|W~psk~ge_;VokO5{9IQ36Wa1sXqJ^3ux0#-;}gST0h^QY9f+G>+BJSBHF4$OIwH zMuHOf_bdP(OS0ekk2XU^@7PrTGzhQ8iQ#%7BT@t9k%`6FfZwf=xD(kjZ>)(Gy?t=> zfPGx=SW`}l8UvZ4FVCwiRZc6=9EW1;0~ev-(`uQSiz$i%8CasZQWDI(i=39s|7G;BEib{8HKPatOVeHK|C zWnR9U^UUFNq5*gx%Sz8C&3$k7(GUzqGVTCnci{~vB*8AAbYFR5HTauZNiCBv^8M9O z=T4)jhrDyzib~FoNA{4Foo&@LBm^mOo@tVydpo&YFZ!V{4*vbr`ncLXeYt3$g-@V5 zwc3s->RrEhpp2eLQ%4|KYBZNh>2jO=F>x@%cMg*8R?M|D*F}w{Jz~>42O^zcdD0UH zc7q|NWfwz`qn;YwYv{9qYZU)7@Swf8xz$o1EI5QAC0dJ17IQFN=ozWe8VS;v7GI-; zK#Nhkit6^gQTGF0RS0NRhJ%R!vwd2MHX}0krsHg%iM=Tlb$M{HM)=;5l;i;P75*$g z(gJ_6-5j2_^|-lsum1IHl2VP3Z}p-K{eI=8w98k^tUReflI;Wp%g=qPnzT4qChi8~FvdBmbD#x^R_1-VK1H z+DQ#%VNJ}{SboUD{nI-&8rrGc2i!?bih(IBqk~3m?76dFcCJQpvjUA{{0^U3Bkg91 z8(8U(#1u7@UD-^dQ-xt<57AZg$rNFT;?_0>ImpdtHKy(=LQk z#-bWsO|`HV!P%cUk7{MPdB2UxrUhGVAiJkMG+qUaGsG^KAt@_LE)V+oR z_Lj4Y^u&>AlNL9Jyb&(>z~Y*;M2KZsHDI(K-wlM-I|%UdNnYF-a#MJi_fC07S#X-h zIlU_K=C>crIWLWzSD$=lx}XS6ljmRto9O76N}E9{;@*&16Ff$oI97zTLM(Z5GawU! zgB$QRse{5T45eSW`)zMH%GZUA;$-~pU>-%g)r8GgP}=LWP;O-(N;C=zFwUot}snexS z_YpJa&;Kc-C)pF~5uEC|7oUGUsQ9S+8!xu7 z84aFcp(9=0SF8>I45)i8`i20w@v7|P+E+XL zU!7E7zR)mUw7o4bL&^SX6OIEctEsBV2=7~FRg%CYrpJ1#%SX%NKt!R>haS$v^eiNE z^?e%c%sezL7#jp=6B+_V0kb&36J9^f<)(VSh@Y0g(&Q{oQZ7IES_B-EkyT~jo^d0!>qPcQI{Xj%y)#j4hiizQ8 z)LATEn1v*9BNEilfOjod)e8^DWZ|wTnUs@48J?tEt8Tna`uA3ulCf7xpQ8ZIb;v!ZhbW}SAB zeq?*VL0t2*E7`r;#8X+}$ArcXti0L4!;1 z;O_3hrEv?;K;!P*-gEDnJu_#{x$ODZPj}b%Rn^j3>s_mwLdT_kddZL1I@fVj=Aifg zxClKSh!IL}o3lB^MC~zU!zextNEj)k$J;vM`MOMz_E9yzGXCpu;66x9^cshUmW z0gE%h6HG{$pIcLJCD*f~JCl)owr44wkmloHm%eVwyf}Zg5kZ+Q(!;^i^O!Qt^MWuc zVb3aP+0-&}hP&M1D5szyiQgk+JRNNeK|*RMicqSKyRzvVPhg&YRdA=X6R{wJgIwNx zh%e0LVoy8Xm#Ab|@oEJZbc-F3`~AB|kygg}cQb*7UYBK)l5isU z65sID)KExmGUZupqfQe))?_QWg*kHRNs@}-MKCz z;1x5pu(@n~K(7yc_S@3p(-dp8*=`08qeRUI|wjblv zclE!g1hn$NY4B8Kg$Q=tg+Yz}#1b5Mg~ZleFYg|^AV7V@aL?j!#OsUcC&W*;A#BYJjFNiU^4its7}nRqRg`ZrCv;a1eobmMm3NSLPmcqfEO3-DzzC zBTgQGj3{VcC2DoZvz9XDM!Jy~59{TfeF4f#%z4IXEUqO_30EED9RGZNTwnr*oH%N1*^Y6C_|! zh5E+C8fZT?M|)>yCpTTc!)rO=tQ!))B3Z|BYRfLUGf9kXH!)5$Q2iKqiD?+$TCX}# zo|=VJ-fx>1kYDCJZC&=ec80r%-9Gh_qe#u1NtUc4(pd9|A|ktPo$LwwZ4b=XZyL;4 z-YOvop?j<;PN70Pv-zI%6rJyl(I2arQ4W;e>vf$tw(1*D*X{oly_cS+jUVAY9x708 zmng`Tpo%en_+>VZiI;L$`7ol*s1Epbukc%E`ex&cIvD$$wy3z1~ z=-oC&Z%SyU{z<`9?6+|(dKE3rz#6C3wcHC{s;{BBN=cEk$&_+}KR<9wcm!sk0iDEm z_CQ)5jskf|_=b`*MIMg8?Jx<<8D!L^*-_OR#Lq{ui|{hlSO?RYOcOjTLMsvqD?}1m zGDS7p{IY|v9)@Q>w2&6gD=qFo8yl}bxEuBgr=>YDY|QL_oT#V#snZ~1@|lK$u|@62 zSWgLrOsm-GF>?|v=JS^3&}7>?r{W@LV%HrZKagtauxR|H_vs?(SBuGt6X1GsQXx(R z4i#~m-IK{_M~3NM1k&$P==;QEUPbNHD(qK1RkBnVcWeKuDV zMPIZ*jU@bt$rs4UvQgQz4#riJmIsmgVGu;??c4OnvJ$z8 z7i{U5QAtms7ZzJE*R%F;tgnZ>l+1@0nl2Kkl3T6Q9*&42x2|{I;MB+@k)iZFehFLl zku4nhH@F!JkWi|41+%5nAvVvdD7?r z$M&eI;qnrOI^jk@U}_ik%PA6X!@{3I;YsqImz4I-N!cH1NWUy86*zPf;!_I zM?X%ykJBGQ2$f`l+&Qx=O(+z6Ec6Gaw3?-u#6EAYmAuSlW$)Z8-^e#7y>_ISEggtM zbpADiMUG>tYw6=sna+gpA#puWR$qVa;!@<5I~H=l`+N;+4>V;d8dXm}K?788UIuY# zlsN%+gE%4$<|lP#PCzM_jsa_a!CYE6^I_)Xt4#gH4;jVb8zO8O6nEr`nKFIxm2WmQ zOS?mt`%l&#Zo#bf5~%NLjLVvqChE`-1=XNOdtelvv8iqxuQ4c-ZU)0kTsA}=X-9MD zRLECcb)+dJ(#6x)2rH`dz@_I+Q#sP!Fv~`BdDI3O5*VH?*S4SAX|7zLUA{iD^djZB ztne)S6Pn6Vb6cL?osZe~mif+;Q(uIBuh*yat`i(OyAD?1pUM zGVUI+>+H%h)PgqNP`TM1Ij&M=zTMiQQY!ws+UzYEl+Tixu^EP_rr^GL5#XT;#F?jh zNxkR}eX3QUxm$ZTwRt)H1osY#wK(W?+mm%MJA(ihGct7xk){GMkjnePtYBx@91=3KdH>{~-JD(3tQ{XH6I8ATU+# z0|Ebf^b89(F&)tH-XDcV;8!<_8hQ^!Z>EkJ->tHRaIKhe)0?I)tmqBulc`(Oo~mCr zA@xHWW?VpcukE&AG6%yL?hd%#S9t@HT;u(TUH3SG<8A&?HPuB~$`J?D9&U!8{4JD_ z4)rQu-l>Jyi}2&%W!20FhLPTc=x!54Xma2LSeAtx?qmX`Nx$?QfH36gyd+cd`Uwaz4c{a4_}zEs zU#VhZZfyJ%R@$h};+^J(0N_Tz%3nb8F0q$#?2_pWL2js4p=tSGWre~>YT3@K`BdjOQ(I<#*v`B+((;gw4=JQ4y1IOTgi0k>hu5RU z*yl-KI}?7ns3`V(aMBl^v$W$*5t|=5<$2#`zZUC!Z^{i?*gxFtx9xH290-u*3~PQ~ zpjL@GNVc@(4m6pr+hA9z>MC7O?w;CHTj!ZNQad{n<$e%n&Az?~;?16H_#|qf6E}Gp z|DAzM1~m86t!6cO!qAa-M25NSyhl#36(jcmG0swa)Q_HU*wHarFwxVC-O$p%Xgk+% z`(1s6q1oepl*=k#|AvcH?fPjo>XwSBtP9+I)wd94GHF^@#T3MNAZIX|8+_c8Tf=-<`tF zjRTxHQu-PK4CT6hy|K0|YN9RO@hdq{mK1QQ5Y_R!ph*{8lXEpC-LI0J?dNUgH^TF- zcmJ^*Vj{`5JRz93l6<2SL!86tbe!-3eyl*I`*oy6DFd0#-q&RJbM^fDPpk2)@B>`U zb$PMV?|MyQm=sp%RNgaO8@OC=fk3-Z;M{Okd#h@8AOHGZdcnf-m_82FyNTf z5s&!b?B)EuadJ}D7D^hcokU18+$sn5*m}KEI=SDZ%P1}X)8Xgz7#lsdhiZbD5=gxK z$v!3^)qgvpW9Sz`a!OY0F~JD^oYO}WUJJFwW$!eGE6=+L?VRV;Ih4oWF@-sOfpI!V@@(&U z+fw6G2Sbr6TjP1&`D0ZY`{D5wxi*o-Dh))WsmO;(#uP1d#NHJ{or@?WjU7JPrjbN0 zbUZ%OJ#M*x*`cOn{v@oAx3$O;fJ0Hxc1V_FsvnJ3TNwNGGQRtP%J%6lyytL+a+&2 zR@283SG#WY2uXYG;0rtC6X0I8((wikn<8kltyt!gBIS+G!nM(kP-JY|KAJ+g-G#17 zlHt8lt_1kiyRtEZzVfj$HnpZ%l*7K>aVC%pQo98*6!nE?;eINu!mZO4)7O3doYvbvG`B(=y?sWbTQx?Pj^s`AlIRi>|#E z6vf2oJSNFu^K^ZCd&~C=agR$lP-IAoCA;b;W+H!VvH+-mNA}$X4K+;U)MJdd}8UF z_t%_guWJ+5wuxN{Ateu6|82zTSpxxSCGqguw`taeL)JG}EIpggNM*@z!9B&x0*@=m z1mL$4JT!w7J0$?ugc+RDa2k-8KUlA@MyveiajV07U7c4p^QsvRn5+8(!iR%W4g0@g zSSDVSBMXYsh#PJuo3edN>Xc-nUSOFH123<`H4ZGez%&I46G>kOW_!tG@7`qO)>{X)WRLn!Kw zYR*X)qW&I%_gx31*~pte<#|3Pt?dD_qb+ia1(H7?`vpRiMph**%^#Jg#wfDV^$Q~a zgA2S0!nF%4n=X#$A&7+hC<JDlepE~6-`TvbLED8BB==p-K5`3cOK0eP5W(ldGu-1T7CVvLQ z;*H}p1jgy5{8$W*#EP~4jfkp!1KZM8e(%sj@VsCVlmN26*{x;OUu5vIe5E^-{f_KE zlK|K}vHR!ac5NS2UPISP&n;p3%<@k8EXW^g<+YdZNy55Q6c%Tnr4VW6_H^R-K`dA!sC%2}Ws#x^$)#qS%FVNVocn39n)DkLT^0 zgdF5P%Tz(1SK8sU8mBHf z3+;pHI>uM9&RVKL&ycZcwE=qy!dc*>C>MC`E#ZJpGW&bU=znP zaOb3?q#Rn`l0P#z_z$~*cMXGfe>3JIuCAV3_6OG|9g_cNGtq))q&jHK`#|vVKUs&e zZ#IAqgX`~%_55cnF?5V)cmOnr|FW>j3(-S|{%7<%#Md_+Yt3Mc{L8#c`x-h_PO=;0 zAF1%nei~P(tTkX=xkT~jZ_j@*Pz-b^P2M`gJkfv9D=!d!z-DYS_4aCPi#@Xv@SjH0>dU|8_l)lZK-IWO zav0xpjpTprG?bQ5cV}F;|K9b(f3WTfUt@ReZlL3O)JW}2gSNw~kz*mOv3DQ1+&jA4 z30odonkw*rhW&N6TO;>lV69OLt|HszZudqqub(lyvcG=;zZw^|lNz^nRM3hBa?bGG zJtj|E)QW;ERK>08A;Q`;zsg=GPgkRpYO6yamWpY$hm&SSvhNkc7r^YMc*Q{3oc@42 zmUa$DyXr!mBuki+onI>ueNFMkZXW)8e}7LyV7wHZN%NOu_aC9g7axCtolL5~ z@)d;$^lp%(c@F|(4^S>T1m`#y3A>#&wll>!T~@DnxfvmEOu2B+tnE=8X8}^G*^h1_=6XjVGP1GkvpH>72S7sqiyU@A${gt_uz?b<#?>P+9b?eOQ$Urc&;jQv+ zgu_V0*M>4@89hwwo?>L*?CkX@*=`wTo?co4`xb}o4?ttoI+L^83Q;8Mm-;lHZk@HL z=kG^HDeSTUXS&&uc$%)t+ue8;*o=0311@x5LUV;w2jV=R#ju4|;%1wrz_|Lzj*!&u z51jV+2(-3hXs+%ZeB_3roU3(T!X&Dwy!hOHG`oB1S(R@zJ7V7EiNktPPUm*c9}RJo zA)6-WYK^utM1t9#KeMdgu*;+i48o%ay9=$!TyUX24ftL^x8>1^IPny!ZNz5USNbQA z_TxNMuq^xY&RnvSoI?|9M=p|1yoPD8w7~}xgPpIXYB32?#+6z6h+jGc>e5h#0h(@p zahvXQv^F-jVPZI0y>!1k7nN%t#f;|Ja2%`6*1nR7l-XD?PY;>xUL02FO<7!4i7K`| zQjb`A{AR9F;A8qzD0128^9w}g^sv_=30&FPm$)g5LD&21i)et}vp^OG7%YtF6=JFb zr=9yVe)Uj?*yjqpUx`o_6R~PIdo;6%#Og^%qX%VI&{rpkn2{P6zTyut?n?lq!S+nl zW;6W7ON@pa<_OnDQXs`}9S<+s7ML(gO)xU+lNWUur+4TX1$oC-t05lC@GzmS3zpcx z5(!Q^Zi$YG5jCp7N~HVQ>@DIsDLcx8IO0Z5a>~`d)}lgdWtXC1^a&icqZ~crFA4wX zfaCOC^>?FfS1j=lDu4z3=o6eBuVHS{Wlk93_3ep%MwJ>y<%2oDZ)v>Rpv2Z&a<7S} zHk9KcrjGAi0>FvlT&5`xcW6J!K4$Wtyw{VJRJ1GJ*v5;|C4h~uz|+_8kEV>2isD^@ z^4*Fm-%2P_8Bau^g~dTQQm&dk+eaxb$%dO^_nlWf52!a~_bX;c9oSN5K`(>sw$ ziy%cb{U&q&;~=4id30iFDd^K2nGz15t^Hex<8?7l127F#E`O16;6~=_+h~O23&<#Mqd44pOTzW5x zQJ<#a!zKFu`jz^)37bc5czBCo%Jepy-de9OQwDZEH9CtMH^05QBSVO{Bdf?3ZM`47 zr|YyACDA;4d1Qp!@A4@d^cEd*>)^qHD^Zhj)r=tD(+JdLK9~nWS>N{AhaY%(EqlZA zRTr-%+j&Fw;KWR=n<)}xB=sKNMT_n*86C8-Yeq`>4)a|0Ek;oa0&sOPHZ1$!y>`{N zVe859J1bWY@RNR2BhKYVM<+XAOiZ;7j9!|RUGkp;k_%E^?O z%fIBlq`nt(Y5}odz0P@b*eo^2g3Kq82YQ=5`6YJ}2T}g8hhD2fdtJS9Qc8U*P>q~R zKvNr>q^Syeb9ZS>yKJEYd4VpQ8g)MnQelSL1ddTg9~1;1C+%{@D%E9tn@61f#Pr%S zqUQVPh-(mBGpw3JJNltls=Vt!{+=pFt*GeuYPJw9BRxNJ5{b(T@Wz}MOK#U%9`5miVnp)tYQ8D>NUQYH9l(L) zXHu853Ak}0NVpUXIavz!qN9K;$!&bM(?{v*I>;@&=yzTlN2eN-ndrTKLv+4vI9#n% zGQx??MB%}IfhO(xx$qkGz3t-&!gSMb{ujm&)r}tZ*)d_Zp^f}X+T=4f=2Xa}>y_uV zbA(2sVQ7;1ewWI#t;78bgsyJjtHWAjQf{Zuxv=={?nshp+5IhSAx&4}fi(SHi&gdO zr(1lp;M^|e3YtAF_v+sr8H&XwjN<9)p?!5lbXp9!O5YK0i}zZt%em|}{m^H4Z-T42 z0qfM~+~*d72wDeQbVK+Qr6N8_nQlmKs?DY>s%xHmtJt>N2Jq^Oq57Z+E$?Xh5`~hQ zXd4}^t6%Pom7py`!M;vOIcasC?A9^sCQkXpP;{KR z>b5UVTD4qo{oWhFkowg!Oq`|?(&eFEaXX*66(i)T@8UB$SJ{2FIrA+%Ph^2@->I#L*ggDK6xXxj;*s#&_@eK=hcl+ARS@(SJa;V^>Zj~G^Oa=3v-hv6pjEOVlSSYj+K}3_DS7><9l-_o3ny27A z`CCg)ThrY%7X=j?{eIUQff19*Uc6yh_)A7Hvfc^s9IUl}PsYRdGglv4U_7~KtzFTCH6x+tmU>FOFnVIV2{_Q! zAJaCiMLyh0w|Va~?V6N^76}+BiCCi8_KIvZ$r`uEpa*LA-8?;X%w-q1R@r$2wWPAOdOU3QuK6IRci-=sWM9+7%qH{n``8>;3O0_CYxr}fok3T0| z*Z{j(Pv5_0VXtkheAwiVu?Ag+ZJDvoM+JT{OH$0oeNn}-NP61Xa28{Hbomuo)T)Va zg35nfn@(K)9t7rY{lz(;8swgbCacYKu+?0*+3bke)i1Vwdh$T)+E)jV{65cJeL*U| z7-9HDM+*yQ<1D3$4FIxu<1ov+?{HGN3zBeL|!sqPcnaHU5nKbTCseSsTZB> zr_0pOkv^ZxHd9?i2Wwz6BBuV6Yv}2b4pp~?Vz9*Vw?dB&i}<1gs8%5}7|l6XU0h#( zp*w_l$=!z|VQXW$TvvRaQ>tEzptyZdx`^vA!+jb|VQH4HqOUx0+bwmO8rMb#qmd!j$je{Uj?I6^ad4hrfQmP%oduc5n{m z0>lUJ%p6QStuP1ARZZs^b>!=AK!_x!=vL}cj(o25*&E?&7y%Q~o~P;Xdc{_3eio<|Ft~fd|aus}u`UFB@9(NWQCxc9hWFmwtP>*Rs+w z*4~J<`}l&0A}jZ7Ei#eE=r~yX0p+6rtl3E9k2?@S;m?YKIF645G}cc`kB>k=N6sd_sHOFNZJ?zR6R{&nIxwYbF@yBtu%gA~lsW!FhjZi@9A0kbzo5CC`7 zHn!F>g6xQ|m+|rJx^nW8*6LWh3Hf*8JB<`}+j*ZH;8$7*z{&Q|-kq740L%LOWA%O< z%M1*`*3L;$!{hJ9L^BByn>I49sqPaiXjcg@Ir5IL8*V#<%yHHwT;A}8y4X|bG=bey zN)e}9*?~9Ax%SNuZWznAM2MJ5b>V6aER=_LulDK${XV9$CUw7gr6pzC-QKp7bg38e zYH2n<{;jC&N?63J%n~M1nOYyd2t=HP>P({g+E?2h8LRAQY{zvt$5`jqbh!n%HFN?Y zu#x)Tj?SX>e|ZHkJs_vJGxU*#bv{m%qX}P>td(o*)GIcMGlzFft6uaa%8A72XD};! z9#)?x?U0=U)6~B>kWZdz4V5`f)lO(@m{GhBqwp1gTZF45@2vcEeQ+?>Dnq7iDIgDP ztj)dU*<6xF5GH%yzk4VDOAfLsS6LV!$0wnM);-QAfZobz@bevBI|ENWut&LKTf1D3 z5wTo5N`TF$r^v)|^vEMKcWsp>M^8}gQ0rA13uU`2Y9*BxA9l^DVDA~|ktd&&j6TCn zWC3=-%lk=>(&KJ*V?j+Y_osy!Z<2*esnI=pgl>vqBgh?-Bpl~S$sbUp$(i%vXr&aS zULNpbM>fEjKGbDc=Jup0_MvEGK6rkI zm$%D_5if8a?XOB2^QG}L3fS#6Eg~<(N^=)D(gX}B_(SD7g_+ zqpsZtUoG&%-xXmBAE)geYhO0;D!ZJL?gh3smqn6iWRzy}L5k%L#b!GguLbzYv!}$f zY(Ap9Ur2QZrrBNV_ME3ch_(3LuP>_tiBptR0|JdRjp2nfjiHN`2N_p(qp3Jh>Bf`+|55)KOZY3fFC=2Iwd~6Mw5{C zeF1}RVUWX!s-6Y2WN2lG5dLw}Uz6Z~0=Ct^lbGZD8;Rw-HNLcpDM}52_9q*J+73ME z9fuqfxPDpkAws@F?_*-nPklyts?#*sU*gWL-e7Mt9yzy}Y^VPkWIuvTvglUO%Hc6T z+kcjSZ3_b&&k(Q~<_sa@Rerd6tf#T{?8o$^@pChI+$B%m98e9N8f$$cm^%IdA?;~| zXW*ISa2{bo9WfDRLMQLji!-y4%HS_X6PO6Q_I}dEg3(5jr;6oTUl}zT9|<{k{8e38 zTZ*`}##u#ZOL?c_$tINgWyV|4z=`{}LDMk1v5K#>-kuf8-Gf{UV&&J%!|&(34UG4;1X=?vQ&Qap77`5q`S&Gc}9L31gliTT? zh#m?)*voSS9~);fE;(`8Uu`26F<&+Z*57}e*9GhIc^1f!cplQq9*rlJ{T#+itzQU& zOp;kl5yB0hQp?NMFJb$WpmT7~D34)$2Z7E8LO2TD1_J|zq7}WOO>4*(;3ix!KHg>P zESua~7TY@(ENqrUx|1P}hn<1-(HfmLztKhC_40CFSOm#_8nC9B3gJTn&)x%5IaM>2 z+Ycd5C30Yk<(paqnR$U<2aV&rMb)I|_+1lNpAmQ)vPnWlLOwfsaEIsNefO@|3sK*F zz*WMzR^RputoylKd}Up5fmeOGznPE+-|9C zl25-3Om^8^e)v=nbyjzG^pR*Z0jQ%^-Yq&Uf9dw*SIo`}MrMwEK-43%lwz98n30e~48w8hop?LBJ5q^=rT!i&d2kjos@G7ITwrC<>r9yQsHj2GI) zkgSnNPlb(N&#K`2zg`cy9N{XcOih-yNNP2Wdn$G4dPkeF;Mf=eyJjjJknZznQ z`Fd?Yj~eTZ4$_(kQFW}WQ77eAzT4MLz>uTY!g+R+hpKS77bEEU5Ac@)r$OLzz`~oo zHeJ)@0(L{|(B6rD!mKWb>B8J_w)_y3!wH=&$;_h z`DRQ;B(uNCo&IogpItOdh0=C`&b64#z3raJ@PVy4SvBJtF+uji{A=&Sg@hwTN)|&S zCXMG5WRmk@k1dUTctF2&8E$j$>L_TKU5ws_f8lE8kZOI#>yhFWN@c26N1=V903_+t z$E%ZHm|3^Yygkl#-%+oYe2y<{9;2FA7aw^&DHccoV07yRa8S*zhnb6QzxB>(;3+%# z1&wT$`Kou?y)B_E;~Zp*qm>Z9*#g_RX7p)6d5bng?e~Tqd=J$^?ZKDkpxc)vokcRE zomqPdauOHwWmy+W6z<8mSC4q-BXYnW%qbRx9VG4I z^}-oI0H4o}16~1MIHCgtxJ|9(+(7G@URBZvr_e5Rb`Y@N6{EF;f9o>Ba;`~&n*}6SoGxG4ZQ$(-e+G}e!hZO@9unz< zWo}UjxY!C_4yYj~4}V+W3!vLtGDy2M5y|5Koe+E8ZP1t|@y+$NVg~DZuC`7J>)@23 zQQFI`dC9px>DrGQzwN7E?O!hlqN>a-DX+ivP!C96pp-PA-Sx0G#ke#T=(5@K$niY|W$0!$m_{QnAxt-OGO)9#b+p}!bDY}ms8@DT-q`;>W zOw3U;ZuPef&C*w4F53C4%M7_`tc#QTmRSr&+D6JwKe1 z&YUn8if=9`PGqk1axHv-F*!E@Torc{SWB5|^_Ynr3=mq=LtyI>+Z-&elLUS749l^55^$y2**GD@ub6~AMmuE_Taqem25iPhoOQMu>sVGr zj@PMlu<7F}IAV`|56pz|2nt4ev%XG| zV`WbRujRn?INd$Y&yqUf zzWt)+W1jTOJ-5Bpo&H-SBIOM|hFtHou-VCccG*FTW1R&XyYnc45OU(+rZJbSTdS)< z{RxqM4?b9vv@lcsT(OE^&nHhLJx12oyM7thkK^9)ES_*PeEftzq-))a64@Od1JfNI zgJPhSo^E%({oo*6q4;AZu);p9d8YK(l12uFUK78dFWj~B4k@X70~TeD-(FJN>HCe( z6prNhHs$ZVf9cF6T{U?v*4ay!K~DIZsB(f86Tx(5FTGdvFeI&t!Q#VplE(#SpFCXe z$oAbm2o00&ktcMz4CtOP6G-aTYf7^(`<+c4;s|PIpwtU*lT_7sEH3rVIUba^6o8{tK=T9B>3sN%zRoAAmjt=aZ&feEs`;*DCqd5>=MgX-NSzwCEu z?(;MU>|77be)9UF+^c<|lN~D4Wg1cgm%Qvn`F_YiEVQroZ4>4pcnt7&x%;OG&cFFq z8|tykeLy>4JGqt&Hu z94NFX?iPFzNi2Ww@$J2yha3qz=K8aBpT;;+_SSmS7*JILB08@KXEWVsaV;K|q z69R6thED2EBaXKVaSrvsuLn0Dc$)mysFzVwwptb^*R(yF1*F!+_Dl4Q3(lWj*p9Vh z_Le~{|M;n-GRraUZt)NDc>jzmYS#gmyhGH^$gA_%SSFo-Q_L*SbKH2Y@%<*Ow$7XW zrul+%)o;zRoTdAB-u*BfLJjZO5RTbRu|P9U^6?G-YP9}GHECOl%Kb(B!ZxVXWF_=4 ze^viDpV850E%6=JT(!#lq$>@g)zjCAFZgq~7kkogpPXW%rB8MqEpf6#O8H?2AtD@S+PBFu{z)3*x9@i@hWJp z(Hd0qm`KR$_~Dij@>^S7L3p0Q_HNGsQ`GuEq=(zlP+&Vr#3Ias)P z1Lh=MDL}(W6q>G!c4d)%Lfnj&Qqn~_Dzz8fkXu}Gu)}($&4Kd%F{6I;_X^T!+?s}V zEZj$447eeJjqUGOlx12!nM3*=-pEv51x3}bZHqKE>|Q+(m&xVF9-KZ_?>M;71orYG zRU!1(I9kg>%63{QDFAvhX0Jo&&@0PjyVnGl!foqK3rLoL;pPH8JVc)A~B#%H7KN!~LAjC1k5>YX)cdjd}9SXYZ z=0@|oYNRM^L2%n_NMFyZ6w|ll-)OA=L9{w#Uhnub`XYFMo-n|CZ{IYP2Cvc7TaL7J zD@IQB8W%2UMcwYX!+)g}e%Fb-J58=2xD!!W*S39nF;fRU`jd&O?E2W2$xhBNz)&Fu zIvoYgZOS5Fl9sQRbiCtnew=1YyFGnf98E{1tYO=U4qqE7a?S+U(oS7>SND?hVR%^> zW)(iW93}0rsK~HXC!Fgtx#*@(g3&2`1Ho z-Z~07Qn)?3BfDBuSnoQHeIMIs5hJND`?`(@{}uR2Bjeq;RmBI6&I^08`zd|;XS(k% z5SR;u7V8GR)od^p;G)QxF8N-*R9xFY@I*S$@w{R5doD`n|D6{)@^FyfnvOxE8hK{K z7SHxa|FnzR?Xc(z&EDmJv9!PVYUolbUj$q-vvHUFWo)M(M9o*&19Vi|zHj2)wor=# zK5m6DTmOY%$KUq5AqXBFG~RfuTt@GQE&5=IZH_qIiA_w&R34C52k6q&RiWwZ`wMy8 zzdaQB(;uQF8I=nRhZ`E`dA2(R(tx<1~3S#B*ey)Ng&nFA1@a~xsl0J)#tRyqDpFY0*H z5qn8Wp#7H_8V2=4$qV3ug{m&Wf9qZg3hNk1i-pM?%3qR!0UJO;)N7}x=Xbm8FkEZZ z_1lPP!LR)>))xc@%b{Iv&bH<%l*Pq76qM zuQDbrh8ufo3fr81rR$w7J7gDzY%>?_DBjddbXcZRT&?gn(Q;_iua-6?F6{gXE`pkd zwfdS$nw7KOJ7L0lT0bEKx$h1zP{>>0RF^l7u|16Adqxh!1@?nY&@1 z2HW<+8qh6!j0wjd798gnyAb~eyY(j%{Ou1N_;2-jbTD5^zJ7Ak2$NTj2`)+wWa`)L zVYS%I*gChSnAHcF+InRHq`!JxKaBQ zOXVWzBOpeVoe81L%P;6oI$e0*ZhGKLhp@+XfO_k(S;QD!Ae#?KSb0+x;8LZ@1V*jg z2+IFD{e+nX|BZeFTQFNW&0F5y zGOLAdA6$Qh-~T>+ONEnT157Z=y~M5LGv9<39r(WB%Xkr2($TDY_qTsslx52TI#e8B zU@?_j%ts?I@_e753i@m{jP6-QJ=*=SbaSq*fS~>twTZ;aWDmp^nW=%hUxym3hrR`W zwV>$jVv|7T5{QcH@9W6f=j?f{>E@{MD*?YS79KGG#=rmv8RO#j7^wD(vhwhpau92e z7!vP=1rt3(@eJpyw?68k-1kM3dGmf&nm|I4?b6Xe ze-koY+l7-#7}fvijccY-%xky-C>pM1SIg1#3x>h5oA8c4HEz~5z?nd{apqn|u}^?i zEA`DCo7PQ;dRaLDS;VIIiZM!Bwdi~kMJ_~G07+Y&RU2BVK;8x37Yv~drng7Y;L4wue(ov{z%6=f@l$ki>KPkfy^PMy> zB-yp6zn|*eJ8|f>+-#H%%yr~HYYA1gnurI5g^|By4i0nbcUQea{-%#eE$1l>?>`CF z_xNf-8v$sx&Qm-cjjif>8#AJ65(sY&|Hi8ib*`9S{R6O+L1Tg7gP{PuBehUn;*a6S z&CeVo_cB|uz-7)~#_9l<_x^8*-u5|X;23;LbXc-3kstZow>PNyU3iea!oULOOCcxQ zo}2h567bKJG6X?McZWe#HX>h2kFR1xxNi=IwBPjAy54<98eh*)*cms#du-Bb#Er_c zgqTZ{$1)0&YR;dWY>R%co24sCBz z%M|6_C=OL`SyT%>re0qP&r@$Rk8QQaD_$|YPw-}bwFjZ0eW^ulEu}+WPo=aAm$#~L z+qL=1r(4p^>Ybd^`HTErxM=}1jnIP&UB8^0$*Z|r$=x%`t{+%BRWNmiwOx5fgEbd` zxXRCir0^wP0q1C)gV)0zKKAGB40T3o6WYM zp0z3cb53*n6R997fcc>;oUztkpaup?%7)j^cF8d+tA z@gU_7Ou2r_MF}kja3qC4AJ}H5Zm*QPjGCet-v}-0h)ZKp-M`p9wQ2vGZ>)89e{RR7 zVuN-bDV-pr!*}LS3PTgMW}l#kg@X_rMGH@*g9LN`-|ttB7g5j zH#8)vpt*Oegjh{{O&rBmohY?{dvFo(5#7hU)KaU$eF1x35p$4@=w$H~mED(Uh=sbu zi|L~BEBjv*IML_xhhB1>=gK`lSSDIW{fg-=G^i1h1psmkpB|pREa55Tw&f>`_=k$H zz6JnYv^6*k_Krts>0LyRZWlpKc++e7<<6J;*kPxoXEU-U>^AUw2dIifS30cF&JTs9l(tSc|UlDA!Hy8}}f z)m*DfD&ht|#U|1oiH_GSFCM0KeXA4|LqfZo4WaK53Xqu$UMhj%{qlzIH5md?2$sgdk^-W?g`}uv5~BmdzOV~2 zInhYnCB;o4D=?mN=4~ORq)WO2k0+vXx}!K zcxI0g<`&70Il^grfsrM0QUJ`DHj!&A%|>_m7sSptb3i$_S7$D;((>vn+K^S#TvUUoI9Sh{Iwk%~o*_ z*2(jcbzpW&&861;Km_7bWs3@7tGybPF43niWCFG?UO7r@$hrFn4GhjKf*R9n_nP-^ zi*;BV!rBOxJDM+E2n}jyT7FzkRvqQNvna2l3TP zs*hM&@KTJS8~dz?d>1drAc|gxY*f7l55E6K0WGdP-DmBs28F7s>MV(vWO6&-r{UTM z+V-tddt(&QnfRs#qluq5U9}13lqbH*UXiSvg5AtLk>yN%UH9b=l*F0oAFWuw{6NxL`i@yy8f$PI=$#pElcSyovVSHICWN2)hbVdrxGG|3lxVj5l`hCvDFBP1=lO1 z8$i5PO}<`Fs?wvhl>z`uCKaHDT;@M{n<*KN!I$+;7YK%s1Z17-ptp$IqB z8EMw{RvjqhuTK2-9w5Kq4!|;>WYxTTE3ovNB!=bnry$M0s|U0&NgVmq8Y^duQDoi= znxrzw#z#8e)CnYt?FDnrk6j}(P*o!iHbc;(VVAH;KTG3#l!i!U{~b$1Y08oSG_qj zD>?T@+sbt-V!N!244v07(+DnR(C;faVG7lYaV%{+!-V_5n5(r+hgCGs6)c?|fWot$ zfjah>`UqnyJ$puJ=(#z+sHkdfUhMG7##eTKS^&@ysF#Fe8%X?qaG_QJ zoG~~{qsi+4b<&=?cgQkeMJQEc!42-Xe(^AzqGlu{h}yD4^1vt&FcrBE%j!I-WGYm# z1%&XVJ4%P1@V6(K+W$s<$iXgQkn0-|MHGs(0$4 zXzB7j(-ggHX0Cf^!@JSLKhEs}44;1cr{FK=`{GmbA4Ny|tW+Hy@SDJFb@9I7Xn;vF z&wBIHxxQ7!L=t~%Rl=clmP0TyDN$WQv1Y%tmREzs$|{ zL?tnItr9=ZP)UWzJUPWddCh}#$4fNSIDp!s%9W9KKt99&qFir+NveiSgd zM!Q=aw**}d2S=SRsB&ykeUqFzqfXSRwS(@;}0a(NsK8?s`=_O@k`={8Q zyc~fxPHxM}D_%u<^oA=49{zR)Ep7&9(AxOS9Jh;X*4U{LZ|S*EX)AcVwVKbzvN%S* zV+CiZJPkM|Z34ASh(iFzMs65t$66;;kJvx?nTas4ry}`0T%54}VSNGhF*a#+mL4YN zXu7ZOlDjeCSFF91swJdN3liZ>I2SY1c2CWw6ZkwK0xxo8*bDy@POwG zo6p&+FVb|XCgDUfy)13lq$zY|FHZBuD>jr#i&o+Jr43W)c~3iRs$U5{+EreGvGd^&XnRE1dtJ$|Xz=LLL0uq|gd=2^)@{*E8w3NOYs;TwpYLPPWU zyT5{U?Wsm8qH*&9x|TbVz?*jYwA4+1W$dd7#+*hU--orQ2Xok^?R#DZ+!RQb5&?0V z-90lpYU*N;8^NOXG1e*r%dwhi!16LKX1IwgNj8fVUK8=-+^dyW!vVhw_S5HzI|6!|_?jl=!G5**PxZgw@a+aSTX42pLY}y^ zT;mEMAafA-sejKj75%5bq|B z8>5teLlwED!f^-DVze=>+{5s7YB#b*ITRr-)A-iOREvNM@wnvdn19n~=FJ6a?NJP6 zZlHEQfSgWXc`dLjM3vJ3fpF5CEt?@h^Z*kciReYJ{#0+)ugVEieaSC~%*Ufv2;K>! zHQZp5J={k_iO{mq{)Ex+qfVXK8wB*hj_C8GMmx=+Bgt zI5d(Yg>+h9j=zbgx2P*5RS|y`UrkqZkW*=@Q?GIiaigX5z@aV^8#PK{B;kcYVY)QK zi*xgfp)e>@uh2mG-kvk|D4N>pI}ve^lx?YI-M)xy8vv$~-EqvYPyCg0x*XL0r zkN&&mkY{I}$?DI?d@}KP&u@o#QwgWhB$Dfqw9A94TE*>Tw0eO?CBWJw8q&DBg{QcY zN193tnWk2q-ytx4!GltA3WrlZheuXZKNw~m4i~4j!-G_RJ~Ng7h23P4>9}MX^;Nfr zWTSHjL&oG7O1>Fx#-DEHaxymuLRaa(;!BU=hi33r;S@2H3SS1m7xKAcA7jbdNuJ)B zRvNO!TT<5-P|>NMu>|DSuY-`hir`}z>HD7It{KnLJH>wkv3BcF*WR=Zm+#^SW~%NI zBWPb-GS910yO(dhK_$k_t!MBh1nu`4NLB(e5rfLi#3ZX?bwf$b_;)p>be+S~!0{(D zas~>VY@&!k?IueyFy6og4gXYmhRmKizokm_jE*!Uq? zwSeowV4t|28T5vH2z-y9;00tA_d3WEz5!Z_V}!5Rb<0+BjfAz~kyvZ6{L;4gBZL_~ zuxR})_%81Hk*&y-kQUmp4Zk4S3i>hc^**qC5DGlq0QK* z69YDlGB#qWg-{Ez{TdSG@K>5}?SD%59b|wI-Xe>}RIKRx0+P+}>9qk85ekN7gyBcS zx7M2LN*rCUw9gk->7ekQBJy**A(py`Hg1OYiOV&)DoC^`BH^Qpj|X#K2qB*hUfS=% zU5MBc60&k@(aFg23K`uSakfFiE4yl6> zk8+cxw4|l#@KdmU?dwitW$yHRnJQPx+VWS_Q=S6Lr5~?`Vqc@h!G4w0^6fk<;x0P9^vdFp z3G><&&hga+mg!KX@}ew#_&}8yixL8Ms38qG9i@4JcsI)+JZ02$?Q&I#I#Np4CWc>? zw?*=2O?juP--$xB9g;`Vkq0#fqJCL}?Gv8k^C*F<1IALXJ3WVQkiYpCw+H68MIEU{ z=O?^VcfOrE zYq6FggzVfrn-ddiC;RX`?>m)s#J*G^zNM!yqJQ0;e+{{+(o(|=KMx1J4fS-V1eFavb^fWhJ$&=mtAQM zpQAX`;QZ`EH}3>bzd<6=>~%EwNG>+Sn%#82>YY3%fzc^Pm&{>_7kbBRk^xQwM6*CL zPG?uMh|uQ>TaVM;*`l3@W9K0M^}*P#AyJ9+n+#Ufgy#=36;)x=;2}R*AFRu*{zzi=xp|VKxejhL-dy(yE=&uvetTh$8jf__M(cSdXtTknp zjj-=ZLcO-H(3tc=PSiTGk;j4`1KL?{aK}B;pjC|}SFD-5G)hE1 zvk@Xpwds`cH#<#*WB;+qq~A4Ao<6xn=b>OaM0n_?Wr&5FAW*~lPDVW-hK6E6Ou*Lv zkkiP1$%Y7A7)e$=!T^hHdpGWQ2VhrHO7fewYedC~bJsE={447e_rVjCQEY;ih?I@2 zS%h+$?PpYisKL=mi1@#la&3@C`Z_jy0kma0tz}?Ivpa(eR1W*TQV$6oc^lpkGQ!os zf4}RI@^S>6m5|=89!Y*1vv?O)=jv{!a<@Qbk|N=pVL!^W@qA@<{NN_lkt6@s;VIC+8F9exJ(i3bXTs|I#;xjyi z?LHohWP!Lp?eVr>Otu2qI<_p?DK*YiRl>O%|6VRsNWPo3UW z9hu>;y`hv2P8mhs2d1X-B)RrhF_^ESQ?4)GJTI~Dc{G&dw8?3yoG%Om``{Q&Dyr;8 z{?i`Pq}2NhTuyb3p+l<$59opMIh+|go9U{*NWCa(3Gr;Hdc1VKCpN{bB;AK;$<2>% zDVRb>lh!o6fwNQwr0r=Vo|`;B?@$%kqoBN{90wMd3#y3sKkDcxB2j~Yly z21^?p^#(%`5}^siG`){E`-*z#aE5y6To5n?fv1Y9L+{;s+~=7*r{xIVIl6+O@ihqy9JjM z#?dKRz3UPr*<6<1h*joQFnJ!i%Z}J3)zc@>mNb&< zHMZ+w4gHueG|+BDu9 zdlcC)a}+JfLmu5)h~xBA3G4`2H2)&!TW2=TL28@$naJ*z&E;3r8HErs5`D&1QZIZ} zATk3$O0aAPUh!&Be`KJm5O4O!vDb5zNfcYF@L9Zp%g@Bux;A>PQ+mTd#K=kbsmdBR zf+u&27Myqg#u-z;o71|q&~lc9S@g#oIr^W{*!d}2^JK8>RG+-PPl0Ra1>+ErQc_Qg z8uqgo_w!AskCr`6;LVZoCLr_plF?*6f-N(c%n6+6r0sDhdnTO?htV6aTVx1+x`!X@+&^Jb9oM<- z+|}hESKZT;7ZEH|;WQ3-SV0Anxt=)S!#4iK_0MJz@{f*x@njz+98##TX6^O4O~g8E zUJ>(?{Ad#_VYa7-V?X98)YRm2yO7ogLYnGx{*EWdyVj>onazF);(~^H%I?8)k zl8ntzqV+{VBM1tV8C3q=&5Ip%X$xcR-4zc2pl5c(k;$pam*1gv>M>1uaXF&{r`ici z%6-21y7CfOw|#}JtgxI4q;!;8r*fv=DMcXxrObQE&$H8X4j|qIG~J{>+wG%OX{A(( zySAJDmQhw|zk0=DIQLp>Iuj^w-3AG`YB+ikIq}h0u>!s8%@<~FD|!COH~s0;<^X0S zVn0c@$`z1f)$TES_g!UqUe$8>4kBDf0zZlB3uCAubpU?iPY!&z8&^nu6ryc6ymG=HKh#U=;^K#iKBG;ZfP7iWIXsS-Q)A^>z#gyj z#oF&Kt=NNw44TDdi4{9^@u&@7lLNQ*ezOe)|3Wo9c_~o4UpiWG%F_k;urslM9TD;8 z*?Wt;Dw_s}StX}zTHV@0|G5tZ>|y++GSwQ|+Y^;%B|k&$+_##LKBP&kAIbHQad^0k zcullvE7T&S3?E+%N=-W|9^~t8rG>*RC`z*WOSK|``phc`9dD1+hW52#=j>1JS}>_G zVG;A~hVdesHLT9w;a@rWM;(xG;Y~$|k2CUj8?*O@`**phAmZ4jGLmsuj-y(djyBGT zwv~CHtyxez-p5+@Nb~V~a4!ad>sB za|wfau5h=YrJ+kZdG4gkmydkZ@o7Prw^3CGksC&}-0mNn;w}uON13xtb6i}k9>)*n>bijEzlXzzY5`y#KR&JV!A zE{(|xH8&B2hnbO(zzq1Mz!A8%v)Gd-ehp%XBR4Q=^!k zagu!Jt5#>suRO5+IhL(y3`SR8bG>``pUOn~W`W23=2|vvqo)C_V@C5KjxhW=B8jWf z=O=>#;CqKp-YkFq(l)?;dbAmCqS4vf;g}K4{nq(Gs6})@4W3c=fCNx-ETR~s+wmai z$m_L5sb!IBsaY~^OU>~yBqXGXDCF^RR7pDRiKi}Y2Kt(_9(gm3ky>Q=eoBXJkVEw> zv~dSU> zLdQ0W=xjtnYz5$*i_|6e_{)Dy*vO>g7}pF;NKZg{G-Ub-2$ILzz8^IvW2Vb1_>9fk z_>_^Ox!gt@MtFdoPtEg4Z&(zRyOuF3B9T|d^ObEh!MoL$7##y+K+e7Cx24D;(Ke%l zMRL=R=!A$#dzGBp7KNlu%=q`@YoQvLK{{Nha}*s=tAT&3;e6!-HXqiJeWR0EI8vGh z$ipAbm+{4{NeF0;cI@tDRcz1vUIn=lQv-%$Mk6gw@CHX+)>T(-^wb3`tZZAMk&5G^ z8t-RS_3z)q>~vEuTmXV?7nptPi1AF?;;GD(qMGzLk2GzH>-E1GL#uK#r=p(q9QGaZ zYp%4ol4YF4o#8l4aAxJ9OO$$C`HP2i0~>^(d$`a$K}Ja*pOMi2_(D))I=0|l8^Frp zdW9W9z#Vv$>{gn>4mHX#kj}!5zpeew3+X>L%7xjsI%FUx@8$jpHB>U+#SrBkpJRRsl0`NXCx)ZiP<0N+_^FAA>}XpPE$m<3V<`>`cA6f7ypaSW7EL_NlJ8Y zN19a3q1(b-U|125U)XPXX-@{ZjY5POh9fdYVmN3~*F=Vb%rGnH9Ggbi_~Nmxbv9p2xy!NHTR*0%ACi$nTk7S(>KCZa&TmJx zO@l>>r~Az=%lJ1}7;pR9EcbF7)M$|#wN@zN3G?T{8%T-js@Zk?&f75uWDGfbM_3k} zE>5Krud0w7)ZX61pW~az-6vW!vIp4m03qCA6jxk)|+2`v~7nb27ju!j; z)x%%}F*aLiE*Il&&{IE}g`QEZhOvpp;iC)i~_|!TD>LQb z!z2c~o>nQC=p{m(j&CQX2foWsnGEl*6DDAolWL0HoBD_4P^d5K_JjpNEmbcPp~bp| z&g(_1S3s;{V5<9_gbir7K@yENLnFREc18NT&OTRCJ4bWnMYY--|7H>zYka`YM|8re z?^rq+_%x5dlHc)zJaY^sB`|OJdUjqQs>SpB5mm0$yKFdQd85FVffN`PY*@MwA74dwY2DbOi|f%e9q5R2-RG# z!sypceeG%B9$ac^G$DC|_LJ~sNa)?xcGl#I$e(feCwik4@F|h@cJ8aFhVR?^_d3@B zB>nWe0m7>>tA#>E&$W|J!wISaiAsqcvi=(ut!1fA;kM zWZnL#Mled=)7wzgegB)vwIKqsD;&*cv&;Wj)%$-_Li>cG=}xTmJ~FZY-=4yz3JeH+ z!B3*ye^#grb1kQ=Du~{6e;(1?zy3S-0FNivX3g>J7@3jkmc&1tFrt?MK~!qLd%jZ{ zagP6yTH76bK=X}q%56#MSnjE_J{WHWyebU8*^pAIcN@zHm(@l!$BHHVgI zV6Po`SKQ5=L3O*}-LR91FuHnkZFl;JLeTUqtC^>eDol2l2jHc^T0;@iLe0$FCcg;@ z(=S!XKh;mM^GgFL;eOl6-OotGxkbBOr`|LFip4B0hJAojB399{nP{KfDQ%cXLlc?T7e3^D&{SwG)~U_)I8cMmBoFo%P8W z=6V{^ov`eYjYBIy>GhiPD~J*EC@TBDd6@A!6_Z@5#Q-r%`18L2D^62amb-Yv-&270 z_DW*tzen!Btk}O~us_13wqtKJ#de(cKC`@h%rkwt-nzg2^_Ib6p|Z2>T%jzLI$`s+ zH1vJ>+{j)}51ilWR_<&bsGJ}0biIonz&lf?<2YR&iRUGBqhu|$CCP(zc`_~DwUccOYM%V)$_UHK1Sge1dJ~O_j4EIjz7} zxk}O3 z#v%H-ad)>A+fdN{iawra6>VUAd2T&|TD!l*v0t~^aEdtMwM$zEx?Jb5Phgu$n_8$2 z&ya$xMqtzz-$EQpUbZ_sMt#?Kh0W9}iXMe3N2m5Dc)g;fd!2ENwOye|8sp9$2Hc|E z)tx<)lu}s%a0acVxN{hd_YH3puhV;^02<(TP%j_s;T0xn)Q{>?d+^k}x^&2bFIL6Y zdxO3B!$_?aA)A5G+_$BO1#6mnVZg(1iP!<9l;5TsjqHW_xc#r{N|c?Mf}c;#xV*Dlf*uS$Jfpz z%UgaV?&>~G3tH5q;+25tf?YP5r{Ogy(=>puxQz2mTHyPS%JN^stN{w*>FzXiVL>&O z!?tUC5rL3rVf!_7eG%?$oaG}pv0eQ*F*@{2pRWh3 z^Ij6R)sA8Q>bfCnDS8oqijd9S^76D4mXdYlJ>jmE!dsQ)bWk3;a5Cer!|&Y|%HfIa z&-p;Ed0@>N?RTtK=<_M+(a@P|ed7zzqg11P{!d))l^#lbb*NMEH}R-(cP!l@%{ zHf9(2i3QAj7|bIdLV21su1Ozl%Fj2lapaa&A;fpt;1h}y1zW7S_3-?R>$TxuOVETc z7;u+tL0w&sp{&jU_=^v@D;UQd{1w5RA-9SKNS62|lqWk&u<1U}?l`9?0*SYv5 zb80PKp{v1GUlfuRuc>J!HNNJGRdusgQ{!=`B;^rRZlULw`lg#`kAs_Q&9Zybmlm;Y z<1FdD`i|W;cAH&3_SyM{mNSo4_)?OF%X;NfkYsg1+qb(sW0}4cGIM!# zSECi~hEfk3@NvZy>#LpFhAFplsN}Kke?Q0Bz(S^bKA8f5B4T0&dm~AN0IioX-JTx* z9LHLS!yP$mP=szX;!67{ofX`zQGY_M(+|G-5+A-qORMp$?|m zTGAcv>4JYm6Rd_Z3?b<)t6P?~MQt}NLRVfXOCINHT$}`|sb1=l|DGX4if$5EwMI+%HUuHw z68<6-=Rs(6PRFZJKJrBiEXFXn)Q&)P%DpX=Q^CUq&kl3fm5$irq=J!IK50X(vC#u> z>N>WjivGw9Ha`W9iPwNiraTpFzmv%lR*R4f`>+Z+j(BdD#Q>V)#m(kb9?5?%bqUJ4ePK8zli840S64PSgY@=-g4-gA?1>}pc9uje(8%~a1a*;kQHw4@8;ho(91#GtiXm(w)2mZzR&j*_c2 zf9|Kph(L~~PX0@N*)zdWS%oT#2$XtBZmU*nO2P#B4_INU;g)8cLxp~AODLr&So2QX zyAZ_)Oa~?Wbet9KFJ_2*d`ocruNh1Y-4!3GMt$C%F--!a?sB`R#qxljBH|)k%$nIi zp$xCWIYa5;-eT60O0&3aDcVO-v+M=&q_~qr_zTYxgXeQ?oiftoIE?e0P-ufba&g1)nC!%LYC4Cv44xoDvhffq z?42`;nWW?G_f-FK2i()yG`d@FxZ5C?Mmh_`*ZNgzes<8Aq?%0b9!kCiJY^2! z&>^ugkg#ODTAmdl%NL0EOw3IK-}v?U+jH)5Y~CWfMaBaGon6J$wK2ut1l3Mw^nGJ# z+^75+a}`~r$IhFQS5xvfi~K~`69yLa)zx-Pq)a0q2!?CI@wS9U`FZ)h3j2deoH}{Wg&jK>GahckF3V+k=KI>0V|E?ys9?R+sCvX^vXO{0YRl^{h*E zaO4N74Hve@q|W#eO|=l*P)NPW_i=Sx4|8?1@rzkzP{EVW^MyeW3j%1+&e%8 zzZ=qyQWPp#Y5Ux+Cn0qspMfraDLbMdcAVmmb5G$~RDl_XF?;39_?e4-$=DUBPkHG_ zIK5boSlT9&(4Y6S4F58=*4B=$g>G*Mu5rz-)`ywyeR^7|n-_kJzHsezD9Y%iCd^2G#!TEoB&&hp?rY2Nzz(uHTZpMi73#an=897;oKAcR$-- z+vnQn+2>T5K_l5TP&1)Rb?WlDaiI?R{RIy|)T(UGIH76tbS==B0edP4M5MfbvOR~V z_=x{QnW!$L^VkzT+KZ#}xOV{-jl!U7x0_LhIO#-ID2aL2~6+- zgtBe5&9?*0>A_z?S6-hRS0pZ8_Pw1jg3m8P=A*k;=vjV$C$UL6obtv-&UhgS>nA+P z)qab-bZ<$t`$7-k^i9LeTbE)qP1XH$cuQ#g!1;+sRlxRMr{$TmC3yI<-96u6ETdd> zo59(K=W7-5u7svV%cs_OKe-j;pwTACzT8*h`w;0z8VHC$hQA?^k&;~!m*dR5ymfZ{>nY*(;$tFf% zqX2>{SxVWC`vq!i0yq;xe^>7sieoXNA6)4=it{?Fk0z$%kB-80u@k$pv9ST&L*~|M zqfBCq_+}_31M@}$7MBi1&8=5`TcmvoJ|_rcePgz`$+UN3)4N?^5f<(hH^{t25ivJ7 zGpkv~8||EGg#Ct9hIie=nwXjzhN75P=(-m$_^CW5X8W*EKPv#Th-teNVD2sa&tSiBsd}yZyMl=vR^M!l;56J z3YlG}9O^QnrRB4mG}ROW24^a|HKHTh zSHxx>#@L5@nkq-&ek8yvGvKqN0-l)0MYO|8!W1g1K+SosA#?ilLh7mzn{|iSV*JfYkcWN{6B9 z5MhM)qMm)dV3zA+CG176i{8@ zUl%l)ER~`GRYK=YC$ML~;`QQSzxYF?sPx%U{YTMGSU>N~xd(O+FR9Is%IG`YEK-MM z1xok$3BQV+F+Ypnwa6Hypd8^NlpDs4w*AWJ@6r$Tq=+eTh(a4X7QJ?QM0rhdDYwmj z;Cn4BRljai(x}P5odepSdW`;_Z2sp(`oK7-vm5NLNKbWVQ$9$9J)oo)@Ifmok9BI*BcXBg2u|+f3J0~qV`z~c9STKKk=xuma z=FHH~jN#$w|2%*A=I=%HRs%xH#UWdaU?Ef@xC_Lq=o{+O(XD;4D_?47UK@~Ns<#M> zSjIXawx75sUyXpX=j!v4Eq>B;TnH67GT`q%SJ7U!zF;B{8iNY;rqMmJowSXm-qRsx8ke@Np^-YvX=xM_+lq+$T+c+)Z^UeJMO(Z|GyRO`-e%4(k6*~3 z)c^U4DQTtr`UfAqii$y_M%tjCQ@1CqcX=N*I~U31VHI^b|L0++jvs~H6XJQ6`W>fW5iDq*zS%XG8cua{rKV%c1@37lA4gwUrn)L1v z?p^v?v&$=;h1}f-9nHd0 z>xz=dIq4rDt0>)ip7|l3Co^E#1f^bE;T*=y=}{7ug!47B#>2D&jh0-J)L&rCQuUwXd7w8ni4SrEiP2hb+*pFF8xYF5^KQY zb;OG)`36PqqNOp7fx#X8yIIDI_29-spd#bUd2(2ySTnsTC)4<`{~L~uF5&NYEKEwd z5W~_o^q}Ez8LRk-XVWNi()QV8MEz~(g=j&!xSZASM6t9pV^<(epJMvV_?+$Loltk zTIIsFNZL0I{xE)P%0We|Fag8Sgw)(-!oTyBya|)tnEYcESMtsGfnK zgFxYOb0g&xkMHIk+eo?d*tWZYBRXdb;x^IjT3C8#=sH8MwQ}(V`?9cEP~UA`{ZwNkn54p3x39UoE64hR!NGm^?KfGXA6|t6KR~E^ z$h$ZY$-+aA&Mc+5QZGv0@JI9`c5>`|;*UL1CClY-(C96C-i0UWbMH8yQKeNj)Oq!FUWMzZ=l~ga0pRD?)7wvn@ zeF_99E-cQty{TQN;=g-(iT=%K%k^eXmG7Li)01jd zIYUP)#i#JM!8Fti^Go#}H=D#5l-WI7KHb*%DAO{SEQ;4=jgjg$>zr{7&jYs7O|ca$~- zh&J1RI9j84x-+()NbCc0&AVy1+9T2m>!UK@l2f!eCgkWc=wXJxr?bGv-Ke$woDW>w;}`Q^9G9=T&COslH7?5Bk3 z?})bJn5$b1nK9WsJspEhJo-G%D{w5Q90=Fn9Njb6I{pmFNTXxEiMO>B$Ms~@2?4TS zcR0{39}!V*Mn}tzR=>ydGua3PiZbIpoQ7>8u z9Um?HOf~Uoi`RPLmU+Qc0JU?4OVPwGj?s?31=-P6F0_{&yravq}xZ!I?So;8m-EgPj?&5p*xS)1S z=|)S?A5a!)-^DTz!fwITksOe4k<()i`1IqWkU{g#FPo|q5!`*|r_Q%AMd*yB(_e9l;`)Sz%t)O>xP>mOL3 ztqJ2aWQd-g9`blr;%UQ()PuT7tnhoBc2dBf+nVU~^Uq&a8{!t{#uAb2Xb!w(eZ4na zTt=1cy92Qp|IRx93GjeGdL{?w-UR8AZ?1>L(Nx--L8n=F~Fe1NTEd}ien)7L#%J$WAmrSG)vd?BJAFrfJ$E! zeY`^MbN$%HJ?WhFdKCJC@tqZQ?Nre7Ql_sNRcP-j9tesirPE*y^bx6PY*z<8kz z80?O7$4-GRZM5tGou83Pf~vE?EUsh1S?@B@VV?|tLuuBAL|f*YzF;;GLUW&gV8oGx ziD(#NGTepq?)*%YtN-d7SS_mkE@OTRzz23f)fy}BqQ}H+@PpR% zxHzj~iW1BFQX>V#0Mc0jt_uIA2KmppyLT_e6|nxm_`lHMzd)j#H;*%7jZ!&p!)!pp zV}tfCsfF>U8{9|zyHyeW=!9{N<@Or|D<{p9PT%$dhl~4!tWQeehtG~VFhFLh>KNk~ zr84caS#8dr>D5EKo$jeaiZ~y^U%`K9G~-8l!kiU&rrKuWNv&m1JTC40&vk}$#!*5h z22Zk=`H(y1&-a)O$LiGke*Rz^w}?k-+1p~I%Db&tKdrb}dC?-zNY(Qito6XW(lD$+ z<9>5|)P5x9)MDC8*H89Fh>Dv=n zj#N%mJgZ8yVmm14YeC-cWAVrmX60W4`sbn*ykr+Rn-=u=d=@hW7mSXT=o-Acm(c2z zuEzxJBcU&wW>8}r^EZcI~wbHGLg%;q(U@ z+G4Yn*PeogkaGJWQcMTE5&Yg~crMAe$LujHtB%U531s>*OVwK9*D5217oSQqoektCJ@( zUBlK#QQ98pt3MDevATGs=8=Qvb6Tp5!-Bx zYt(x`!_!j?kH;;&99T(S5!2hKQ}6i)J>=!r6*j8mNt!kEv=8#;oM%}y0O{=3jV}M` z2+jPy&8g9UoY>C94r&Kgkm2Ez?u|(P83#`N;u+C|l{guCK1f%-Thh4co=b^VBTD=& zQWq~#^6XDNNIWW@0CU`=I9O862|Yo;x)toWwAC@U$E`~cjF~F?P?AAE z`lat0@b6ha!`jSZW{hHeRUyX~fRt60v<>G{qdYcdM-?%0`@yo0{k?zO8@7qb?2(E~ z$C+Ctzr~|NpI(o|V0{aTp(QDd-8g|0R_9aiN<8BD^WcDo5E>RR>V7m?vKb2d%sC!PO}DK z$t+k{^H6K&RbS~|V%|62P0=?QFdQ6pkkICg920|_YDu~nfS`m9=O6ch-*N#Y4~@ba#{+S(2uzs(SU<`H2(B z{lN-#1e%YOOi24H3RZHGNurdOOrS*uvVkl+Q zJ->;~a_01r_1$bMi5ZFjAWir1!19Do3(}ceq1R0DwKfi^W{B-J%a49>_W9|`NEBVq~eq$00O;bvhr zUQO_@M`k^RKe6H_&M?w>+zJHQQ)%*VD1FqW`U2?NyhTvtI z3mZ)fl@|M50Ddq$h8=FK~Pk=t~4JS&tw&{_WYj`G(j}XI@)BvBVISc}SiI za^(w0$Ne}KvDMOaVPK#nZ{^R^!cx1`Ne+-qY#IqwqG~6`2sCzYHng-qrQ8xEd(<7KULSviSArh9uha|O)zXEND@2bHFce&g(JtQv zyH)U!33CkW`EhlD&)2Q>AAZFe6TiytP?9B(vt7i8Xj8*!-T!)b@H8}%7SE(huKt6rfglpUQe*4f!d(So?;pG=x14fXJ&+|i_8G^zI?WxMFYCYr<2&? zZo!@ku?i$AHw#w%Ey#)^M>;k8S#rpdYyqZOf7Q)maP46F+V;2h$_Z5=YD9e4amWzkaNis3)L5>)$*VXpjI*+zU zQ7@IUir}Zf=&(+|>U(v=h1u`yoX+j0E5h>{{a#diibO{^T+!yq&grNA(s5&Rb0NFh zPbob;BF%InR^(?<$F{dyWbyGKo{?EAR|L1J_#fkZ3>3mbb$Si~2USYyop@&a)=Nmr z{Uk$hk}Z*Z404e9FZ$hLpXT#uQJ-tL_f09cv`3%iFEZpTq87s<9kUk`U=iy}Ja9UG zaW2dLs`>jr+e#&Uk<_T+sw^bD{8m0$ENvk z-Lo+!M)2{amA3c`VXaKVub`T&cfVkV!zu>)hYy{6M zxP)vbn9FW^F{;?%I5F@$NEiutNk>FwUW+muZT&Zwv{fr7tqe8G+#iE^TAaiQNr#1m z28!WHv>Cq?4zUW~R$$A}TB;Yc1hjmLg^ITxKdqnzhi)YuPe+vvqeouDjkwZq9|@~F1~PoWS&5kKR&0qsBbA%CLa2Ta8P?Z z8u2%${7}`N9x3X?+W$D&yvYN7uq=)@@%%QNNeesRYTH-ipZx0xU)-fEj@v&%9&7+} zw*MyPxfVYKH|%k%1>Q}#e_%7Yvg^z4H_#XuJi?}t4zneh7vv{WBFE1=E<~LsCo6z) znPJw`RG>aYDaSa^S35K)&@=Ga^I8@um2xW-&Qv`6PAxk2s`O?*X<~;XDaOGLt-h>% z=y9q`5Y}MwW--(>@lJ=u-0X>xijRMpg<;N}_T>bN-~yHjIvSw$2^eIyGQ#^8vdV&7Fa~rX=Uez)h^HdJM)7aV5JTUXWU_ElHr7` zj7|wvNAdt!9TU&H?F6T0KmmFLVBA64l0cnhVwe3CnVjd5B03Z``@`7&Ipi1;|g!injDw!nd{`I>V0GWWvml*rpYq7c>*E6xjn z?$u{9oj0yMu`|b-G=|V_K%oAq{WPIGOMNL*4;FZF4L!#pZ|>7Td%a8Y%T*VT6|oTp z*^spvrK&?_gQS!cAU;1Sn1fr!tra=J#qA)l&UQdINo`Nmh_nRFaMY^da4Vto)yXBK z<3{NE2=iswf9~19ST3#1j|B&HLbeynNL-RIb-8>9Ex$(lL>H&L9i`J#-FcOdu4A$% zvZ50-6uBj{(r>;1UFv+PCI$O}EF@!Dyx=8BMBvUL#OhWJN>NuY^r3fbdYY!Qot6PA zMKb5%di_3MUql=yP>d*_MS^YRA=s}Nse*WM%346^#gly%T)2K{xdRx*z@VsTAj7SD8r?%3agch2qP`Zt=(QAZ>Y6c&Ffo+rCImZrDBYbooOcw3mAvt;T+UZ`>(&T zHvbTQe21^XcE?s~IwlYG&ra!LDWeC>EFx2MX_88msj2ZUG3wm>*( z;T5XmvjLpjr7ia2e|HfRh+};zMM*9+n{lbnnIXE@(qXa6QYo)$_37Kb$x!(mtCo9K z%S35)pV&v2NSY7Cr6%YJr{ag=DBhZer3$iD(D>(=ou)$)1!Vfz2+)&KtJfl=-#mP+8+aflzzph?_W@T-=?`=P5;T%y0+{?Z6R9qLHp5`~ zFuV?;Bal@)$hWKd7)P}^pS}2|riMgu@bSZL4-MB2x6Q!IY}A{ihC3l%LA!RsaGzT4 z%j}OHqntdu$pOP_~$3y=a&>#|COFw10k^5_Oa{MWw>X#a9WT(gY+_%x#&>7xUf~VR^uw< z`$qvE1)GZ=T@pe-RRQ-;aNnN$Yp_-rhzf7JgDGI?PvgB*cRkGkN~! zJn@i0GyxUhD#b(6>@5GY&^@=Cvt>kKVWO9)zJY>_CugF^6C9D6)wnD;K%pSyOFkNJ zz)-YE7vBO4kq)ns5HUdu79g9*Ai7y|?iBr;{)DBD_3&^a(pJr6XwV$i!UcUQ_`4`P zpfsR(z|_;CgM|A&Ha>QrFg95=q1LidbgjBvnHo+1CQpNO)VV7hmL@jP{LV12RBCIb zaS?hljViYyQm|sZr^+{9?x(}#S!L`%lEOTd61Fu8mLGkJdN}7rou(3JIE{-q@eK1* z*0~8){615Q+q~J^+a$k~U2|WrvSjT~;>%~I>9(g-t;yH1PgVShmJW`^4ec^ei#OlQ z+(r)p?AKLjOfd;Un06+{Cm`w2&FUKA3hyHXM@b=?=xrYR<&-art8FH~DU=ivJk9%_ zA&qHe?Dc~qwNJAGxh?e+&LMH_b_~$G%LXaG#m47J?;BXp7t76WQ!r7uqpR1y3Q~Hn zjV!X}Q*bhK$=la2#-__$ES3W#m5T84%NxfCmcYOaw5~YeaPIw2R;0~^GF}50-%Tl$ z&b3akDIV*(MH-lLN00(^hZ_!Hg4BXvISa~uMs^O!CbsPPEwqQ%v`e-sttYi;Og-&0 zotQ=15yAY$RLfQqx<=RKzh)ruqH>noT74N#bEI(VNzvjekV(bbnWJB$rRDlz->PL% z8Jd5q2Q|jcX}qqIl5fSNV0{dXOvQZU?0UsM9#%ZP(Z6uZKcXe;cZj+9d6Q0!_|3YO zPQLeG@pcWIiuP8gR@l2E2a@u@iuU^oY{5fBPZm=bzmh?dJ=KcH4BpUl5h?a9U#HBQ zGBuZkk~i32Pc9vfS&|+B@pwV2;CPHSCU{#bvV z0Qhg-QAFqaaS3|S(H|HK`GN{C&U(lC&BWN%XeROeL~%q+J5d^|-Z0NISPvBb#qbXG z`72nf{PQW@qXL*!4Q4E&c44}BigqxjV>1XfD*p9HCF3ZOf3oRp(NV*Y7u7}L-t&p| zk+FiiLVE74UP%}YgRKhHOihp6K?#zjH9OHLXZjcA`eTN^YRm@Lk-m}uFSPH2V^37X|1$VVw=yxfTW`icg> z@H47G6&jeGqcfvBW1o?A| zCrrLoH)ac=6AP+g?cc+ zcI|_WaGUYtvN3>T_&Mco<;lOkJ~e^}qt0i8EkHCGC0#?WGOwz1#W`TLldG=AvEGjO@^70lHoF1WE)={j1D> z3T4N*uBBu#lta(rgzuCyC3n=3&D~-un@8%io z8F3HH(fV^5@?i}(=#Tb0q=A{SCb?}Y5G;l_RK?=gj?^98Sn^0gP}HS0$V3LhZIhP10ipDLIiGaXxIxQH`}+P>HX<+cl$ z%H(4kOJ@voBW8T5K~&~TiloA(xwEKsP)x$G+I2xx`f9a720fT7uMe`z4FxgJV-Sx^ z8y2TgdUJ{$LLv`^H*EH8(PB2MZa=x1ap7d2B?Zj$90&hI4J8t+1rn_DZ@79_0ucPJ z9R6X`&$wvIK(mRuR_!-du;cN@jlZ&*O4K79i^eU;UNfe&dSJMt|9n%ySm785&&)|c z+l(&>og=Kf84w^t*=csZjFg4>TM+tUAPEtYm@P2HNmLTAwW;Eu?@4I@SE3gX>hzY_ zhVR+Sq^nKS$7C8xH1jGaRgpP6>;eq>JC4DXxQx67@o zH#nl7XE{VD+@J3e*6m^f7FC#=x#IubPk0&~f}aJ|XhPK*=@nUh$+1kx?S#SBl1gw; z(KFyx4jUc+0tuq zPBUF_*Y&J=feRoZ&?b)Kzt_Z8AP}8k?jJ3)!Che11h{Hzu#?%W2eCPo3HLYTHd#ob zk`VjNMz>2MiY2kKkId*+O##I*Kj1t0w8~qG+oMU&jI+169`WRp5yyNiZz4t+3sM3t zJMRn)%;MEo#MaW4uAQ>ff0FD?zyG>!)J+wB-#BF$^{@aIZS<|BC0hf2M~SFtaJUa? z$bUizff;CzUsv@;-!?XrLvb;y-6n2DNPdYp%}6-&_u(v!d=RKfkCLGL8Wv3xpGmoz zDiV?DQ)Ypq%(mS4nmJ{g^|+n2!4JGq`=-qs>0yzsade!&?rq-kzRC;kw0TQPkoX@k z_Vp*%z&|~#TG3l=aB#ey)tV;4aXsNvP*Bk655d=;3FxPcFhmoAlh-cFrBjyNz+`>vfr94w1@bW`d{X1CrwW1#|@52Nm;L z-{UNW``4a}UUvxBKIdZT!eJ$q<{i-J%&L{O88Lxp=)036zr2PiIH=TF_knTqgdGbEwh1xcNOjT0bJ}_J3nR zJN5l9mJ|X)AVw1@L=iI$7Y*YNpSs3c0hx09RC)9vC;%aVSyKwEqtbWb7~8_xQRTZJ z99h@21$Y#@O3%jfvL6Z{ro!8d_I8^L8T+CmA6oroxxq-LDlBhhBOI7k?#^>sXEM0p zo>Cx?)5X+xlm^N}ve$QR9CVm38-@e103wJ-y1`reG8U7CUz#*sT~(c_fYBb6s5Jp)MUMX-j_2l0;We|(rWh$IS- zkDjK1!Ow*<72ip&*B^=MwvZF-muk8jB_+nON~e8|aKZMbb`^Nn)KQ$1BR|kSdWPK>}d2%uzQtZw}eZo+7<+nvSF+^561>g+cHioGpt$hw_Y+Ay%(Nbz(M>1iWqR2cyI^ z(*qe10&UYbaWFymJGj!@nSVx97Vf({r(CEQ?Z|2;|DDjg3iR44VsSiM;af!;I!@m59GG9%Jz(fpAomBUFqNR2O~(1mUrZu@BG}x= zwKsn5Z_{$X;Qw+UbU;S5K;5VIkV(Q5>T7-K zXPb)tRNZ%c!|HPU)j$07c|87benIo|Ja)v8^;K?#De#@BpOYOLF<>Ubq>1d+H>mHt4uJ1Jh=ZRdYSPj z@&G^R@l@?VF?+1fR2HT+dTewKWxZTE<;&dscrCVm*bc@3r{7L=bhD)(VsUub@z!Gh zJ!MyQb9nebWW80g(CO}fX9Uvb+uL{j%tGe2989`^fa<{|)wvdUzz;NON7pJJHR-=< z7da@yaK$Pq972bdKPyvJt3*O-evZTQ2GJ+@@OaR@9QrLX3;~@C=wU+gu7B!!&F%6B zGN0vte=@pnAh@1yHr4aFph^L2FV{Vh`3;d=n6_$`bVCrVXGdqpDD04tgq%b#cT>~n z4U%_I4~(|6p<+RLM(wc{!4L3-bFC#oDh}933UxEfzfqxJVwj%42GP zSum%ipC6G##uX{MSmN-IHXvA39`0p)06MlJwi_~0Oo%1jn zs;phovPO2FrmI2&cqFh(q9AONtFQ3js2O&U848lLa_}>J~Uw0rv z_zakkcYt+Sdv(5}fv#B#cq%v|>i{Ba)s4Wy;|zxYDW=fLzgc7#IWIm311w~)jqK)4 z<_8F5WCTB2pMz`*gVufe+`v3R9LSWVCF z`oway>gtL#Xmhrajcmw%h0>fQY6eiuD7C~B|MOg;p~(--NoW~3eu-&*`u9ZscGcyL zbH!1`cXCO`C8Usv?#aZ=P*5jN-%lD=pXEKOXN_kT(xP1(g*n#Vzx}*_WAnel(f{|| zCI*P(#ai&|WhdV5V5G3ADMd&~h-7WGXXrgNEaA=#jeO^fFFOr)5CZoJAEA2=*=OHo z$8}8OzSX)2(vhrw%Nz|d2kC5oQ!G1sXEzcwEP-yA5I14+OD^;2W+LGhWxMNZ5?z<_7_M55F#4VKeC;Csm>#T9mn)>(dvBe%dIg~uOE&A zj!V$|-8qg?MgilCo0p*48xo$nrjX+%7jclM;>aZ>_w#T_8eJH-#Fu?_8ljzCzVm-@)NR*3+YBFM;>= zl}yun#`BP?HYTD^(of-2A$@gv;6U4j?eqFyz?~=v@}7?7W3L#7ao?~U7j<5gxvLZI zzk&2CnQRG+CSaD4yFN4R;sISAUST;+ZplDBZ^hv zh-jVIVtl-KF$g{l{@t?iFHv4C`1k#y)mkF~4#Y2vcjfP8J6B%GM)P;q@-GiFCdlJ4 z1Ry~L4sEqRh)DZs^ws#W%dh>l5#b+6b_6bG3ByXY9{9)eq^Iuf!MN#i@x+}Qxj-U& z0JtYLAVY#7!}rz+ua|^ED`Pd>QQeEFQH))CbrZHXs0ItAg!Y zZ6jQLSe%Bv(~ak9^NDo$9L|F-J6H&? zfxplu)d6{6F4MY&FhrT_l3w#NoHv;aet@I)Qh38#cOQYVL)2$0)~M6F_ls5*+MSC* zQ4GZpq>bO{T2KdJlU{tD=T-I_aBV?A&)<4we+nK#IT!ZBANFpW_c?@);y2uY8KRm7 zvp_jmZu!CVb5wd|%6lz1k3VcpYpE z?f>8u|6j25-+sVa5g*=SC}1?+T`|lby!*>>kfM3#izT)I&dA$4A6CqqcY;Z)^n=lf zB*?i8JSP=gVS8(72MAuly1$>JGt*RWqtUir*;~RHV(rosykT(=vY%PCrd z$emw4;O}Y%?xii{GVqU5YK>0cPIQ~C*6V3l;JJqbFocUH{2w22diSt&rxvlj*<)Ob z%po>Ru_N$;uS?RDpQFyI5*d&s;;ZTUrmI`*Nv!U3dY!K>_vD>D;XE2BVH3|U<8dA) zy4k~tOP4nM3%HlBat9-H7VU(!HLyW_@g8xE0+GgZjt;j5m5yMw!bAHiFA8knVJAEJ zWG*YP3Sh4`J);eN32hJ7q)r^IP7XphjuJcKO$3>q;xnWRS8ZVg5BhwDcY|=(M0O^p z{qPqFT(BC>h4A)`2c5n@mb6u6J!g8~=CWO?4N?vdnd9K>2RawgV`VH{pT${>+A~{! zYC5S>Qtdk(#bwe-fy&e|3;_xNjFO$T5ysc3VjxSEXgq&0* z%Bco!9;}5GWvoB-S{dr-5%DJmb`_3_3;YKHym19oru`C zJrZ7srSnHb$0yYv;4)THH#C`_(^HAGG=Euji zElUO|ct<1G2dW-ngsIG?2PxnTGA?+}{&7anoi55QjSHrg7zjFyMqnM+_wF*bHa%^p zB7{!}3UCf7bAl|Kenr`n^Uk;&sAh*?i4xvj^S+1?)VcyT*xdZQaY=sQJ5 zxl)3{a|`u!w?@vDC4xu82!QP5z^fEwhqti-5xUxmi>nH6!9iH`u*6^>FXbHZ+?mMJq7+kgy<19jlq&IHSIC%NQS(2Z)VXShxKc%%j z_yB&)g?D2(XWNxxs~?{;WBo(r{>X-vyl^q4TyK{ii5oAt>)rX)fZ`2%9qZcuNhF8k z7~DCY3tZSs^I%pxoRMuRb%D`ZE|+&&M_%~5b@8>;mJQn)7WuITL`Njm&Tr^zm|g|qI$z3*p~fca)eG(Pgjl-?-) z+TC>*w$j9y>bu|iQV9n{Cc4ZSv4_1a28EG!J?YX5P2Eh65s z->*Appk&4S^q$0g_agObA9hTlx3J&La~$<6W)7M@p(MSfh6nGtn1AHH>Y)EQmUcMP zdUixRDZEcsLGhc-z&Sl6A)*q0kymq&w6G65TC1Y-`#DQW1xK9&7tcouZ zPAp(DHkwh^^s5KdRYCL(LNg2!eT^T0TKJGLEq&&0xt65`o64MWbk+EIGgBm$!k;~v zR75wX*(_tH2_1cz#GcU?P$DZ|-Zbyd9~|Pv=*KS&M9HRF(*K;H))0M=av-06N=7_c z&chB6yeOKG8|?>-sY8}B_!Ep2G;t4|WE?EzBJB zltE?5*!{ih?B9;(e?ZIs{o?)WyZ4t8PVebz+e0v&)h1`NWw$g-#10w73ZI-tGcXXU^GIn1(OZcem_hyjLyS9bAsImJhY=5iFQ_|o-O91 zCX@NTc)pcNlHI{1h3csuCy@lN4@c@DM0n7&{-W7^ zwZ3D`%&!zed@^yZ4@%2A$V2V2B9X(6gp=dUFWs;Ba#1ouKSoX)9p`5mj&@n$b^Ie< zdKy|nG>ZNw(ofxX9-{mlmEx!2 z6PEta3r^g4hH;r{JW^N>T>`EPcRavTwm4|q*koR=15+V)c8@G0G*Qk@baR;>RZExb|Y{7NtOs`Es_bKMnPlu4p-T0T@ke-MP$` zj0`{cl2Kecju0qU$42RC>or$l%r(fgR;S6d#P3X0@MV-l>b#a`u~~x{3967z&r0n3%fm_prHl! zhoZffVObMAjka-DL#b(c+r*`S+ zKRoe|vgv+1t(0COL)mbAlzywG!n{nP^|Dan$dI%ldiY~g&5I$zy0MHKs5lDRB8C0X z$&W|zIp{WAOzm<T*J7boodB|a*;d^$^m1Z%@K8q5(?L5;-`XZi!k5%ZDO$8EpiUEL}54o9jpcx8dBSSbxojTP7wBKEk82n{FBPx-QOtwNr)=)sCK58&p zKpKU)2MJbiU!%J)Q$hYj^iWHC(i06t0e52pm^{-P%rX)rj1l`^%F?}a<6>95hWyz2 zb_V?puUrF8OLP`7oPsF}liOIXcIWVCtgFMGV-W}^?b{RAsA}x<%HWRLz63lD9 z2axKvlVTiaX8{Td6M_8HGLcC|4M+u=G@bvQg#CXxt*daae%))0?)<+_8^X}&Xm1n& zzqGu(A8qHG?Ad2Qs%6i~d?~$Y+}4~ekeog1%TC{koYVCbO>X+s5!=?ztR!w)QpbHOpjz1% zVr&bKPH0;$h)3B!VcqU)jmVjQ~Nqcv-sZ)!c)0wrv- z7wcRQGk-3gAY#;ACyh+Yu*3QE?zN3!(Xn%(FzFg8a(pIo7HM1q4X6W!r4q%bj4Wn5 zX6)?kU5cx;*Qpr0xl)r%iQ1`egvZ1^n;Mw(a!0dA>*g3>RJAjxIG`h^L7$JK+Lpw` z65ze28XR)Sq0xwh4l;yFEBSS*CF*5#+hK^E=fK=<*^bB3fIZ>h60qZ^aNqvvJz8Gj z-k{M-^gki}8OWKHrM8^{mvkwg*o|1IeM%KN&UK)wgA2WU>~B6$}j84JL6DD1NPP{G9_u=XNYvuU43 zYvx7<1@YO}T`qb}Udhxv{jdD7%L(jz?(|bS1|?h^0;?Rna+5WI-{c^kXV~ooJKWcT z6<^{PK$1NRAOpocRO2*4+a(UVGZo375^*B?o7`ZP>ng(DM69G4@*BTLg7`k}@w1lG z?gCz4w>`>rHuHSvMRzFSIH1X34c-t3=rH}vZdNmG`Lw&*z#@6DV)jYz|b{zNPA!Jtn85y1}Ig|(f0v!9{< z1d`SItBLn7ZH2ns8-2s3C`IG!g|9IGz2hldEhV8WP5OfKena z@{ql+b=QG0_LtPRRHG`>5{x_U$%&WKsFj@s1NN?rl>IDTTZial`K&LjcsgXZOoawe zYNv=_Vukc}I8}6&5@G=8EDX2^!zM#&D&Fbnp=yN^1516ELz09!U4QFm_@B6J&4#~a z+_*E2s^UrP?RRuAtfM0`x++po6q`n->!xXo9y2==_0{e+*0*Wmj3h-I0o=^<=WbT1T9m~!G3Zd%a2i(hL>yrQ-7iThtznKRITQ@GB->3 zf)FNiL&hC@R8^IpUdCK!Q-&0gz3Rf@*9fv@SF5acLc0fbDlLuhf`)IczT+V$`?Ux{ z5iT?r&}$MpgU{V zPPew?UqqiC)Y1ZtG^BG$-Fz{OrboYU=<2$;a}mZb@~VI~Ghn~n-6=M`0hN(>Of=a) za_DcZ6-NcD&US!r_<4fA<6$cheiwkwbH|ZAZ0p?|*Tf9zI*xo(4Ir4!+N;^Us{m47=K^ZLct}qP_ zZeHZ0RCJwpsOGmM#%yFJ{18p$LS6 zDPMC+ut>y%dY6*-i`ZFzU_kFXXxC>6JRy%2R!9`Bej*n9e122v5fi8R_u=?|=z7QS z%A+NFxMSP4ZQHhuj&1YARwo^+W81dTv2ELL?mPdPd*{x~$MZbD^J(u|wQAMcRsXEl zw>^+7w(77+is^&%T7V`^$n+LH_j6Gk2mCc#7iy29QJRi@jq15USdT2K($E>Lmw@K$Q+(fJ^YI*c5%ap zUCgHvJFj#m)JpqlrjDOnLFX7Nyv}uP z{3nRGucYvmjGj-%V6I)N5s{`t^js>=tHdMRdQFaYs`9o7eYxt01HlEd;e^=P&4QqFzr?LC&4m$@@JXR|-EN zBoxx?ueEg&gy8K;{Gi+5x3_>+2fhD3;gMBWp`;m;OwYH>TA)fWr&aZr_?_owJE5j( zDCxARP>Lm*NXw$hqkMJX)JOr+szbR6z zzQ?p&CUWYsetPA9+dQA=e=1P$KhZx>L;Vrhu{}3bvG56WJIknHZQ4z|_hwq@FTDsW zdOIGC2qB-hZG6EP%wfq{CnjC#QfzBQ+WT{)BMpT2=u5{u!ctcxPJ=A}BjY3!fZUzy)6lz%7x+oB}bgHsp8<4OtXMkMcj zHzNiOkV;Fs6x|1&`Q8^fIfcbIij9@nRE2#qb|Mmv&vy#fad6!tN^n$)s6YY-RZ%mI zY#Q#U!Ek8qN&VRs;5bgwO0D-8Esx4|#9TJd&l?;yUZ=D-2aRJdrI>yzeiiAz03)*; zggK(r#1iRv?W;!8(sQ6LK9L;MjWViseVgX8nC11cKC6ETkua|YT!Zu_i+~pv1-W06 z*B_33jfzsw)C9;O^};9wyvvTh_T!%K&u&z!V^lV}zpxVA!^G4|vNM23rT9C`E^G%%lBbTf;&C>^`mbSR5I^7~)YV~{W*H!w@;u*sn^1~*S!UL5Z5 zw#%#(?hzq2NQnSCq*cX(p4BBkSYjX$U?1@;02zVo^cBMet!V?)0I zH%EE-JG`i_+_N-`Aw=1xNzWF%#N7lDX#U{?$UF~AA|Fc7X6w*lf3fyUo+gma^C?3X z2bFT?Jv)_OZ((Me*UOjPfmYIK_@x9xrdNqZK1G1*eE){{`(u87j4~3iZHhG&k+}FU z{$V=Wt-1*;xb<}AWRVHNF5Mp^hpVX+aTc`pePf&TSlpU{QCGJdmYPXmw#GQkobu5s z3q7UP`e1@rEsi)zx=lQ@`#W|L^I74!lEz_=?^}no@yhUC{@{#@haCs-ASl7c;Bs zmq+k#YYOe|Bff@Wyi|Ne8HB#2VeN8Db1pVq%;fTTj{CYj-3eYYNEvI3dkyD57Q?S0 zBwcw$m5`-*Ji+K%mVs~^8`3JBcT?Sm8YcK@97Ts+Kv77-;|^w(p|{*A;|Z2;Y7W30~?EQ~UGfUn?{l0Nf7?r!rRKYLTM9e5tq(_FAu4my5Qp&9<}x?X5&%(5IhkEE_>OpA3zPERJR5^7%a!c$3F{ zGeC&3(wSfPaML;aF7wD|y4Cz}^R08*`OLyXB96TJppPA#>D5xRtqPe(V`fcBn7PO} zg<9A4<53OT$lec1&%63}7}E}s3yKVgz!##0Ns*D>n$WkL+frwe6Drja3o{6C`So%* z6)f9)zxDp0pMMQ0Mbjlnf#~&!uexU;Bu14qBfuyxMwpBWNiaB{aKI#6C0#Y%uAUA7 zYwb)Cz&@XebQoLgcw+HRXSS$b(EhSN195fkRI)k<;#tMv1|Ck|7R zO{y-e^M%sBWPtYrv#sHQ|f*YofiinL`H?d#=5nPiO9R+96T_TAuIM7X{_ zAF;k{zAe}3xNxr^xFC7+{H!R}m5ZZc_9NyzKBZ~4i?f@%)YUW7?gKYgqmLTPNCy}6 z>JEQLG5vU3mv`PUx^YYmio0%xuLZym8sjmj2P zVb1&$4^v%$5Q;g8`%ijg`>13)J1Ao9iN6xkb#6t!cLu+P&Id#*Hg&E_iZkMb#Xsa* zZzwuu1uTrQVcXsEDEEj4%ny}3=Ec&O_~6mtPH{Zs#h(*WdA*i%|73V(b9iCq!MS^a z7|a$z4F_v+-#E+yx{WcItU^pHw?ewb>*M)WD^#P&=W@Q_OsGo2;$g$(wcuVh^IL}k z1cqIWX#r*W4HV(-+@61y2uuj15Y;l6a|D$_d@JH?3?eKdmQQYgq>5t_K|5>Y;A?3? zvE`-dlo^Yj8jLh=&2Rq_lG0VRQwy$_Z;{6eh;rdbwgu?3M(zm$}WB^JTI>loUbqOGj3Lk^~u z)3Pwe7TapZGq>x5h{!A0pk%Sc;l$1!c|m>X+OfBD>-FAolu%%@z>b8hOKn=;4Qo0}hF;P=^2WBJ*d&8!p z>8xp$&vH;6*KXQ*otsLFk?fujC~d@%99q$ zEakwm;D_AnJA}tc6}71-ab@gre2-uzpA1_O)2Vemut`^_s5CU<7bBnZak@z3aN}Z? zhcHXx{bJ`v%muw&9cqwt?1H*em8`RkXcg1ULMonk*#2Q4o7st6B#w|IS>b#0(T&R{R-&n zdm0m8@TGslth)m5)!gw0RI=$j&AslDOfCTRD*HZkGL$-RE*}JiaX82N4!=B;=3DGp z4J6oB*!NzeO_Nv(w>0#!{6C=hcidP4^!zqZF<-kVN4QtekIj%YTj)^Xdkr#bH*=|* zsk=hOr=i%{@k`^mtdn1S{Tc2W)KJ_@_f(s)Q86w~o6dyCqO2!n9Q||*j%fmf{)avY z@kU&}EfXUb;iu1{Um(bV4RnNwAkZLz?}v%FSKYFVs<&HW4)y#U{l>g&>5c&Qw7rt2 z2+JXnT_5Pg2>R7bL|cQ_8YA6$eKFrOoRCMh(coHR5IzeV|i{jq;G(m_n zCh$K|I?#X)7_Ho#klGI^p%4_ZVdm@s`6arA`1d>$m3L~8oXMaSY-FTFV1k||Or3H5 z*+t6;(6Yxs^YX?R6{WG5BO@h97l|tyoJgahS}E#Pn$5O5f8evwCQeFMA7X-l&S~uagF} z$~U?|L5TM(_`~~lTf)7Ij0Ys(ghTT}(PsRIZ<+}(Cc?#RtO6-1xX=r{nFm>O5(ym( z2AJ*7m9bry6OOhV*_*y!$PITJIX9KY=v|CAp);fhqTYN)!;eDQbYa6NBVl#?U_%Cv zOqa13PD`bW{i9BibH>``ND|*P%dfxDN#7D&{A;8&`1ms_+)tQjaSDqrQY#&RixQiI zJ}vyG?7;;kiO)*ua$L8h7CK^4hM{JTuS{j4N`%1Pq{-5mQOsj}!DMc`N`Q|z#muBx zQXF=0*&{2h**6aFX~>snrIXE<)ox)kRx@E)BYNY_6uBHM$G2xH6#y~RQXw@O@^iUK zIK4cIQnpG|^uZ!s156?w2{%IZ1dw_ZS89l)Jb>8K!!fKNgB@o$dJGhE;{HBXF#zh)O^k!$sEGIBiOodQ8+t3B6HMUFW!!SrBMe4zO2%u7ETwof;3@$tmCGi5kJW&~~&8EwKAf((o@>5GVr`=+}Qbn_}y1NdpY5 zsQ~i!V)!7S-BwskV`~uY`q^0q1o6sL}q7N0Q8z%9GUq^)?J=I}KGkpcaF z^ODUzIK#-2I&rKf zYU~$m9ir}Ik^Q`9%CR%0lff=91qh^t(&_;*t1$-;GPYuuy!dqbY8|ip1@9aC{PGU`5j0^g-mH&kjLN$0ZkGY6p^qBQ;GSr0~Tim8Uq692o5lg!i4l zPl|DF8cQ`RhzFi+hiMX%)r;-tu^-Bzm>}ep1JrupWZa+B$o`|8`sQ|f|y{ijH zxpl`q#>l_Pv-C z{7H8oz!&er9lx^xjHlKY`oy8)Fn1FH8w)Y0WywEohN|QVx7WL12CVP^A{*(Xj`Y0k z4t$s4nerMNo*quN#7&Z)T;dr%57$dhDdYRuAa>x|m>h%6oPzwZ;purv$iz?ujUoz|Zm->{Iq6WvR? z4TLsNMs8ias;EMiYg3P)(`!N0%0Z!U`I%W@ z_TT|VxM5LBK|?i-blD~R!faH8tG(S{~yV;166;-MzgjT-J;m?qKQ?fA5jlT#;u4eL7kKtF%$-wk#u*MN-9;f zapk`7$BLSd)Qw_%xvgzpL#Bl@A501RvPU@%<#Rag`)-*67Z+Tq%W-$%GAxn~2&lQraASjY*jMJNi z;D9UkHtU{qnOx{}=%3LqX-^8bM_tsA{PH`1&g64<1Y%|u4z2}vvO}MN5ki%3xq*h; z=W^I+?4+kR@$N~NpHdWv&Z!4)Dl}JG6SQx9RrUbH6QVzP`j90qdNkSm9Uz_f%>ncK z!PO!UqxcOx40%TW)(=Uuqd&Cup)o(&e#;ObDLlWp%s46K|J-m;Gkci$E^d-F8R|L( z1}qePyYW}>IL&g|*c%ZC>bH}>~h#kJgvmGJi5d>{zuOm%n zbW80sIW!ETgNk{8v#_g;ObdOUS0tZwq_||ScD{HaiNJp)ZT%rl&1bm5)tM;uxsz7! z1ChvlrnE%cL02jCBjg;?3HtM*S}rK&^(^zfuUM(8W4}y#lw=1R+rHQm#NGx5Cr9*- zSSS+%H3Qn#RNsMe5=M>1W(mQhh)(>V0j#9>1~ChyWEug(GrRYHI|@jkNwl~d6*~a$cygp08i3r#SD?cB(iR8solGenr;(zbsxSt> zl2O>7C!V=Xn*)zQK?Ej}6cS#rzL#HvM5ieU7UsnnWwWdf&VSExZ26+tPi5@OHY{?$ zucYid45GxY1U_H>!fi<4tEV3+vH6tX&Ym_akFfg$e>Z>mYu8hIG%U^EOj?HT$L*wg zo+!^#A@dj(oeb43icH>TC#gsmjT4{0PU{5hpGsdFS|s zxYRC!DDob27jpfbGqNOMQTX-2oy3hD9kNFEl6j^J%=8^6%vZcb13Z=XJDA`MN4FxKsv+K^D z=qibF^?fI4!-8jfW@FA)ZT1w2_%g~psL^R%isV#O;SJGr_`-`Q=i?>M-h0JPt|_KI-O&>O078jmk23@( zPHm3bHmHq16XA&VfqGDkrZ-7DpJdWPLqdc|&sShNmhX!A82-+AxOWi@e8E5?y&<8K zPsC+|Z{e(wYSk%ZVq6FdI!mVS^({ZRFs!nLU^xA!O!H61g&_c(&PYn97fY9c{v*e3 zc<4b)^LIv|DD({mq4$Ed@kUKH@06QMoJl4M1dYWzuh;KVX<$Oixh>+37Ikns|(At3ve+ju9rW*i{F7JED2Adl6=(s3+ z%9RmMn`U$(l(&pEBu}1{4EFGo+3g&&Q(E_Zu2ED5Tr+08!tjzA$P>b@BuHN zA?a-BxkO2OP!>#u_xq#_tusibJBHI_3{)ayw~)K1JrPl;iZ4}SN))0iuSM;d5y$fm z7F$8Gkmag3W$vp(g#pB{yCEg_nPbo8UsHnY&_+(>71wtu-rJOGr6l7@K4zwgNu`kl(dWkieS>kK8L%FL2B9keFvY6JAMi=yj*kFU)qywQaJkL$K z#EG+ArJM+!o~Lrgp^7WuN`Z<6XD7=V1~p@=I*8PqQqV2Rx4=Jz7j_QeQFR=TVM;%@ z1!sBk6p1YMK^MRGwm!Gjg7e&g+TcNX_74v+Z>&)(soZfjMMymDHrKszsh_?IxG;aJ zB~IK@$=W?GejM}7s8!T+*~%4*DfK$1@p#2~9am#|R8p45bWO!nfcM*XY0+vdIa*7o z47=NR)Q;QY=uLK*1&oh7_s+A)@1$?O%~XNA83CS;RpzyK}1O4mo!kAQ)-Ws|;zEw49iu z>KjFJ;H}k^9)HAuLj4cv20X=kx($&H|^we|6Pe`adW-qQEX-L zO)zDt$nyeT1^j;RgLVQ#$>4QhRbBcF&Y)s%&|caX>3_}w;1(Brv=J`HlZmQp zW81`dkcc1m+z+eKamg>6{&P3t-Pr;D@ABe{^=1FxnZ`jVf#}5;pO0Iwh<6^N+>h_3!kDsjV^L zmKVg_Hcx(39=UO=DaXkrrV6a{0)qg!{X~$;O;Rw>Y%II&A9!xIh>7BY#uG94$Qbj# zV!{_-xW{S=?&)3?q;TSNQ~|tSI77@%F{6Oy%~ha;#ncBL08|}bVYuaYwjoc(!U;K{ zbu)u#&}Lr7F^>s}QGVvO-S9%jr4YL8NKpb!^+rj3OIVGx(Q3*!ie@eDF>i(wfH) z;JX`*%OUmmW)R|P_riT(0Vo#0>mThIpmfFm=V334*~oQf$6ZPOQvNqt>=zpnD!E4A zZ=2}I^c~I1$%N3ua7z1uR0+L}y(}xWtdqs0XL-_}>53o0H1vzB)rB*w0o$|p9f=%n zu6cQnhnbjv2Zf5k7ILr$ad&ukBmPb2pA+ga=8qZ@H{yYa+TDDatIg3F`%lsqcqf(% zA?f{@)>MStB)v*0I^6*I23LE>_G{_rf35m~VtW{X0#~PLG&vehI5lg*V2)%(CZ1;@ zDL9;MT`9D*a+8Wl_s*Wv`dG`7TJwL1h)4H41ZNK{S;dN9#p>tI8?G0`MU}x)&4#Dz z(_S7;q!r$7lFxJV4=Z%#LBhbO*rICQcB{lE-f|?nlKWKVZh*iPSnVm(GDl95(7eEx zeh?mG{HdSV#gg1Qj?h(&TBs+ND#Yb8lQq@{)*2*?j*-jNLs2d+js!5+cEH0Y<8q(S8-hA8pB-9e z`o!hzK?_kMM9e5|;GWW{jPdjxoc-jg>%`=10jDCD)>tfGAiY3^+<2??VR(-S36ip> zb23I-;a|ykj;wz>5HkMscJ7V|e%~cToHqkV0v3or%Fl~ICsfPYXP2XCd#|$&6Qg!& z3~fIvcq=u~QR>KPjVmpk9n3j(P(CEK5G-ALqEEC+l&gKI*G}Cl2MT2i(h9~sA$Q8y zM*KNp{H>`a#iX+d_oPSK?cVW>^+epLMUiZ28;Fe6!@h#Vr+b@d^}?L+pb-N-6SJi{ z^{t-#s1G&E71PSJ@g&_UNZW1Q8>3Ex*P10gyN&SW^hi5KuD>hS(u?y|+5{WjKt02kORVCYd#&H5U~$af41g7{ zcQpJ_MRliTD`reEwa>P8^TnD_YM7U(lixZPAGsP=?Oc@n;Eat((rbPruoWC~o-E;X zs>))wDe-E?iby&`LR<*|QtER92I@acXvu{lSjcoO^M*e=!pCqN^{AgLmF2YQXRbRi zGl~+3(B?qSpJa~x#j>lp&U?=JtK&>QC$}fmy|8w~BY3o>zfVquH|oFQLZ4^`q>!(A z^{DUv!pG+aVngmgFEyWG@p*CCxD?)Ntgap<;(}-Et6z}R;<8>&WU(WlXwGo@?>c4S zS?s>t4b-t)dDF6}!hi9whT$9asxvtVRP3M<5w;0DF8CW2jC>(TGD!sUE1guUG`h&%o{n)niX?n_1vlDRiL^+} zM|)M03o_Mw4fGAt^kd}a=u*xj;mar1DXmmFdZptW`TE=&IOPB~s&+h|Y5xj%yccK* zD@)swyhb5Sk}~QV*AMA|hZxn4^MejXkVh64SQu0Vw-wA;4*0XDMM?yw%0q1%y{u3m zl;u}SkCpKpj-grv;b$E;;#!{#5PAS_H&L^UH6hCE_u&4O?UJbtC0Vbo?>(tV%R!Mb zVw$Z^Y)$H1Os;szFC^q4N~keW4Q0QYl&vjn( z@d2@BT>wHRKe&iqgd&jLZ+U{9SNYqf(C@s;ZDW)9``rQtZHt~euw`|YNE9u4{uLuTcXp`EU+xkORqtvyWJ$4_On88nNcJj-bn&#f8|LPjeuz_+~ zR$I5+X#ZbPcFYWH)I0*32rLRAVeBRjZZ<4N%*m{O4>^qLAMABg<%2orc34xfs|rg* zEVFwqDhjR}Cx&cvQO)9N+&YoNf+jm0(A6c$vMhxY6tLGH$oS5*HKpzNC<+FB(;{fM zlol~!XfQxVZk(_1gIo&EdS1q{nR>m#0tOIKDo#8%j1f$1Ld2C4Oka*lQ^bXG8-87axDH^y zPh%XDe?xKtpuQTQTx{OD)MB29zMCkY3*5QpP6725n?>#mN7kGr(hB%QC#&A>vynjT z6Xs=+#>YczwNl~CiUjSP;RK9uj*h3Ps@QBPM15@#zhJ3MLu?79DfL84(vJ>%aC|u6 z`X9ag2)Q@q_zTm1o`U=MZl5R>Dk%N;$Mf@Z&oBq3ZTKGUq#J;N$ z_+63Y=Hln@Bqz^Or$<~&csu_ON0(3e!W(HSv%Qo?TOi^Q7+r-*nA2fBVs^iRoA)MK zaFhfptu$T#V^ee-&t}|hnLSYxG+yCG@&pu?BQ8=(uB}?;pTj02gxWY&#s3IJaZ#k- zjA}QGzjZZZL#TzTXNtB*F9Sr=&>$=33#ugy>NpY?A+`d9!sQZ1dS>$O9oUI=TLw`- z-4cC2<^>q1EI;zZSw*hBJ%RH3rMladEZXiO7H7zaTW7jBmzNM}_{oWlt*oG(U=13} zS)w5)LJ~_o=L?Wzt-X5!B(=XTyfR4PRsrUy41m(4&>8GR$s7`;dB)6S|XE|6VuP{aId-+dlIt*5%CkGKp+_2%KH|0W>3M?=nBef%i( zlpyGjWq)ww1r&8Npv%q@vV{17aHq85mZ6rr=?zQ?^h>BZ&Z7$WsA7l5vDex@3B2E& zCUvz)SsAK{qp15;Jf^#&TYo;I9Af|Ok#m3Qy?hFq)%8V7rwLZNaWdegUGU{<$HI!l z;3ZgR*E*doV#iSMf)H>dh3NXqdp}cGHnZ%PFP((&`>v%MOg!P4{F&NY(ZMr^h3AF@ zzJo0=2L{NRMxH%=@FID|&-KIkcmLV}qzeXcDz0U9trbR7M&>S%D+cz_pJ8G92!+Bh z2$Yi@p>I#X?S_ZK%h5ubYAeY`%<|9WKOJ}*cyvy9xIj;kX|;arlm0mo%bNXmH`-;3 z^R3tcX`pUbY*#UHu~+;olP=Vhmtu0q`1xKl@u7iV543`pHo1+$Im>bH-HlFo{q zpJl9CXQT7N;N;K<9wD~IxXkMI4>kPSN{yY8J{I)HDElb$PPH4APar~We4}-ZIVbW1 z(jsEbliq3!&-DL-iru83!VjDsEIG%d!pwPuJUC#YJu|2CO4?m0v|fLJej3zjbcqZx zi0Zw7lUn@taTH=AVLCZ1xEBN|k}@J8!S}<~;uZ@1Gu_Q6&x5S<(}6J4e%-owBHm_( z_+m$A-)hXfWXA3QoiyqKnXRZrlNik^LB9osfIjP->a;Sye-FO3Xr3Y-q5_U*cBTou zU?YBFWRmcg279iJ2>g8_9HCpLtlBJDp^40k(YYqC*5w})XVg!2cCE2Ffq(?NOh!v# zd!eA%QBq!3^vemuKOl}=d=`#!w6ayGDaaroZ!2OxDG$6RDo;2nG%kx!;oz*lUX}G zlO`H{2GS)~J|g>0fhQl<_nusw@v;Ph8Y#+(qvAvxlYPAV;J!QLnRuAAw?%m?;QC=_ z^x-BDTIJqpoT5RG#~L}xQr35eErJPHzJirGz(;5Wz#$V=8ooL){Z$j4leQVHLs>~v zf?BSrPqt_jzIE0T0j$%YIc5Ni zP9|J9>x!6`1Alm}Ww^9s0191gDGM3}^}#J&1=Wq9=3Fa+kl}SuV*l6Pj4H|= zA?h%NI3El4JE7H1y3#ptXaLYzKlY^tR!zP-Yh?7~8n^y%70V!YH%E^>A!X^T!LLq} zr1*7>|7S_`56y5}S)zxbg5eiEOy91+gRAp}t%%8<*59QZw9e_5)-NfmupTe@WVHRs zEFT>a5r(^({a`=&rGAmHr=eq$BY&(0l#kk~S7}rINz-4Gmg>T{p<<(^f;8|x)sKbl@ZKCu z#@lilEII~$%?mp((FYDR`siwpeJvUENwg}ym^$%)5JA;m zqo0S6kS*&+UAb}qSMAMg(CPktwMmt>;U;Y(q1VDhNdGDg-mZy9l z4fO4u^rp?4mrrBy>hTIHtMUfx)0F4TtmF6N7CgI_tt~BUp>%XIly5rRu}{;ohGQ32 zwA+;IkKnB9*yia*Nb4fX*-woNCkVY^=c*NT$_;lqUcxH)3j5SC&Gs|a z>Aim{2iy6A+Xi2+sTY^+S50D~qTxAxj&X(PQFKNYS#m7*Et8W99Hpc6O?{<7vRQPO zU$n;hGHOX4xG$ph{SVuLUzrUjqQ0{FEQbKPk5Vrsb(Y3ap+z$u>oxV6kugAbSai<%X5suiv; zfl6hg3_5Gr@|;FJSKsCJzH=O}zNUE>>Armc$PmQW=((DM(vWbSgw0utZE=6U>{6{m zSV+Oa?JSqhOHmH)u*ASMz&2`>8b&H73n#-2B?F@VYtvLF^)MXaB=ebPG0X*wqW!f^ zk0RxT(7p0y+j7mPw2N_i;CnFuI7Rz`?uMFG1~+SQ@C89rK%|?rnHCCoY|O@;cLF{QHSV(Z->@-n#x&f- zrKK)Y-lt=;CzOw@{ne3=>Ply8QB^(wJ=!R1b?;v=Ha39sgsv#bP{c`O4in4PocpI0 ze(m;%C2$wjlq;k_1$&q0zts<@pznyt*pZ5>!a9x24t?!m%oS$TOx_v9T*3+;J9;GaQoL_z>9$RIy_l8#e}i$`5i^EvyW42Wib zJyWirRP>+3Wc1Jsmt|zl>ad2^zF_DSGDJ@P-dQ;IL60wp!YwXuH7+ak?u-eQL`1Ah z{fMixy7qOsA*$hfa(8?q-D}>5Q#~+AM;L$^#y&yLINqk^{}+I)rUf?ZCyOgE&7n!@0nZK|I!@q5epPl zS4aQ64inhEXxHZk%onNQS*Og%%WsrBS^RdqoAkSaw(dS%6g+K4= z-giGQ&rE131i6fSsY+u$OsMC(m$efrGJZ zxK0h7bJeaNqMC`oXit2$k#m&YZ!tQTXHt`FfdFG6DulVH^-jVUm^Ga|CN9$AWS+@DwfJxz=Z*TXrBR|MhvWNRj-pOExd`SSXc-k_Poz20!Cc5qjc z-smi?la&&_CL|O4980bzm&Ax!GiPf$(EWmPB_3AvaOpUs{IlZ2)#MmLWc|WbIw9OK z@*lN{gl-lFt<-EnWZ|3IF z?(IyIxYac}%H5il_HbR(fGA`@^nz`60hWR{{Ms{ja<(faLm2kdV=o zF$7*OFqlEkIxJUA^Eqsf2(O>iwU6*wHd@4ZUGy0~9N4>9G-?j4`_933Ay`wt!tKE@ zwbd%N2TVv1CBSGFYJ`d}+!Cn@z|#mPLc!Yu4_AsQx-+_qjKVZKjG;7tH{fIIMSlt3 zc|EWQW1G6y$;~G0lk;Et<2RlKJVBnXH6wi834HA}<;OE5Kgn{3IVwF|;~h4@VZ`EG zV?-<;R@pMWRvy4+XAetSb8K9M^)G1bZ0S_@?JT`x0?Cw=5PvQxORf$_%dX9(1X@s? z#rfMSj%qX$pRp+@B!+mU8l5H@()zX_sLZW=-RF;{QIfwu9>pTaiU^Y;XGrMr&xp$y z*&t-27#t&uHlk?0Z*B0b5#DV+c23uLJa=unO!{IC$^LBP=w^6`(2Tq4p2UvC8$Vsk zXv{P45U6->|BD_ns`pxyg=mqQ^u!%J$kp+dZaU8rXhG8seWu?EO-BGGNh*$`{)?Ge zPh$2f4Y6s!NTWuI97L7U|{YQT6X-wp8d5wvP zzmF0gAobEL?dODzcuEGrhLiV|qIXB^;tvoyP8(#l|a@TS- zyo-)b$}7d>MdedPBa2yHWs+*Kw^WPygqc(|mN{-W`nvm@M)DffTy|qEv=~;z^$AV@ z5D2)TpaUZ?@N3_aIq|QGt#L^CMQl^i*B%OHWPX1Bkqt6`J!V#%rz+Gt60qM9t;9n4 zDqay7U4Cw)#44iFp(PV(*kq^LwslEe*mB?#y>JV z;6f|K!tYL^42ApzYnF2!&_vVAnF*QZQeD8JUVKkNHHTiI+wrR(X`i2VA($*~Nnx>B zp{nr=*_Ui45Z?$5Q<_!-9%OcBI0P;1e*}T}Ffp0%ozlfH?vwXv^zm`FF!<~;E9SY3 zsx_~uiteEfLel1Efr~;Y`lCdVhIXQuTXGF-l9l#1tu(VSkb8dq z2R1<+f4_Lp4ZbCjjpzM(1hk4Ri%t!Rr^7_J^s{JY3IHG;JioWk*DPL%VNZgwOQhk+ zuyeBnA732Cw1$(&rZAy0{_#v|0SqZ=ySN<;NttAa>z(rJ$o>Ui5n7hhtV8}9XsS5uG z67G7?G6`-tt#;G!@zL%}?G@x7Zo-b*i;(hxi1LB89Gj_nLdYR^!B;i7S$Z@3G(6$4vf z1^J=aMr7#_P%^3=6+`2kur|f3Pc?BDgO)|yzF?ypjWt-PRUS5#Rm?YT`A*1QF?jDy z{5MO;!v>gS_jjZcRSQin>-*W&H3#pEa`xE``$CT=3B4;ja|sX3{)4 zJL(ak=Gdz9S}?_Hf1#hetX=QSjpE~zx|FVXr7eh#V z{g0|5N>t+DLjAH=_^wyIgZ#7`He?!HuuB%UlkP{g%m9`5-O+Up#tKS|WBq&W_9p+> zmAKHSfDG2`=|MiroHMWYhzJWhc_eqSSu$?PT z7ZQP@w$VGn-7~s3&)qeX6YtylM~?cC80UIus+b)1I{!cltw%b3R2ZKJn7R~b|IfSh zWb>A~k4U1djReUg3AxIW*UyUQ3K^8$Fq>cj_R7cMdUt5P)6-?$&|Na1@RlJACItQ1 z9gb(Cl&JUG)JNP0*LlB45NN9hoP)$P;AWXB# z=df6s8ha5!CE^aBmPQ&P9CLwrvwe@+!Waw{afvTw1crg;AKR096gjz4aE+ z+@&<&L2=Enl*iqGqFYUHnuWX_)kn@na3!#~`u zGl_7N5a2~OVm)2X*0b!`PUIKGb0p~U*Y;m8Rxp`H`z_GTjfI(F%*L^6Qhmri60x!D zQ`E!stZ}T{3EA+$lp_KfezcYnf(2V7C1oOrkP{z;6#UFOcg;-*m+y@RkU*j!NeJ8{ z!h)kjVhrY`32Wo8)gtDp{>8g!{1l9o1q+W$pkO>2cld?CQ(q$)dvj8xJ2jRbA3lqUKj3R@Vp_&e-}T%0YlAeJjc0nF zi~6rU@;A^@rgwJ2p}`OScC>(|8veyqu)$6i7gyRQ<+$S8<-v1i!#8QugkzXE5}uVK z(Su|PDk03j(ZJ0|$V)h$%jgwX!1Y?p*wXkB4@0E6WQKoZfKqSH^W<-k z$ML!b(Ouup(G>A7cYbH#zk)lBb;=mEpWb$%(oXRfICO32kECXN#yuc6Hg`<){p6=gQU}(sBnVP7DnCGv{vV%u<}5hQO3u}AP}ne0}%L|PjG*QUKlmeW@vq5 zIwpnFlfmrm{s9cbh5zmOS|5y&L=~Rs+Bsk|fh4#gR!J90`168-S!Z5<@WkYeyBLCDcrRZ2Vt2hYt@$}4i4&@uJ z&oL{gdHBdd@SeCzJgku8gDdeI2UGD%$y7*bzK)w82=mS=*?l5gtF~2|o&x82J&W<= z=2N_RaJZyaRlfJ*T%j$*jMBwDeNV67lFqMkf`4HL7sR-E+w0+%zXha8WwIg`l~39U z@18CwHB>H#9N~(SthDyG6Smwl*nNAj!zl8ka+bDbk65u^i zItr}^0fDLFKOxw$WZkxZI9sdc_BQ?adJ|40BvGsHV~Bv0bbn&kyEg9iMqcs{FRRMR~;}y;-O)MnkeK0_a@^5hF`$%-eErD_u zaxBsRkO2JHQImJeE;#KX1~6M1B_5vC!+XW~VBG&d;BY=f+c3ahpT8g=5SsA6V7)xt z^PfD7=#<+gn{?WFM79z6yLm$@UU; zpd~~UNqm>ondID(IaBi>Xu#pl5VZvAz;|8$8QamxL5Kjy-Kf_`5KoMBn(r>`jA7lC zEQiJO@)t&gC;r&OoIk6J?lM~c!b6g=C>ou?t|!x(&fhw{qX*E@3Y%b7UuCFBP1*bldhUTJkFXF=Ll3zG zdP}2#evBvO6Yv;tvj+LiCfu~Cp8{gVJKw8`<%i=WK6)glHv0r58<5759n`1aPk4?- z5X27IeL2uj`sGmf@Oo4HqSkDl_N^v+I?Hs~?5Q;x+wZ%4IVa-A!Lut2tJ>Cf$|hLa z1uG3f7@1eB^>Te~z*v?|3W8avyUvL^;chK$)v^KDOWbO)bFx$zc=BGl>JXfNcpV5l zo2*z;Z^Z;DRc*k|km){nm3v*3tP6r?e_WU7HJW{J$VH@u)oDMMnP~U`qcih<)5B4}nbVVS#Wx1V4DqTZS zktwji&SUpRGVI+vJES9EeqdBawIc|C!6I*5nG0`4I~It8p#Ox%Thf=5B**04;_DpP zPl#$0dFxZ3QS0yuQEi9qs{$1cVSoqXMu=}5*DK(22$Z`y6{dZ+ZMLWA4Jf*YX|~s= zN5u&Aw`{i>?Z$qwM5!ew*-tizJSG{BD4K8+We8hQbOGhbxn|*WEFqCmi$$cMnyz9} zke%IG5CI{2`IViNXSu

B7xj>=kH*}nl`6f3SPfe;pY9bJ7~ zb@!>^R?DqY(1<3(6LuPs%#(jdUZZKas&KO3-gUCsC6!nfNFl;>?IJyBdGo!U&XIzyJ+y${$;QZC8Li z$EEZzk-pd#(UdOi0RRtc@r8_EOLt>N2WT6W#q)Iiau^gfz`wT>f@qv!H_>TR-Sf)b zW3btAWlPd*F3%dMa`-sD&6p0*wV%zHWSNbc-%O+!Ms{^v#onL7lHxROLIcLsVdd3* zmcFN50h*ev5xV_L&YUZ4p~J8pJyslQLt1)w@Da99$3T!c2@qesw@P7AeawI4|I?d@%cC&3@lyVdeV{B(^}M*J zgnX3>+?W=oKaSHqS7=~l_IXr$W*90<1x{w2b~PF3M0A))#^p&4uzynd zo{7rr(c9bfX`S#qz1l(&ycZHoA$eH@HcrxBd&tb5;N~(7)s>AE(a3%zYrx^J1yH}a zic`M*Br|0X2EYlfg%9O>u$BZ?JA!jEa+DJQRI+F z2gAF7=vLZy9m?{^Igt>V52?xyvouAsUA}|63SKb$+Um0DGdVEPT^GlN+u|V`ZGZ5? z0A=u`w?QUS;ZWQMPi6*YVPSc`pOfK!+^J~V?DYeM;z|N&Ogjd+30!=o12wO&jZ(hE zN?Z*ktf7O0r0H@!7oggpQ-z3913DBTT=bg^JZY9c327V?UZ+~&>Mn8eOgmm1IjbE{ z5XPsTao@uJ+Me$8;5b zlRT~33Z8`QIoMEto?&thorGa|`>*9S#xLqW` z-^h`H?H=(~ac^aA72ggr3f6QdcJAS1uGBg50nah6txr81P#r3k>{A}ROaJ<48c?#i z4+puVAJHLx^mxx^P;G8JqQ~RA<(DV3iu5qT9=gsy%52QTXPrg{p;wc5FQx5UzeQ{e z)rHsZhkGVkuL1VV}|Wp=sd@57(r+cqG(eXA%UBDxUO=v|YF zfx#mkQt)0+MNjshU3yP3$%>r9QLL^^WYi@U>_t2cN*BXzR@WDt=z6)Q5)3WSrsb-U zrm^V7LycC{nyfG>wZe~&O3L8&Csb1Mp1;mpoV6}4Ud&8#Ti8Q8Wz?_D>`?Pe_Inc8 zaO$ihY>`k`{VrOf^=9+*ix^u#@TenWJF(U*0+a3jtFC5CK@ih$P$@B@X-)Nw!kQKW zNvfwg(U;PW{O=8n|6fn2SrLA@-HqvRJk>v0+g*&|rDh1X?_1V6aWCA8j9!pX;2S(D zVejGCcFHgs8%@1TgT79!AyvR-_ElyxNwBZ?;DEO~cws$g@O1?ZCVd><9zi>;kG(Jw z85fL6X!lW?h@-QC;UJTDFM`MQdY(89K4K`{0KHaK$bR<>3?;pJ6pe%7$?&PI zv-)0>()(Z}$wd+9oU~?NG37w31){F5SOCMe@!RP%A$|RBI1cx;@!e6~(7L3A%sRoH zG#wUiWs6T+kA(cJK)@~-JadMu%93o+dZ3ObyKGNkZ%ArO!b(-pgc;`>t8oF*ukjs3 zDa2cAvQzt$Zu)l3I^Pb?YSCMi*G`Wi$7I-> zp!N4R9a3*l06%lN1Y)Rl>}%>YqS+L22#O8gz#Y+zbfc>vhWdCruROVLlly>p=5ulr z`9g&vLtvwpYneybFG7xpR^?BUz34Ulhw%o?L}+Nocc5)$JvQT0eee8sl5O65RtDxW zzIoniPp>30tPJ6WO$(U&RHJX=XCr_t!3jM)mco+adsg7j!;EN`5>l2vd21h@%wea$ zh>aHP0gQ`w9ION~sch15h9-$_1i~XLT2?ZZGT9aKmi9_X&1aI5_OxhG7SIy5o*9xx zPDHtc0bzw+>Rc&>0@_8f5F*h=n*=MdLtlw!MMRWJfA{1dhbb(&`Fl7G*hpob9UCBI zUOD_-g00+{qP)vkhJ!W~}cym8L8I2$0;lo~r5(o_Xg6ic1E|ek_`w6KQK-Z9cflsq>2K zsD&=(`ZimcBMOf$@e&hVtv)ogJWQI51g%e!8~B}*$UI<@iq-$?WS09X7wc)NSR=TDV(hpeou&%s#oUfWe?U&E4i z_!!+)2>;EU`lYw0&3FMB7G3;MqupJaT{Hflh#-?n_CU8gWR5UHL%Mg`R9sA6qk+o? zV?)7v{zZRk$?^kHDBL%s*me*x8ssY1&HKw~1ga>HEhwvf__1W%il&B)zMiQ`eBQl? z%-@aUI-#1#?kYc&g&1U<%d$8^1(ZB*G)E79p^53U&R<&e)DB7OEkuuprAqv?ZM(%6 zu9f`dT}Bk`DSYDUYJNqve69ywsGlfUsc{8{*qJg95Jhy->u3S6d z0J;>{x^dJ$PC_N4Hy<}vXd*V>rJh6=@XMNrKab5|kE?dF;x+od4>%2NECCn>7u$-A z@x|ifS3FZdC`^|OS7?|w7(dmn7wlR#QC#jm#g*K^TVCF<-cZv8_8ofjBQmSJ>TW4p zEi1K?$&D6TwY}^l9IZLXzt!JAKi>BGAx5yG;u{PGT`#~!#r*g5Y(wBhADN}QypfBO z-jyM%F35@S)Xq-rBss318m0gQWepcxp(gf3vnIi>)K1GPv?}HTZX+(YzI!GQOV2?e zOCLt9aR#OK`=#6>jM;?NTQ%4N;h|?W0yP8L`*u|;4`Qt%E7GXgUUyrr=GqM=CqJfy zx{46V+>_rbqovjT8v(P|j~ZZ>IkIsB!eodNH_+u0>0xfjTS`cn{zIW(gBcendvtYC zt0Ne#YUp^zHnUhAmo)i!KPvkBWeYRj`EMKhMGd@s<-5lhl{GoNh{a0D1o3IFEUJ#n zo=t&U1AiN;Z0u;~WS@fL_vG8Q-{;dSI%6#6qD}$Nwn0ncn?f%xtH(5+X`yc}|0x*L zPri<(@OHz2;5>BX-RVbak!IcaX0XnzBk2DiOHSDPI;rFjKUE0{O8KIG3A0X95~>9o znu=O=NN@(sBQT7(8|G(L7cUSU-2~f?*$~nbd;A9Wb_M;J7b1B2TS$);MbA%i8A+Wc zC1AlJo#Ni=t<1iw%uAw|EN5;OElwQkwRrD<{6y|QT5$mZgiXeZzrYJf=XB!slf@fLAqcGjn}k1R){lZ{1X zw<+KSVuup`gWGW=@sazNTq6Vple}gaj9YlW?5I>MHrAr>pWvji@oeHoc-GCQY}4=NCp-P6ah&5}RbhQU1^ zTZ*;=2c~PrN5puH@KRfxR$ZQhW*oG7T}yc7&z60@RA_?vCLv9f!75{R$mFX^FD;cr zAmbe9Thdx5Mts=Okkr026IAADI~UD5(uO<7YQ106xbg(<`HQV?Fl$H!-Vrv=h6-29j%8^;6t-i&v3)dr;?+^PYW(i#9A%&*z8|7&-V2ypRT`>5)^!ux=+ z#zY9?+ga-V3WlRM5ii_@X+%2po*|rEZgG4Zu6$7a7zBU5y<>#-*w;~QG9Xw~8!!2A zXZu9OD7#%IrCa%&E-pYaqtlo-Vn#>F9FbYicO|{muF4zq)pEEfsZqqQr9GDNaU>oW z^RtD7@*-uU6&u2ZOaeH#g7Ev{jG*O(1bc7rd)(x#m7SAT6>vj7CAC?z`{gB;$r-;Q z8tFg_hz)R1Yj(mKL$O1-*X;YhS^0s32T6;=IlfKeV(S zk_AX&UF5WQEt!vxdf@ytn61{Y>Ark>QMfol8UK|A0H;dgc>nRYf@@?H_y8zJzybwT zWD@?a0pcWND1DfkbXRWwTQMk!C@+(VQGeBbxrib0=wr*snkDCT+Rfs_4TZd%)8>;D9$9sN%^zJls50tO3hUloULWFYc8G;}ijqYI@MDBBn?L|n} zt?+br#=;YG{ZWwMHk7xRqi!0@$<=v~FgBoEM#mRCNGEXk`iH5Xm%rG2P-rKav)5t> z!`?-e5z8e&=k+m0Y~2|BQ0h-uY^QW>uUw03yT4O*S6eC$TZ1@Sd&6*K1Ja{?BhFxN z3Vkrg`+v*#{}-Llf4;RlA#aJxJ0E_=_xWY%N7nemb@t-B;|?gO@RUi)e>efa$m}76iV>;FR7nW13uh3@{Vb2M90mxzz03gI*i*`cJFQy=-v+7_^|fkXnSMnFTkE-r|8?=!E|)( zQ+KWvNAsn#Z+o98dkyhbng{)>_2|~TxKOg*I*j!>Jk?81_S#V`wWKvg;0OvA0xa}B zZ1y#{e|35WEQv`yB0}t7@Gk)M4EhK652%FdZY^}I1+JI`O1#y#H+{{<)WZO{*FgXne>lj?235`no1G4oAI4o9jynxjnr? zDC6wwssY%$^N?adTdD?`v(3P5Lj>3QS138O9YETp%nXP8?dv=fPE%3o^^4cqA10_d zr!h9I@ISQxa`N;7Tb4N#r}0+e;G5z;p*8eUb$br;6vwh3(E=)(cPoFjQK$yRwN>l% zR1DasTJ=>-i!rUMSD>?r3B@BLA*?bi+5*3gx=PB5N`dTHV=Q6*RXTNYDhF@M2aWO~VJ^+f(ZhSa;UP%R1AWg_qO_#{y4$rg2t zjP0));~R&O>Dg*QZfsxcjTz+>Xa+^jIgCeo^@<-5p)Ft*o(FFn@LYrLu4Y#;P#h>U z)b7T#ZdXu4wY~I;CaaSY!$esn>d(I``IyXkW4&`B42PG6IMOf3(|K@545x_$K2@ES z;>tbTcP>q4k1s-i(d&x!KKs@8`%nAvejNr$r69#pV=S*HL?tQ~h{yoKmhWmQN9AL8d;3qhH)S+07dBcF^@YHc#1`q{zj+PZ!VT8jx?_WxnTQs*G zz~yvKt5gjz0_CCW+m! zVSp~xQ*~O?8&AjnW8P@6w|lhjU{uvmXveu&{*}^RA=PaFF;@{Z{Mb@J^l}DPI<~ZT z69@7<6S1aFb`LXBvZ9s>g<`U|w71k7C@)v35~CAM>ZO zvk=};D~f|@7c_vaq92(9A?fg2xr<9-Ja0@4ls&9)JCHLgqL@8|E5?}DfA*~`7C!<3 z5++?4pG)agdWP4r*mYB?Fg0(Fn?NgBCDJt$4Cdisk!?A^749nb*F9&j3JC}ScFGSV zU0_Kc2znsWZjk@|!8bhlhug45+ZLQfn^h0~xVsUEap|qA6lU1uM_fxbB_^Lju>PGF zT|MiUPh%9*_5DK3ULv3CXK1AxXdlamjmON(IGm|vVg{wFX_ZfQGuK~r^Q%WEnV9nL zsCLP%lFIh2ukSiKcq%9Jo>xrR823IzWbcmX*%ZJUGxymap5pkmE<|s6zJAs2e1E7p z&Dip_Y*XiHOE4mIkH602fX-d=N zh@!lp8Mtm7-oCnl{~V{S9xUR&u~3XtG2quVZQp*un^P!tDw8EpI_=Hl7~Y=mZX12ZU(TBYXX=b2{aNl+ zJ_k^G>BfBa8j0aHk+|A?gsm99K&EfKZIreUsO8C&aN%R)&X9tB#xjwP^?p;vA%|bD zh>|Mt^Ff3R62XiOD6+Gb>*fx&AT9^6H@HS+^M-#IFye$Z)1<*zb5)g7K?NepQ7}9P z1VxSI#DTP=S&SiGW8=xw8Pelc=Ng|#kbmNcgih`O=u{F!bdD#*ec0k)|>s=3gWIHZYGkd&I_F+M|Q5R4&IqTVj_o0}VQs{A{`AklIUR`E@;y|!nxSm7-p1L|L;5gAGiNF zv>X39CLVZU@97dMCRf{Mbw~cGWlG=_-!=AgBYi6qoaMD5jPDw$-R+(<Hd)*&KW|&@o_feZ~D};tTMTWuT9qA_u80V$^;Y_xZtr z&85ag?(Fn0kIKzy?ez0*l1RxVn{P+U+0A}iJ>v;W_mK^5Y%)TUJsx}cu>n;@y-Mhb zdN!Fgz!rq23J{U8{SJOd`YtkJz7}`UjBg^j%yLhGev8u+SACFz?7|}Z#6p8p&f`Kh z!yo;yP-LvQT?@r~S~F0eurr#pK|Vsb7f;Ittc0QJzg%;OGW*RzvmKeLzR35!!J{`Y1Ecm>}-Vx_Xq7~cW|jn68b)i9s} zWFA6m)I6t=3i~{_+1?SCx4#X|eu0u>VWY*;)Gt;pbuTBE4Q%epZ(EJ3rmZ!Ai=Uks zD)^OIaglWfF?W=$9E?bKg%P%qL)%$iIU+97aL|42U>-UaE6+MQWlMmSHo-Bz01V&2 z0g3W1u0VhY!!!wFz%!EvYT=Ei_x{HL_+-mcv65n47F{l-MjOuBIo}cAug%Td7nte2 z?Ic3Muo_b2cuv(GU76_y^C8rj3rf(R=AFztiDtDzG9D19LtvpK0ns_jmI=dsRDe|~ z)9Rx8z9{A3yp2Sy$u6EMU?m|$2aT;lz0(9Z1ba8>&CWS!`Jf_I4fSskDotq z`;pg^^}U=nJR#Sy{O=GRoylBPrG6J&C4_W08+<0@_pOmh&dh9?3ETp1-D_KL+F>6F z{_qMXngPV9V>M#S8!ebUw6}3zquOIuR@Q>x7&`8*uW@VIZnA1;tobc9p~^K&)4Xuc zab8iEniXnv!QV2}n4oitjG}iBC!={C4H#sgDac%09ihegSS8nAl7uQ9eLY+v!2e$D_ZlG zv&;P6!3y<;lRpKC^W+@t+3D*uHT7_~h;^sjjDCJAd%d=dQ<=AHbT{=sqbFY=A>Elg zp*$Wa508yS3pCaHT}gHiwCnOcqcN@e zw8854#|LYHak84w@$g)Ngg-Zdii~|l+#hSNpRIOxWh5V(bY~@q4sPe>p2OF}4RT>< z7`JXM(w)~4?$ckrR3(dljsZZYAwZonm$e`7tr!wNH9g5r%>r#qO?o1|3UYxEru%YC z*At?zE$s);cu}Pzt~1U4==Ac;^Mc*#5>1b{49G@3%?`i#5p_U3Hj}=Q?0Lq zqCfSf4tb%P{fTdk{NteQD^#uL=aEoztJ6tJb3JBkU*VJ5f=M6X9OPXQEowQn<`;}( zMP18)sg|uq!gp9klU~w6c~002F81(G&pBmy0=Xtrv%R3oxBhBi)4OxdPBOJ$8P zgV!^cUB9l1IMAG&pCTUJP0inYCqySDorVhqJ{5Lg?L3D6?T^jBAza@gg*-e6yq;Fe zC64uu2Ha>>E&hA@;9Ch2kSsYUITk=$1=dhIlXWo9Fl|BhhsCY3c%sm7$?+B|f0~gw ztC^mLvr*^o*D}ROct^?jkLP2q7Cib?e6ypH7z8k6A~q%jvx5dz9*LtAAQ%l5UCn^K z4;=RN8tBqqyg1ppddzsCHCY&%34%w)ed|wXcEkGO&K<+jFZ5Jl3Y_6@A^Ir-R@uzH zh)gBJ4|0&LSbaQ{wLgqfL6Tf8guv+vq!K58W$tQ(OCtxrb~M5%cNkZkSU0B{PLd|$ zXH(zLmqAnMCT%`YnLnq<2{~8Ts|T-JvPAy5?)25a1SOM_@vv3RYgqpkV|9a)=5z|U zPyRI(`Q1rYA@UN%l7c&3ypU0sMg8x&mTeVBO zInc8Tx(_N7zIHhU9OmHbcVVho*>o4tbj5qMQu?|X{07fSwTx==35k~=77Vw+Ef$r$ zijR%1&?o0E1CEHTbyAuISaEOHd}mz#jdhhQU#C>uw3WtUJ7n9%k_yW7KiuW_8A6{1 zG1g4x&3Q05cunrsRUc~dcyjpA82J&mS-j|&)~)LJjn1s_*S1-7?HxKIR26amkqNCF z_`hVSv@*blkIFad>iIL>4kd8)8=4hVZh>dkysejwP}0=N^}cedwOpgLYb)6GO{@aV z>%3=Yo)Vnu8l3nc*Vw1vRn`?oD`4K$VH>Y4h>oGk70S64@rl_O$yI$X8bTiz> z9-fW{|60(gl{mFLlIi_|X?!YRa}!54=9ZT5dliWp;>Kq=Vd-C_99v-BO#~U})5dhJ zrxq!h$b$3o+G#q@m-7HR>1^rk zzerz{Gj}o!lmjuK#3?5#3=l`7_il*y_=4bz<|HenMBXci?hj=T;i&7MM3+tUK)4wF zNbo>_H0Wrvo8DX%H>gYpSV|89s{W{&dNeff!C#yR7|g7|-!1Z^E1{;PtD9|G?Asm4 z^F&@5V{Y*u+j%?{c_pK;i`nK|TLAiR3P|8$|1SjrUths(mosBnVkgz)J4xN?Xxb{0 z#>|Z*PO^P+$Uj5PQAn*t9Uh)3=`IHWZh?FYtxbB%`ixVFhyJ3aQ0_Nyog{QLL2G7{NJoud3!Xmn#E| z6hKNEZOSKm8c#J7)Mf+;t7?B?xF@o)F`}JwZdR^sG@F4I$w;7ZubhNC=h$z8Busv$ z4!3(e4W$&`qMB6d1g{Ma^&so`H8w!NTNkICi4WYNWUKgbc5e6>f`SwZk)r4i%ogX~ z$vHuz1k<(^Yf-vHFmT$c<%ky_kw|a8d8U+b)qOM__ml}afPw^Wq zlJ@Uo$|XIsz4OtpLN$Xj4V%l*kG>UDhAL~ts2G@T1;XsMNcN822FKta!WgKbbdslm z{*{LgcJac$IL7M*Nq$POzcK8@O=oJQjW1iY`sSU_En|l@r!gy{$P5Hsqp$ptn`P!H zMx;VL9;hig-n3-gZlXw-D(RnAMvD%chgLdUSHpRw@%+&uz~N1rz{HEjjFOj)43j?k$?t0BTm)|bw9I_eIq%P-8;$}YJbDO zyX${-v|uAu82}veykr$pH_J1P%9oth^Bd;9VQBI(fBi7ClLF&U-6krfdHyd2D_RiW zZ0W`>9XA5UZI0CL7sBiOJ0*mrzJI#O{|vE~f1PV4hE!vO^<=@icWB=$lJ1j_MfKIM zS1gWKaH{A@E<=~~WW}gJ59;>^JR5lWgBHt_PS^%0BE4i0ab`;I#78}52$H1+f^JbJ zPH26?rhb}{(}BqvFYBG^-k}} zqD6carO;4&gpRmLiFUtzBmoth+Lio%0ngiS5t4D~Za^Ia{pc7ADL}+%KuYN8+0h`Y zbJqx|e8C(yvxmN-3uhS8WUO71V$MU6frS*68OIe%Aw*p)Ek39aJnm>+kdskWJ1)4Q zONS_SGt16|T1?pgA6uYY(&+yU#MNi%SKdmW-LccfSk_QZ8lkP#OS zcNXA)=EG?$7-pB=xYFEoxk~7M@6UtXbZRY9fBakpjdy9DIF^2M0v^X_@-7fB)F;Hw zw@#cmpVKO9?M+G(Z&7>IaN7RU>1&^@AhqTQE8DmJ9{-^eTAuMl!AAMwiTFuqDj|eOIg> z^ELPka;=A0U`sYDnjDXgB5C*oze?0O?{ak1^Iii&;Ie zE6o(O)nF!*89iL!RT{X&2T3|yBqNNSVN#J{H!wmCUVe$SOzk=ouNXqVBnnBz-5%WZ zJJN-4;PQ#Z{{D7Y#hckUs4Be|Dm!Y-7~S}-(1wwiKDDjuluL}(;(Do+-V=unp`{Mt zjK>pl8Ya#Kkg?gUT!>|u(wnzNMPI$AN{NMw*Ogg3YTr^(L}}Wr=dx+jRi$vHKn8YW zu~bXfePAr2{z+BT@(e#Il^mn!SHzTF<8DQwZa!v|KMBm$R76!rYXk_JjsRV`2+cj6 zt18Uw*_?eGtl6C{nf1MsxnLm(NsdQ05Y#HDhpvl#t+W@>TLLKQS3U>#nzC!+1q_LV z4PkArkt8;H1U~J1p>ZEzQb<3%M7~f87E;$?qLT(u^Z8SL=r1>9eu*Z`V;9)ThBM8} zgzpyG8M^KXn3XH=PGls}u!Ih-TVtN}rf|+R!UZy(ucsVUcWjiY>}gFD^~Cspk9DRg zjytWYKpxV!5=u&_cy4ml;l`6Kr-y}e&fv4Uz`(Agh<(dOXzR2_8e_J0|E_LRVDHC{ zOY@OhRk%f_s@7LZkDobz)N@q@m6RhwKN@yOIM?fL8jiSRb2>rY#q5Jmzi<&y4&(iw z=lQ=5i{v&Jl5Tzn4~Um#S8S4RE-(7b@%5tzp7LH}$zjD9J&l4Zx?%5#^udN_ddi^G z%XKDt1DYK>M4 z3EePigjYbjU4n`(;Q#I8l|Nn(NI8h2c0ccqCljWpom3JGcP-KI4)%q?xZc&fihA_GKfFDr!ch$NQzC3o-gIJ!Lg<^%%RLf#fVjkUKkj^ZLj`V+HCs`x zsCliWfxXO>>Av8t0#QLow}^!jv^F842ozxgu$lNOw5E7MT*M~+cH5yId)wr8O-oE` zyrX@?aEW&?y%X5<+@tuHH6aYNH4%9?1jGaq?EtT|oBDYWxALZ|4KTC*Sr5V+P2+%-gElUo_6IC=fBI#^J_kftV zpsp_lAm3XGh!kunbv_m7!z=1~tKeK&M*4{H6@Cf1v*BTBNT7J6Yv-1Em2e#<`N_JH zuRkpIhnpl=n8JDOjW7_oJoYe`^HO)yI!1qm3SsXATNbVrQOB|V8Ex8{!PC2J3tE78 zTJTBMo~4eC`!7XWg$=Yu{s9xQ)3?D$%>-h zx|k%*zOenqE%Eodu>D$^tX2Z45xInL)Q&!YO|;5^apD4to* zGT^b==*@n0p6Uv(CzX}`CAk4m^s9&8Tax&q#L*+&W#b1*;F6 zR(ZQ&*+c2ZRkh7!M?39lE|*pC%oa}k3Hcdp(QBPli;iWh*!iiPZ0DkbVDs^EMIrY4 z^P;K4wR*3gEk}kPHw@Vj{rU!o@rT7{mF2S-nA=|Ast4%?1%_f8rL4r&=XWPDUD0e+ zF`~|i~!}M9je+hdIW^9(4aruvxunPO;VwF%R#CE9WB3KOqvI7p}9`4h9f7!$y3Lf2@sM=L9T}BKU(gB?e z((Cp2OAe+YX5zm$D2ws8jvT2Ko}N5j_#_aWlMC=gk+C8o*j(~A4G0TS_VDmNa9hcF zJ#MO&3U*_KTq%rN@8S8&1V(2Qmu0c}qU|2H%SfTz1g5vgXs`;}#ZX*n1n2;f(}d=m z_OG$Zk*18FWt*R%)UVx_Q?dEI($V0i-o#oPr%cw5BA`M5i@M*3T*30$jx5IswrV>35CyDyT(0jnAsrB|w^Ja-N$Q|+l2 ziZSHeuI^xuiAx4t+^&=w#(%W=>9Mnt%j&na{&QEp{qKF$pAJ2S!J%OZJr1f}8WGAy zxzU=scI!b<;R({dQebq+75YWGZtAHDQ5HykLIgaT53pDBpI^2~jMf9^8 z-LP!W0$ah&i%L|>h~YG=-jd3tk2Ig`5berQ@-bb4F2wV8Pm0(LtKhz^r5nr^Br+p` zlC)M~pGxkx%qD1ZQa2QJ%A?G-;*Ob5t;(hPzA6On6%09lu{3V7DDw5N+TJ<6n1Jd(`8&l)UAiP%zUPCD`Y| z+_dBP3Vperz2#NlEUQrO3^xE0$&aJE73n1zT|^Wr%_=~|Jgv|Kg`bP{;d)2)@e6SC z?r>O64X!0nU4w_7b7_435Nw;GBBW&|f)?_0&Xz>gXOPg+;hY%3Dr($r@(_%p6|ry8 zISRh-yT)j0^}GC9Z*zYi?`yppa{NroQ#SmN2j;RkoxG8Jt!|f*Gg<#0#3{%3hL-AM znJAucHwElyZO4t*DlY1L!`BTM*I3k$1CV{8GBiK z4_Ix|5u>r>AVI%a(Ykm7Y-j@ghcWgIkNh*0^*HRK=QWHEjrdtq)K^`D%;3*864HnF z`Pq4m&Wv*A{CycGeO3r?(NE8)1_AESz3)z(7q0b-R{b8gnA_}%!vY5sSJSy!#GGJd zjpEu%<4%R>%%<@C{QUhF=P0)@-_qj_3z|ymzo22__(BUCU<>a|4B#;~OAjDhX|$U% zQFm;AT|B5Zr5MKcMi>vSiTVt=vf^a($>;xCiZ%scth(S-&B>k98OJ1eC}_fPK#kE8ugtshdgpG24z zrxJg|K#C1La-1eJ_?wQ4dAgEk2A7@`#smk7PHQ&7>UZULvNbt!I?oF5t^LNJjM6bW zZ{go$`BknA1|$uUT|TxX2H)K6}xNb`kg&d+#;VQ2g+{zZqU<*&g6+X7?JvK zvr&ozTJ$~TbsnB@D7~^R%M}umZdxpHgjwscCvS$$=Tm6l-i$0(nvOmxVA zp(i4!m1CZbwXJ=zK1t=uH4tCzarI#k;8>ws9wQ&0)l+fHiDe*Qog`79H?-6D+x0r< z^okyqZ3b^Zg%=R7UTd{w53CcO>`i+9QgjAqF0q$Hb+rQCDxDaK zoo@{x=zX_%q-3BY6?^kP1}SfMVbv6-k-{ivcIrM^Ab8?1rjtCT@?@66I8!uUz*!uM!Sl7P)qeN? zG4_qok#1Ypo$i>Oj;)T}v2EK)$F^bil(&b(_-l*qtVv=9v0?>2gs&FH3iLJgD0EHCIZ?1==^xPB#B@~uxIs2~ zKmh>9guNmQ4?fJ^pNOtj$}~|T z+9He6rex)>OgmPMTGe;dv#Pytdg20|OA8sbl({dUkgcp)2p63RzuAxWF#-{Ye(mtV-8IMKh6Zpfmo=<|OwOB!&Z*Q=vlPS20$;-0EuM8BGXH8sr-gA;NQcEMq(iQSA{PyTu0KGDuuv`}8TKT^MtTXf-wFKiy_R!7 zfK{IS#p0G5<--Lm+=-oLQQKT$!Hb)4zRR_){FQkD`}lTblx|&?cr&A`6^jR+u zfU|v}Nn`Hjgth3|@cMgj#-k|R7YjSH{}Gfx-D8P})4__Pt*iz4??V1JsKMvR>|xrL z<6kw_#>77k#`)(GWI@4Pa_f!*5=J%5Cr2N_xf`{r!c5OO!s)Gc1S9Znu?x+_sfKf6sJ^{pM`HG*9^M5bp&@7I?*6Pc-JCMTuUov&gpt9Gq7M zP^qk73fHHFon`|I2e(W^Z*6sWj-yI5KYCzxi3$!(8$_dJyj~FSZelg9-0LSZ0V$zE zT;XW%I-k7G4fHG6uL9NY1F^9Qtun*cJw>jk(2}FfVf3-C(ZkxnegPM$n4~Tylw!gb z-l5rf{L3_Q=g?HRmeSjtFek_9DYmyaRC|ZlE2zxilZ>G%sAS%;23=>-N;*Z|WWYtj z49$&|;t-nM_F{yrUSVbVO~pvz7zx(fT`7g~lBLMC)-Ijxj?0)@*Cat@o50B^fcPfs z*?Jo-CkyZMHs8Z+-0n`z(PZ3q(dykP9>0Ms-2sYH7;T`Rq$J+>#&^m+>Gh4PHf0Ls zhGkpavO%7GuQe?Rw#fgUI^E)cAZT)JAP{|dr4xR_IuFiSwd%1)>FVnsDol&QwgJHQ z%=qq(llFC8YHF5ycAV#s{T!McBpNcNtGWd^T$y-bl8O$5-s8109=9or-Cr-jpV|-5 zq>}aMU3@jfwzeF6ps|90+y{4uJaFd`bxmL07B?oae%P*GyT~&15Md9Ea5E8jxU;4B zt0cJ**%IMW7GhT2&?A}tA-EuFYek>FFcZhIV8R7W zSthF%r=R=~FN2@}kM|vUe(DETX{GGR;*9NTO^1k^#Rv#yq;XmUUpb;FWY z8(5$iz`S0ujP(Kmu-d#wOu;6oDJ+DDnwk*Up7DKOc8w&M2jV!;4q$Yr88Lq(COSqz z#iy&>PT|e`W?~}yv2he1wjby>z0C#wP+Y8-_= z$L&LqM!&w0+QGwFJu49a0DzBMDlkY$g$heRHt>WHOgb(YF78E2aI?f%9j@6*O31z? zC;a+BNhVUcJML+o2hT>lPc|&~pu060w7!6C+%>Dv ztA?xtXjnYq-u1nXOY5@}a_NNrHEe(+T;NNEQb4A1t*Zzf@p;?!4Ej0JvIfd zMNEe5S~t4$I31cL;!X~>Jztwx0KP1ljJ<=aA=od)#3Es;U^@R+mgLcqjJqN<&%2fK zX-DPTpstsi!;10)zkigmHs61;M;Xm`JSxw+^$Crtv;weREW^ohtRJzX_-K&#Xyc$k`R`dYFfg4R1% zHe^0nfAennK@*FK zH^7W;iWh5zMSGNl^JO`$o!f23{W{tuQe$O`hiemkA9OE(R!V=^9!A#%7 zAaut(sAf^aJ4?l4)|D3g0V_-trg5~A=2pIXHw$RrDS@8K*K}!1_9J)3RJK49PIPw^ z;U-ysKsv}(TE!Al>v>I&49~8M`ob2+@{d*i`8gTGH!KW#&HMfJ_S>kSEhYAYdrUP{ z+rn<3c~8O8?z?%Ses|VW3r6^_d(aK4z>y)1wq%;g&UQcgJ2+o zVl#rYI-j%pe9*a#`#0>0&g~3PRTy69ASQy1fL3?iMT>VUDJ(drpZpfL2DPa1bW&tr9PRjdvD38_ve_B?;8PyvpR8rjW-Ix%w!Aa zw8c&b2-wL3ec{K=n*H5(y=g}>asQ3}m-Q%wEjc zsdje~5rx0dC?XY+)#q%LO{FekSIkSE_b~LX0|Dd8_&xyXUn$O>_=?CGtPkz{_elik zn7CR*o~V+kCjqx=y06*?-;UoA~Kxb|FhR>ZZW<^3tMmt^}mnOW8oWr?3A00CD! z=-NQhe?jIt&B0qEAye_!mvj|cz5&1aqa_uBt`^Ggi3IPCt;?b;7&?h?V*yKfIMk|f zaX!cZLKW&RE`&_@&SIS83VH&zw03Cdz&>6HH*AkD3Kq`YiJ)zj$yT@%G0v@7?+Ytj z5w%GC5UVQRi8PwG#jQut3Ik@M`rZ83xvM`2o?`TH<1=5tfeuec1@Wqh5G=%w<2tai z?-_DCiIp|&wt{7#ipffmY#$hD9BRCAAwCh&m*>(%!nGaCT&xYRn ziS_f~l6=s2qY18VO}O}($zEaXhrL9T z&3r{S<4F6H3iYrtG{>_zun6{qw68l~dlX{zTJ?C0DWd$lia&1M)c#TTzK<6&sI+() zg*(eD5zAdYXFetzQVsDjyc?L~6QGx>QcYG01PanVDi@wRIb3(uxmihMGQ$(l*U_i1 zu#s@ebxxo~bQUcoBEp~DoyDDf?+Hn{7C;oxEOLwB`_+!jD}8sfGivzSui+@dKO=?} zW;o9_(>=G18DAn%6@g&6-oyQA%ATpoY)gn4dqM<4We+_Nerdf@^<=3iKZhMI-ujZqR zOI#-b0l^hIJNWc4n4Mv)Lp2dvx0VI7>C>tyb2-qE0#FXeDq;5UETyQ1IWL`M37JiR=_MNx?R-*Z__ zZBRQqyQ`!8=$#LnDn2c(%x09Rz{fWj+$o2apg1#N=bJ)>x5x?}7N!qn|GHEA(`a%n zr*l^VogMnX$mr39RtwZkQCDQX-Uz~@_DU>Z8l~v!wpF11CEjcKQoRLjdvD)7oi498xtZlfa_HcF1-l z)+fA_;$jbeE#v^^knJP-m;G2K)%$xyKo~$@lYuXUD=^wDt-#^=PxEpbQgMu_?3987>rEA;K@B?i{V*kF$s$5eyVvK-SZH{GR}HzD?M2Lfv1_s zz3Y+}50)seLtmdyJlEj5KwlY41Re)A7kYv@b!9B%@I&$r+ZOxStgMam&9z%46Ps&P z;nN8vNHn?0#3bLPoad%a8gT9!R4D0e50F|!_w_)IlaLEH#fC<{gfp? zR^p_+RzaK(>@>uH9i2zzJ#jO7Q$Rj0jg`f|e@ZZpzI@)}I{6pNK2MTtVqB=U8}x&j zuf);F>n1bilST_xEq^;|Ee@XMq=>K0s7aq&(Ja}j9Vsa=e}Os{m&^i}^+ae#82C;8 z@vL0Go`?q#{dB!Qy!9ehPz?fS#&{zSxz6fKfSWmGE^8f2LgMh^Ji9@nMYo7Vk+ZjD zF;q|7L@!x#hi5vhAz8i?G+(xIz&iD)lCVzIT|LEKGm&=n$#A$s)+1rOZ!>y7r`yNW zd%%zExD5TZPlHm2*4=#xm;X_SD;jzkzmnCe#pEY)M$P9$c-%?8>Pbb5QvSG->c-I) z#Jt^IqQEBmkW@KV?s}4m$3NH6AwbQ>(1dC`jmPeQmk-nk2J-yP z5dVX0LSy*3&FQ?SClDr+SuO-^21uImn}Znd_mwx^melDc(B`f5W&duKc(;{H=V#&;y$lCD%!BdIZTqn9q=73B5{1Ux6s*fm3Jj@68p@Ga}xt!V&h`ckiT$S;svc5i?v=X zIZU1OUrmuhR4{Ficd%0Nc)wdcSe>EB3vc*L@uo93jq#OFrom=K;`5{4{sqx}6@WJto5tajbHcq{|1eDxFC>AJIYJ!Jt@M1NG= z_W)wl)qWTVL$Ku?ydfgB(G^e^3KoGt!B)0IlbCJi27ko(=gzB!xBI6C?R89YXeCvF zW!MCxdICN}IzuUibJZx%K);=;b6LY!=Q~kNP6XeP(i$U^L98yZq-gk`lU>TXXv^8K zxN8=7=&EpSlw8x*l8Y6y>(7)u_%YF?b!YG>an8Inz}m7c96 zkmy^jf?U%7haI(*0R<|Z8a!_NZE9_a@a_KU>NYSt+JKIs;~qANb#4m{GrXZ8Q+0aj z`v=8|wj0C?9xZ6)C)JV%_+)ya4h^PTpiVv%ar6Zg{LG%fGgnJ0XQwmT%aE!G({WvW z3nI==^ml~EXCU9|DhG7hr!hk7&FG?P-q~UJvXL)Yo!nQpU$+V+Y$}Ttz&c`T?qOfW zLT`-Tpd+O$4WRa+blhAgr>!b4T~`;a739K~_8VC1)=pn%7!{Rev1nvIVNgEM;-hwZopoVc6gN^%^N3>BN$*2rxxo)E?`o811IDz zRW&$L1$CravQDauR}VMG8?x~QX8y+0EgAEb9$h713C6yooUN{N;LoQtf`*DalEThx z;QSx37=MOehCOr&XZXN#{zB~qAMMX)*4+z(cIQ>SNIbnjFB6>bxoVFuCR8ipjx6w* z*+@J`49dHbBl3*$Jcxd7G<9M{9rh(UF3XLLqni7|>nr$U8rotK(oHj*bHcccoRzds z6ZY$H=W~buoGsA#N(BG@@Bv!2)DO57izjhRKRo-!Ar}wCn`@?$6gIP-7?>0isD-z1 zbBs>0f6RcOKlHU-kSDTkgONyL#J7AyJfo=Wb!Hgcp%9<{aIt8%9H+UA5MJ16!*9Pq zyRE^aP6u)`12WzHl8U0X>K10oU%=i^>SmmTO ze!4vC2e~nD^R4-k-65gu08HDLmiZh>e{f*kCw9SML-*e!`Kbp`H}`O@ zs%0!Q4g!Socjhgl^(Cu%hTTj1IdJz|WvA`xoUTJB#@9`OkYVERLYiEZa<9sEM@xx} z>%AYu5d61)c>$;|(Dxn?hsE(DDd*q3IkexwxodXYLwEa-rOaKE(eKkXd)tR) zZbZ3$S{AJ`_Y;UH!+|Gn6j0A&pga6$2(#G z6h3>ZQKv$|@T@ZlSTtFI-i>n8#9=~Gz~+S5DD#ekrHb@<_?f<*_CdY2#-gR3mxzlC zGpn)N*u%BH2v58}(f@QW!txw(NIvg+Cgtt$b7w4Jh9LYkw~QSakQMb@qjO&-W3qmf z{F>NAO0r&?>Jc!TJSvvYdMVv;*P-Md?i@R-NIv!i3#Z`#-wvvF0 zb0ID_=h15Bc(&Os%wckwfA(`VX6e_Y%rv8|~{9@XSGIU*O68nJI6ceT~{VA6!Mv^Tn-dQAv$Jdw1Xe z)AMOl4)K#O*YG9YMu%m5ki#v^k(h|uiddb=8mW!SJJq?RmS>`j04+5*s3_*G{7q-s z^N-}&wptoqJLmR?QB~vt9a^AM_M$hzY3G<_HUhYgpPNUy)Sn*cX8=KpGC z)BcnA(qAsx<>zI3X36Vj)089}KP$%h{1znlx-({Xw@6K|Xg_JhfZ{roMbW+u&zm(& z-muAK{GI%J^UcM@8D2rzQ~Y0Ss+O-co_N{e$Q16a3YL|7_+tPRZbUeog2#3or6I^f zTzrcJ?H)(6Pji*-Aj4%p7GMV?$J7vgM-qnh6nhsORqqxc9yg7d&_Ae4wMnqg3u&v< z%?|=p(4oS$4L?cSg{5L5=mS_4nBSVMz>`Kt!&EVu2{~3x#;K{g^tv?lncrHE$xmn4 znY`cNb`oA0H(mNI~-CrdEn>@wQE-}RMQa9BGUegaJLIrQ5?sK zdlwsiN8j~(b(_175isDa+>1bk40wvYI2q8TRMS$(PjV+I`UH@K@xPmp9p`jQIZJ+l zmZYn2#j7NzKyWqb560R<%0cluXy}dk=4(K-%y9sk>(q_h#_Vx_0e-x6l30!SG|lYa z(SG7T6k@mT#G&hHW0U)wui;s=*73Fkp*kVn*YY4yxUzBoC&HrSaItEs^B|ET^(dvP zva88=Iu^nfVp1C5PhGW?bJkDnLLsg#A8t^Anmm`ZTBZpy61;Bd7H5AabD6UP;LIE4 ze`yDG9kyavpq=Ppju?pK>%O{~@5yXgOO*DUR$&6+0UHlv)WNLTpH-I3(rX(rFKn79 zSXja`YMlw*{jc)$FKYZh=%OEDC0}2Wtt1VrW98DUnKtS*%N@uULVb3>MAF+Xw#aC0 zif%w6X~`UJdvjla(hXQJCGwcmQiH$>istkg*}FPh&E~~&_-hBfpLv~klDA2*-YrDN z4=*@#rTdxdY-*9~+f1aZ9ruSVnaTEWE%yT0!O}5}9z2{VPZCTC@BH4xvR}m<-m*B; zRmB6XgsRRmJ@bLBL$ahZZ&Px9cf0hkEsg++41HkDTBR|oVh+B}l3Mv!L8akd*1 zBu8BL;o8qZYi>w-7hE5L8l9gi0h1VS-EOOea0xrJx%#G=+~*bnD}c#!wwH6$Irw zl+~}Dg+5d4p(cnCMj#!Z`%`ezKVdSanVoCGK)~LBe!l3lcf*%!Bp?q-4|&Bi*iNd& zDH)=Pi8vJ$yBD!ZunWUT2eEwkiOhv~ zafT)xH;{jIS@bs;618M8Bs5e46&I9oLAB7i`z#;nmm^B$!Wqzrxpw^>g9LMQXyY0q zVM8Y}Kz6v$wzmicGC9)g7Ga&-&4Vt{C@ks$6son|tnZZ3q@=p(#_DLB>znV6ytL7o z<~uvL1*3isSG2BM&A6yA+2&y`9x$K8M*R{f=~J!oDo2;mj=!A&EG%f?@XUL7{dQE{ z?zfOMaIRYSrVqep1BoVUofC8iEHDC*)aL z@+Ut*8N6^BnQ;Mdq5Vvyip-YnaI$7#cC`ROi~8>L5)XuOqX3>bQ!Nb1ZSobg&t)W#2#11sJ`ukdh_rcKX2m71XL(h3UIVWrYmKy| zDKy!E_Jvr3|1Dzk)kcb`UM6)}j4z@^4kw>((;yF^RgNe!msuh-}M>JZ6qQ6O`2Iu#i+76%*ny8;<5_{Se2^_D{wKGPqN=*Di*+MM)ymSo}c;c_1qquXz&_n`dQ8~olE z{}?W!Z#~7cLLpU zR1UVF9Zf(@G4s+FX!9h#U*mZ#J8H z6L@>^*KF-3hKfo_31@9fh$WRzeoP&Q+Ez(F z$cY~tKcMGjd{9(cs<5!=#-w!a zifa3+=34aBQiGb}U+Ga}8c?1-j2rVesrDdt7S49<{IwN7MwQo8eNsXoIsCc)Fm0KB zIor^*LR!XiS2);@Y5Dav3~~F?ce&2Ex0i>XIVPfS#5liC@$Elc`hT4OuYA8Hl6@GA zC$`2;)EW<2U*!--|Mwo(8~|muSn|<%JyCH24bUfedjXf>lz+je#-R-K=F$C{?fbc^=d7)G=Jv$VLEF@ z1~R)io#M%!B!5fMR4UgPqyiIv{b^epm|wTd`2s?%S)89P%r*S*hX=saQE;sdLp3bP zyi^EMTcP$NN-1?u8ki#FXIre8(CNizyldhgZi@}tebc_Fi2w>zj1o>HD9Z+T7)Vwr z7D4r0SGkdg52p=k_9G|B2e&g}3{Y`znfYmZB8e(iAW0JWuNKPO zvRAqOX^fJaHQEOcjp+U9WoD^cA=M;aYY4fN48RXN#aX562p)%2`#9+EEmoZ3#JQ~w zSiloCI9$6W4)ndJ)C7Ne*8V5HXJS@O1uJ7JYOyJT@&Zge>X112^~_OAv)fnP$k7LQ z>)Ia_l$7x&A(8@jo!%H@$L;R z;w(_QdJ_t&+(5h%^H)BaMLvmprTG0VcLWfHamsO=a)}P6trlYEDby8 z{W>K!^N**t9jZ@IR3D>~-Vp{D37Cx+Z|l&}O=#}hqBQ~@V6oEug^S>wq{CjrW9XL6 z*Ipa_it`q7I?N^+PMC*4G-%Mp(LX^kG-_YI+xz>}o6?S!1`KK(gZYbXC~qD-2$qJ! zlF&k&aVgS$VpXoK^GgWYWtZVn10^p68SBRDFvyhvKc{Qv^Iq=bp?3)*K@0 z^KpEStE-~`Ba=@JlMSsst`=rU?e`!47F`Ic^;SY5)}T90j0E~&<0dl=;GXd_)vvLN z4&{^0%Xq+`-@lW0K6sbW?W<_P`0a$aQVkfaI$U*^Ktm33{UG&ZaN4r--<-dKtSqhk zKJIjUcH-Hv?AX)XOPO91y_?R+CAg8(Nz2xf&Mm9uWssk=Z;|Te-lz*=_~Rh`W?f77Vl9|+Y1Er?+S>G$u=TDGR9rK&-L zt*b}Pe;vdBhOnA%p;8`*UW4vf1qXCt%n$YAW<_SsA6Fw(KFBvvgT2<=k%J!}rrE`< zO?Lx9q##{+UQ#CvnPJ2a+o48e!HA@>`x9-ANU#=Cy89MYD(`!C`bp=u4G}*t(w20X z{}36*zIA^d+D<6Uk0Ezj4QgA$V+~@n6&VJs z`du@oM3)OT-wM20@2DOYF8*tFyV|_09$hy288O@e(ig`#B@o-e23SW8@lt9Pxwl^H zY7zAjFSP)OsxE7E_%3@`=v*-gBi(Ju{%8UWT&p*T11@3^oNs86T{KCYx*3m;mSd^p zbuQnby@9*bw+10!{u-vg5!-BZnb&iBK_1bzK}4dxd{)$Xd)_Pz}Xc^MILhvkN=ZdRVB10+(~RnE4p<5^L+HN9+^r3JEE z?-qFkz*O<<4(@o|kG)+a&-_^a5Y-duci10sUkp#ryim1~UzSdQFrYFP*z?M)*s=f3 zjI+T&l!d*b$tST_Nd0Xn#!LUZjS6cr@*hqApWX;~keYm6Hvu^Sz(OG^?U<@90(bMS z*0jk*B{4@%bu-K#E;>xK7&UIht{toPpW6(+bQ0(;>R!wMuNjtq%6vU!(^O?U0ktVq zZ`L&SX$(hP8REeEP0>r%4k(ve62iTKkL}Jv=KT8BcANt26D*A|V$@!`hk1GG_tvee zMJjf}uEV_gXu?5EX}Ng8-Q|b70biJC5mesZgMPO`z<{ZIxLVi|mjyfNFp?}** zIc=-?W(a`zYr(ilY&N*g?`q$uF5>W~!tQPqYIt^>hf;7I#Ogz^Y)!wKuv7}zz0tp1 zG$5b$5A-kpa;hMdaUTX31NPuhi$ZZ>*>`lt+1tt+RQ{bgeMgghc05ltA!{Bg?Q;hW zKlIWBzx7{VY!1}_XB)^0^ZCFD+meeKb2-C0pK_A@M;|IqxSVTBR!i8QB$b*@VM2VC<9oxoT7EB=T1NxpmR zB~=~l%}s*_`(k1WWrt|AcGcQ}xbSQo`){gnaQ2eAI$SFts^ z`~43Q1m3%BKobuSzEh03wZ|tJ!vcZo8~YhuDmqRcWct1mfDP#_2qU>_TCMH4-CoBt za}*45kWM%Ru7vP WcD}f8M+<`C>HyazYf?sB4d|m94x1n3sV8(6kn?f-0e>ZFY z`_KK6NxmffmLY|JmmOE7{(Jeh_`m(UVf5`R1bK>%5GdjcPHR+u)WT(!L@U^w$3l(o z&hkB3*u4dt-Xbik#{=%yPDT)h2y8lO3-V5A&wq$z(HW>`rG17C{Bo(#Ts7c{^u>L& zFsXnwsj&AxRBL;*lJSv42sr+d@$BhPB>5mM*2p_o2gt0Vvc`L_+FxZSV*hZ<`LbqG zt#YH9%smGKnejgyPhbTTz+M`xw;BSPIkgKTXxSJ+f&L++uhFfO$2?t1-oXi=EqB0Y#Eh{zV)P z^gd|6GwbDnmU^W}bJ*3;&sOoZ(Slk~ z(T&|`JZ=ABShF4Z!(U^dvvG>O%^iEG-HbkONY?uHXnKZ}n;TWG4po2mzT@bcqP7|J z*nv8)W@O>OQJ6btIIeCcmWDg^)I>euo#fb}q!>_(%@ZR?(7kTdOC^rPWR9W^UisO| zRFjZ`Icww_f?QJYgKKh+PB{aq0oh86^``AX!W*Oci5Z_R<9zue*vzA!Chh(`8cG8? z$KAz}jGXf-yn(Qs=CpRQmC%;*a$s2h94eW#n-Q2_xIMXNoJnw{y7u5~lBWDB^jh%a zJ>~JS9NF52Ult3CxTX0>TBiEO-$a!Eo%p_H@b)2vX-(kX-nD@!P{n-<9w5%7LK2k} zzfIkitZQj|_E{24D?Ofzr7x4UP+W*}Ze8h7`Cg^8ZIww3f*kjv@&M!NrhA|>D4A7F7gHB?b zQIpa#!|>kthu)7vR{%ZkDIf^%M!(_tcflZ94vSmtdM878XtT5$uiRxD=?%-(I>cRL zng^KV&u8=h%mIz7WVPHloUg@J94imtE^LR||7s=5}mz6l+Rd7M4Hn@~( zO(p_&|FbI_SzSdRU}w4HQ4KFC^~*83#W!s6~VWVamN8R>XVO$L`C97+U^eM zVvmbQr8OWk2eWt0hK@6yGK3ige`we?&dd^x??k9KD#p+YF_ywjNgST#6)iLihp;1$ zO8DFF4~$HW?q?iPR1?MOTF9%XucuX88-!ZfCl)fv#v=G%OlJ6|iPH8g7H?x{^U8%{ z>s%q6EHuSkhpziAPD*b33z|67)yD**!ZO9b80gjMx^W&C9+Qs(fKJXofvtt2$wuj; zcDwocH4gc3S_6D~&S)0MwBlopD6Jn!h8a41(LyAcK)bnh#4Tj`)K81R>8wH5RIy@A zcg;koaooJIt<4;7A@W4t9G({ z>Mag_MI6PSi$Q!ueCwT^S7z5?Xi+L*vKaF?7S_GDPw_Pss467Px=$R=a<cEr8J;(zSW#~luQAkkUN0_;KFQ3mRjG$dwUStD96@Pple+r2VrRq@m zfbV{?MinNMo{uRdo*z{Yzsq<}dg7?VF3Lk5hVBBhkvzHqvwxZ`R(CP~4!@l%aLf1c zK9!y&TDYah72LM>6$;;D$KmI@cwHQO#tU3c z+A%q+7dvJVe4n0M3jt^{Rcp?%9KVX|3lL>Zv_wdgY$rb?i^T!}-gpILWw39C{~Kig zcV*{88d2F+yCzUlA3bk2e=qddDDdTb;iQ69;%)+yle^ftx%`aA2p1rgTPE?_A{iPP zpuKp2X^(UVc@7ugz@qHBZ%)89^hpwiveJ4HFb`nNU`fAg_L-ku5>aPy9Zk5cRhBdL zvs_naD{M9@A2g#M8n3*A#KfH>kO`&dsq5#!O}z8Lj&-;?pxgtWQCbi^q}rz)F=jT0 z6-;Spmy0?veMkto85z_Y2dKiar%Ie8y7+oyN_6a(*?D+NV76B`Ro8!MAkM1DqmZdX zu3Q%A1s24QZL(PD=%?mY_s@QPAru`BlIXaVt*jcEVH22xD)-GSOsEQI4|K02s!o2p z>xoM|b?ksKLnu)gdM?ukfh?d{jGqfVQP%?SCV^tr~QSrE%k@?`8*?gx)Td|_4 z0_G^>NzKpF`*e9%XwJ%X-&-irr_rmJcFvHghtiJ%c)f!(Y*6Zp zFW9|b54{;7ERVDhtH8Jga|CVhiJfPUe$R3dDIIXF*V4Tp69ku@f z6w;4s{~vauf1?1t42i$l7$4b&7Q*66UG1hb?%~mS#febEYb%+)q!OR*^$icE? zrIvohrKk;UqW8>(!mj(?+r)t8{iPRkWXg$gakF&t4I1)|TsPMN@*318o_3jOp?9dl za!iRs$ums&Sc@5d)&%2WYzvuos&OO7hISu4ayc549Go_@rkqC8Pmu2B>mI^s5MDF0Tw^#&1M9#%MJTRkpL52R55Y?=`yM9N-&zKUxvLJFSezS&JEra{ z6qRhTsi)Cc4wsTo>XSk*9XGmk8-(Y`Y@%uYrp=8;^Tz%{{Qc)z|At;~b?1NqGqq) zDUFGMmF&k_gbaWG_ALwmmW#nMAa)(;E7dc@dhYR(DlU8 z(ZH02P<7Md<9hWKmaigD7!*z3RFG(hVfh!5UdEdH+oRvAXQ7Z1P~WjWT{VOi@~xn)7TSy_uS@0;>SfBnE;^ z?V2(CLJ_KO9=c=1D6%}+MxC_cG5!pZpjLcU;(B^*&ow>(wZVv{cn2sRxVvQux_v8R zIC-!bO*9UFo*cWlO^9`7sv{m%SbgzMd!Ujq_0Q5=o;7NYJEWnXW+v2v%dj_3>o9mI zXK+nSZE4jEFK+K#o~>*#Ke~+}C@^*wsI%j0@5@;8uTcwD)^DDZ*fy2COE8LU4{|;A zEFn>`;t^k8tR9wKkcchbD0<-}hh@ZA2{z$$(`e}I*80~E8S4gXiD#S!K@pygJkI#M+LJvEl$)pP3OdQO!82wt4P1N&h6X1M3<$_ zANk?2@3=0l@^R*7e^{0Phkd|iM=w{C-NE5q%fg_CP7PwCEy{e$9+dsjKpJ%BIzR;F z7e7-t13p6>4|<%&Ot3lVSVXWqzH2t$9U3&%#s{bVjgp~DgDP5o+>&fc%Q@NiBea_< z^1;RZQ!*z*HxVbg`2$86%Q1NWZpszSxo1<(roJljKl4Fd|i0T*$^v*w%^py43x+xzdm5L=QZCM8T*Tvy)b?ZzYVgT7MV17K;zg{&%<$*ZC>79A z5$#gnlj|549t0MirT%vdm7aekhZ&@yzBm)u-llB9hiq*+2S!EQ4V4u%hOU{pC zf+bf=ELFLf8Vr9yaCLF3bupiqVYp!>4k9a=iib}PaNZf4M;n){#8C{sclCZ8pT-^S zejt$;i{j;Vj~Ryo{|F+FVW?XiJ68%XU?}P9j&$_!m00%TdFiq}BYvjz;9*vr27-G( znkhBgdY0t?LtY`$klPn&9yLaDL#A8u16Fha$ zw6K9NlJ6acSg3RjB%@;VvQjb1{G9E1!BFY5r&&NsLe722)UtinZ2PFswjp&>1rUn~ z@BmaT)SJUpY##Mc0zj=aA9^J`cR=E+h7)?Rt@FlN_Zj)MNYiNnag~XZsi{~j!+uoc zK{&2;q`D6OkFj@*uB_SDzB_i(vC*+@c5GWKPC7PL*y&gub!^+VZQHi3H~XA@-+Q0^ zoc9^e$2r!RbB$4R)>T(k{jXnHv6|A+i}c-N8#vgGFlwA!4fBi7c}X4kRJ79hyc z88JA^9e3FkC1gigu+?F1gm$a;yK4r2G)SZ)A>3wfNRW10TFcTr5x5dX($BMJ%(Coc zA=U+P%9B{Y%!dSZDHYP`o%~BZvH|s(Wj%|*p0FZkYelt7aN`fh(5)oog+bPClF4Kl zVdkjDYpY;Ve%0jP)FRkPJ9-KQuzx)tE+nKVMR{Gg&+|1 ztZRe0Av0zpbd+N&V?P9CBVz9b)LiHVb)SXZ;oyva=p^1z(5c|U74YH({k?e4X%60f zFGI@;JuvgQo=FfMVe)nTg5hHcg=_Gg(Iuw9&H4lghC~&~$ieW+4h6^!kaQXHqaX8Y z*As&MQS-1<^5! zq}YCQ8i2#gofcSOK0+*$$O?i=?*APAxdSi8E^(MLVULd#>khLZ$ogY$!WY3Z;?BX3 zxlxX+Rz0Whhc?Dm9b(257U}PKS)W}eCW)q1;+d|Oq>|3ls%$Y0NUxy6UP?(<3om5f40!m(hsu35JTn*Lc`Xz(|OIIZ!X2L4u026Cajbx5PR`0fBu}N~8+8Ry}N4X{A+-Cx573ugPRl+)H1Dcp|x7 zMty%v7<%bt;t-mrkNeC{M|x9>x>h|nDZ}w9&C1Xlt9pGCcNVe`WV|GK2b64M`=~u5 zku(o`ipdm{$dZb4DiZbfu3ZqWCp-D+m7*)f?7Ga6w;^o?pm+mFeq~nY?nip6H8)eVC`*kx{Sj zu=oW}YGoX{^wppvv6{x%vYX&_trf2}$wJ?-_^G$E{p2h0kI3rm>j!;@o--N&{;hV| z!=3J+i{Xg|@7P7wyhtMMrxp!bU9~EabC9!9-1>pMw_lw~Ty2qQ(o6l(t?a{1n065%y5V6o zc|F_IXq}CBR@LHB)h5fyB>=VX?vRVu_@TFW<3mh4=>qQ@InwOuF-0i@!wyHLFwMex zy5nk+kLg2p3{MAA@=w_^BLPuf3?TMH{}>CXk4IvPVuZvbhk6i1_a=YmD*T3-C$HuL zH@WBy)!!VKf=)FH9y^$RPM2*YULq+sT)~|w%k?zmdRM#EV_(+_f`_bTs`!^7USPx zBbvpO09k{nudBLl&J>Gq=x2V$ow07Z0rtiAqFB} zCX2#}3Vb6|ilECc$bP%hBJNPfdIG_(=Ia$>_Z>P6Lp7smv~3EOz%$XK#>0UGa|RN_ zesXCX&QoW%jVAb+aH}*ZMG7_wIC7sg?fS@U5TuFPcT)XtzrdSU>Wzf*Uw=WF(n#5A z{v7$BL)cNKAl};9Fj}46>jp8or0cFnSD1{N&H5SR2T1^}5G1l|_q?qqb}Mkejurm= z(6#(ACAt%hR&XyYIJb}b_r`GN5mBZa`MAqNrl=eFdnN)t;n@Ig1#gMGlfrodWh)cH$ zv78VrKaZ?}0;4{-w>%5(YjEq^)(S|d4>qIrih--dkQjqM5$6j67z6_agXx(;@eTYe zBVwqHyHHS>m%o9&8TP+0i8hn$c(XDN%hD|%^VgDTgn2SNpNno^ah~O-Klsr?%wvM+ zTGj#=keV6C-trCqfsjSrT}5GNlVUEc<*LsrvSgdy@Dio>>7orsiZdnl0EU1of< zmG;)`@ANdmh-<)A$?2q3ByZ<*Nyt?Dfp~TXqyT5N%$71vIE*ElfZ$Tz%ci z2q488liz0u^Lv}g?efJevfKSeofM{@;o+m{(k=&2DKT&T3D*dG-Sr+XcEc|@J1Wub z_dPwG-Km{0-{H}-kUONT3=XoAsqxc@1F{Dq%Fdv|1fg&vtOM^~!bcJDbIV0PiuY80 z^RC<^Fa;zzwzUFfCtwTWDZ=uYfdl%Ipz+McmL-*EeJw|-yWz^*i8sUgep#_qZCPwAm0PSM@vC zBck$+tac+odcg4%E{nf8(xdD39?TOXO1JuwfWKJg;mQj2`8*A^8vcCT?=dCDOsG~% zGS)7@jznC{!*-H%qJF&10X|0%kZPfDl*w7v2>>#yPDL^hT1h}+lgSgio{&k0z*&y3*@SLr0!kn(F=T|Q9MWFLgRsBSenbtuwPrvaf#k+Ps7)Gw z)79d93Roj>t`g+3P@J{kl+Xiqd!-CcKJg!)YK?cf7N(Kwn&iE^&{-LVtfrehZizD} zT}W%sk2QujLNG5mo_hcQFyCWZF>1^pafGXKPN|@n<3ol*qpgLPI8M1rS_AG|j-nCY>k!4;INEldQi zue~eX@7 zxXv7rZi%0LDR~WXBTl6!n4ow6rZ^X*cV>Qw*za3@zY#RAF}gp)%_=iJvg+%3fcE2K zwx`kpXVF9|^phD`uoX{m*qNNE#t++v#QB6k1-_vtc(cLgF6Is(SZ*FFUdW^_HvNY8 z8}vvW6NBU!U>Qjk88UkE5uWkD|JIt-_l-n8OOY29*Y&tRxQvG&9{%3HOMZRssPVd) zR#JQAx(IN^eQ(o`!1Z~&R2wW+8r&MI1T4;U#i1|#P)<^t8$x}2RH2hGC?{q&F zPaApevkiB|RIJvK9foE;iQxehO@wR3v!mNf8g*+~bA;+Ea;vN?k&q^5g{K6w1hXGF z{<;ipv1;QT`0UPf09QZl=@oPh4wLx+|kqVwk4sw zh|*TwyX~=G2$u6J@C&g7U%B72b}1O7aTVge!OA>!$&&?P+LelU_W|~v8>Pzw-?cKYxNV3E6wVVHY;QLMq>YW~94qSL%i{PAG50Z1nR(KLr#oY0rO+zNQ)Sg_P%HxYY zfe||G%Ok1y&VK8*-Pg8gwm~jF-iE%Rx?kkx(P}%8B^(5w;Zo~5UC@Sq*Px5RN%CL@ z033ReAmm{_lXUMiE?`7}_t}ZN;R!6YmgU`XPG-uO*6wYMmkqCcVMu#Lx>bjJKY+aR z=U?1O>yW63N)IeB?)e7WcAJuu=ZL9yrOSoLCosqDX*1uZ1lB^RSK;PC(-|(=%$xvV z(++)V=xG0W@G(tv=7GE0XRz$tr1Y5LJh8=q(2X_vi%{;)D}q$?>YX`nin$5aY})>K z7VqLF8ZWK;8C#}ovftWfG5!S7WB!-Y?Cg9_)zO#dXC6fx1ajx#ZqKhmAc z;`2}%6 z^h%_9M1Q%pQRVXi30sr$0ZlsjI}nYb_2Puc=!=gIjm*+dGS~1J-ZM)Wu+!UC(Lg(a zX@vXoWOu}Ny6v|ctoOC$h?hTko&^ke0VTP&15sBQa=WF1qJOaf_H=WgR(!~CPhoV` z4-~R~oV*lF9%0$2!QYoxhj-TLp_}n)#BGFin>h>Y;v+Wsd$xGMrlZ2+`~roF1!wKkVw>9C&w90V74bRI}Ql`trehVz=EPLo#cos zMRy&tS2IAu=g5RH5!&tFM6=hqYN!z-%V@S%hQ$M_)jH~=!^tO(J4-*2v6ax1*{lRB z@b;EF2qMXXDMSF-VDz3l6;m#aIU#iVxD<-44uWk@Woz_n5bIihyBKkWs-Xa&*~ssW z4;)D#VJP?$PkhI{=g25?OlBU(eSugeSDKdb`%p7UfonvzH@umr{OSga@v!c9CcwR? z>YJkyuW86VG9MyW+_lwX%yUZ=`3782sqp=_Oz&s(YI{_(*1t$eZ!E7dEZz=T`*BaU zzuPbmvV5%fkaEhEHjf=QY-z-#XKq!y0`aJ+t3_u3_~(yrkknoS_8f5L=sB1jH{u9r zyeplO=R^nm%9eGImN3*Bg=e?QI&%SH(qE0DA!2-e!04#Mih;UvXwMS2*GcPkSPy?x z`Uu;B2-cG&pomUb%U_J_-4 zCxSLsv-+-J5ig5azV(9K-0vTgx2Yf5FiyaS%q*TkFsISJMkOX%luxB=$hr}-9mv34 zZtdgQ@IBVh{x7LJ7=PneDxeeAjkfAjZlXlu;Rdz8Vn2teWA@#JS!+?@PoMOXgXD%+ zL()O~Gpf?hbO^i<-?sj0ZLjQ|A7QdfODz~yC5OH1dbSk2)aOFgtJK^%A}z8i0({aI zC*xbO{c8V;aNW+d1Er+3vb-0Sop{h~mC8$&hVUB4X@Y$$P3tQ!hT=YaY(^@Dz1@st zQ1QALn!OBGsM`SwSB9NvC?~2j`lT6h*k0TMfQVo&Z>(OzqM{xLf@f~xYi#r)FX&W) z01U&}N*^%p8I`E!{nR{3NWa^-vJdXAcG+n%5J}7tsixo5NcDow;l5-ABHS(C98pZm zYiHWgZ^|VQQ@fwY33AZEdetsdluAFDzoFv7B|(yO@4p|1NmVFs(P^JdG#iEuh8Uy8E88*3fKSt`z+~PhuKp-=}Hx$$7>* z>hiLT{G8o!7&35$XRO^*u?wt3))N?fIcQ5#GPL^;;X1uCYa={Oy(k2Hj0+;RF~GGo zU2W1yJr|KwRoIsr6FoM(g}C{*f`nSn#qb|3P2JXM@P00B)I1pRVc$5P9`|(!FdQhn zEi zxoR67uqrO*T>;@%@OpM)wCT3z*?PN?b9u=k>2`MSmObE*0I!&@x(x>&d*0H$JmXKyLt=nO7~DN1rdWjJ##Tw*qw_{|(+v3hn|C@eY!11LukZRF$CCCtcZh~r_YyfH0><}#R+{X9uv>S+ZwP5O7?xKrfl z&L7tGnXLpq04CYt_sw(hyp1Jw-Qxz|I3=~e@sz*@7Da+l#t>(5&T4Nv>T2(+Y}#G z$+!28W}Q17kn#?lj79V$3VqeUD~|I&z^siZAcYx=2h!ln^jkbU~PT7QjNq@ zZzE!A)x2*~ahrwL)0>C3ngy)Z4wI}(T5N7mO?ljNanTnS{nkJBM;-9hbL3{vj9M0~ z>b1=N$Dl)6h9F?{@j_9`tH$(6K7D%P`N9M(j^-#1g!qHfz(2M589R^}B>Qm3cgZDv`6DNlU#T`Gi+t`wP;xc&W?YL z0IT&72Zfi`=IW5%1K-fV-_G~*#9a(mmUzLi~D1 z`yf%K*Vveu>4piU&3#V_r2+!6EkYg03lHLMXy)VC3@QM5g=2FK4iW$;bj2htX03ep z?dMM^u8kvc7&=Ru@DmrcG$_aHY`-D@Q>cj^6skB}2G)`E7o~E3d6rfh&1t`@(5|%x z%(Sq%npxL8ceMqKK9ZMEWT{T*QEl9cLEgZGk9UYvsNdW2(@Tz(HP^hkMNb2wBpLDj zZvjr{aIKet+L+X55#R5U-EzI?aIggg)UUO$-q1B4ZVG8z|Du&)v4is>TZ5&a-@72u z`D~eAoWdnLWOl5~NZKP#71$E}s72+TS}Qu7T0c4>EaruL=6ieFMosVZZMR8+h^uK4 zaI?R4OG9JeUFyoi-$?S#DW^Ee#>=qqF2a!-B}kO!OY=7S1p%Moew5YBP={2EnwCW^ zTA2+Fi&O%*2Nsv^vLwHXsk4O*FBa6!*vNCTl~)w$NWodgiA$B$R-AVoOD*o5JvbFx z*R$}Zfo5?|RrNpTZ$&QXX)bv-1xziHs+Tk9SZTf>IY9M2XY=zdpZPQ z)XqD%oX(2)f?bg1Qlefqy(fO!_93*CL(Ffh_y;Nn5g&z-JpGq9fJIFGJ3Uwv-MQwv zafY0cM#B*Cn3I3EGYz2G@C&Qc)4$)vKnFUM-O`fck38J72K3ixH=UiX>RND`-5c+j z0ta)AQh87=&l-KKuuJrvfVZMm(n-lAphIAVU> z@4zU$#E=ER@=g5~pP<;hcG(|*NvkGr6Q4z|w4u6C&fh8;r6+R61lp8q-{@uX5^`_- zXCZ5uI7#-Ly`l;{g5hqmXGcIECI@b zpqWYOf$&_s3fJT@mI?Nc6U)meU*f8%j~-Cb9mc+mC2X&TK^&LAc;WhAH{Am2R0y>}*{ z9sfpU4AtFr%Ig%?!y!a@5f`JH$0Nk;q)xQhVA9Z7p$tx+oG0jOI07~% z$te;p7$)JkASy^ErqAyAU0El4^w)4wA!jLUZWRGT$aVk)PgE)@Oc>L#do}%V2xVdJ z*OYm+FjlYSW83T*gB; zN{Qx-#=f!{#RX^-m0Q1*3+)Umd}f!ExE&KL9g`b&u@@ql%TGh5dFO#Gk}G+<5)&hV z7)LVc;^~}7UsDx*9MLWYP0`-B9R%`7Zop3yQf-`q zy5YROG+9LL=BBlVsmDnXv|HCQ|J3C+6IB&%Ik$f$y_)Bh$K~CkFj@7rhNy-r9~M2? zBI5A)N3A1<>?`6q%o^58CE<2bw^dbkW-eX}l9Q40J(@7F7l-w_`z`)r#I(*DM`BM_ zqe~8svI-zf*sFyxC|(D)*2`#FhtBQ1G-SEG-KcvuV_Y4v#%`{+b3&b1Na)wOy_`Hl z{B@p*gS6@!eP>hq#Ml18S^juJY>0Z{4x|2q%j%uj9j-HrUFc%5-?NQ23e) zAnsha)HKJ#ql~!sE$7}>d=CHuq-0W2&mx<@svLA<$c8p3mgBQg(UQDr0QcA%}c<%x`sl@&S&v)8y*)!fafCzxw(lm2F}C-_dhRI+NcVtnC_ z-|~V!;Y@NhGjuQrBkbwZDF2ZCnw}Ov>^1!OBqZJB)j)t$n|m=~%j{N@V`8mbo4C#s zl7n!StE~7*5eaI0duK{@^)20L=-?sld}*PSVI4V`xWRzCeS*AX`e(7F@0KKy)FLdR z{G3J!0|`ih5zX~He*aqJu5@{5M>zcNicvWg<9?C}$fzLDUHK3zmD!`h@1E`iKk8YV zUFn34iz4NBYS_>hwrcK%9h&~0tN1do_v=Wtskb^wQ2192L>38wz;vXjTZ{;w#>+GT) zu8tEp-G`bBO$Fnpao)x*>+l^O=0*dRX~n`|yg}fc9NeHrslR8YIGrAyCzNj-+WS&) zTTGbEQ{OI_&R7)UACjF{!R#;|_7;BurvHdq*#ifTiCNK3Ua%Xinx_UkXPdypz2`1W zC?p{{jL*$LXf^pJBi6O`4z8>dZMVsN7Tt@yWV;j9Up`YaR0$#yTSp4^g2N+LRG>*l zcGx-^iEp8eu5Er-Q)Q|D44cjFj-W3FMAY;fG~rWPANGcti|zC>V8YPT{j19L6~@@q zwA(ZX(~>`B&O+4Mx=j8s1ix7;E%+naQ{c^QO@sC|zM?9bcH>TtaJ%gK=JicJqJQz@ zs6WDGvv>uy@}#~q-p4)J@ltCBONQxjZE7Oor;!iw{T@O5?%Vb`xYbi+=XE%lbNgIS z?@0hZdF#CtZ$Bc|7;JX$4o7cu>l72o4_;yV*x2Ey!ass>cbV4k@ql6+7;oRP8-cE| z2*t+{OQYy5=aEMU_Nk^79N+B$l&cz_Tf^bC+S~J4tXr$F2TiR-uoe7_&F2E%Pd>{< z2%1ljtLFg8T4MD*RykSx1h_%b@kqyC!X}6AE4cqHgPM(#kemoLn1!J?o=#@R4Z5(V z2vrO?F^eQ0f6Z-a2-?QKb0z|9htH4fo*MsmNt-^Xo)~Dg+4iM0y$24VRd&RsKJRSG zB{;7}d)D=zRS;9j!(T01Y0{0F2%5ag;g`Gg zmS6Mth2tNJ(43}E4q(v81L5ytqJ&+z|Tj=}5f0<0q7HHnL+Zl~|D{KPbE^?VZ z$A%{hH8pkZPS$dWy}pSge-AA+9xGP%=+uwcc(B%a2u z^kAwAJX-I_E#+in&OJ**y|vXvrxmlp7c(^PrEr@JNSpba10^bCrfoR_iJ!ZHyr`=K+=+VKhOLn za~O-6v^if2M*4h$;FkGF)!%yYA2EEpXP87K@4#l_D!h9w-XE4cZ?I`<2&s=V&3KZ2 z)x#-Zt^WLF!}j}s{q5&-E=lahxehX`t5M_%P{+o`rYkrxwuC!Gcj^bkh7BJ+;D&Al zdd45t6l8DRmvuB)I&e8w_9&&#@dH$t?NZ$GVglh8+BxcQQ(WKRbZb@0NA`Sc1|s-; zp5cq0Vx&DC2E{6Wi{RkP};eQj% z&8%ZC0zb?e&z=}Abx5%1MBx}b2wwNn0lbubk=@TP69dOD6Md@FRk(1W{Fri2t@Q)m zb8ss!!^|z`MvJc{{3cK9$c^sPT)`=fC>>Esp1vj59)gLxV`uEW5#6ga2!TETu17+= z*Of2L!GyMdwL&16Z>;?S6W(wi_rIpjfBpF1O}zt7t5!N9>*;x4ZBRa9gHdh@dM76YVdM}1nW6a1l zmqf2os=FOmBMmHA+81Hg@&w4~y=s6?`^PZpb=h)ILjv!8zQqlIqQ{uH4hxL@~ZCmbRzxYXRh(o*@G3~V?_PCMW2_BUxK7Z;BQTEx^);lC>aLC7ynh`$C9R$%+Z4;WS%XB)PP z_xZ?+Ud}mQ>WhLt`w(92)wvD(go}+3QjV$cMI?*RI=AfRYt*umb>mrmEAPcZ161Tk z^iLENiv4W5o)(0-LP`5njCaEI!qiH8CbspvW#@7@S<&f(<0n)CC8U4%;bi5m z_3;GiWw5B6c6Bcd!^bSDxCCSXgXL!?z-TwiHs0GP?K<|x{!qT^G2l-Jl#ly+;;78F zGXXUkm^c^I9uHS22G;%hv`w(^l?dPI84@~@kNelZG2)zi;MqU8#suFwKIsU(pFoWs zW>cB(5U42h-5t6>vVAQ~eKD;s%J)Bu$7a$S})BI;j7%s5uw%( z+kQU!AK5-~UK7%rsH$$%(4(@blV?`h!sUavDmwM3VMUB0e7Ao2QaUt@pdk;(zVV>V zTUno+e-KJ0M&JLs;uEk^Z3b?4Qo4`v%7H>(K7W?@d5i`BP+3>Wv{IFvmF}}LZ83N~ zeqm4RY~?{ZxcVV!I(9fvegtv#Va0*>rNJ)-1Jmb#LZdqSt6GguTR5prCb9F`N+@nV z{V!dvBvqxb>p8WIJ=c{=Y71UEVk1G}4SE?k4S7(>gMXjJefD0vW9ZMG#;0V;^3oul zdUU=BR7!3+Z>HI&%^x!Jp%-SYSx* z?-y8A*>i4GHilTOpdiT7aA9L2%rXU0xUC^^zY;?e_!|6dbnuqX%JLGdj@&z=N+7j|x zo2W84lvpERpDrCv!p^jVP0J0KZL?oG%@S!wcC|wpmCp<3;6Yv0 zsaA+m)vcfhy5LIc|Z)Vs(Hpy=?^c4Bma447U*M**!-V`YIM6}d5nr6wdz-AN|^ zI)%@1N+wr=-NTwv5#fFmV$mxT7YuCeM8WIY&s8LdXyrYMw_&I-Nj_(2VF0Q7w!~z2{{AsEF+sviKb@!iKpwL1=;t>l4lY@d zcqvITRT47rZKkG$jZ9S7jjmpSkxn~VaJss=j9hrXBjiPN6yWGMnsCp3Q{YjIP-DMh z!ZhitSr+5iV7fm)OVq>UYf#%NkgeL?-$t((aP);CAGR2Yb52sC~?W^FOs0v+&w1n4b0OzFf$suaEF z8}uoU{fI~xIiS88E$Q&(B7uqSMaVbrP&Rd#`E@a*h$1%Baj$MSSqE>rJ$t?DE=eK1 zq%|6R8Ur~J-}7%H$p4ik)mcf*cpOu1kT-Fpl7HRK*-Mk|f+8lG^-P87aPjrxSpMz) zlyF?UU$q!FK{-R<<2YqL#_P=5yr%W{5p`9r_(p^s=s#@?#*9Y? z^taO`qF|QuPG^*FWm`ZwSSDAQ8zc*sf1C?y7a3NXiAgSbT4@7l85T|mgXb+nCS8>1?!Ch$NYh~NLf!d`lITqIf@j!&wJlNj_UdSZ7jT|PJYrZ zUo@jUWG@j8QJ5UWK;nD4rY&mPGngKP^y%KwmDFM&l~x7m_>ZBGeu(G{Fq1q;V}dJ6|lSe*WKNu zUzY@?2KD3j>056fG03E8raQikUZ&dJ#bjYw2Sq|xXg5!#zerSAjowAW6=P?5{0#r| zqh$1(y!A8zU74iiIk!KwI2-ef$IxbJJpA}z>j~#J_x*zfQ1V{-H$ozhF723PN<6ZH zZfeMSuGlPn3UQi6O<8xY%pBtgd`Uq3W9>>6^8kGsW;%obLKWWS6bUmJ8S@+G4-n1e z)EB$A#-eGI?CnXby6x?CKhl(KWgw>P8?@fVDj6{4WKl7^=ms5s6vNmn=(e;T>|Sbn z`sedFIUxzhH$^Ms$^_uIcp7%!Jp?3)SK{b}m*`TT9`TG}VqwpWgm~Z5Y}~68p6clM z#b0k(;rH(AU8q6~BcM9eONPytTTl)Fj&WJhlkqXP%474^H$?^%6!PunJ({Sr=;OE+ z>8UdBuWPf6_#xC&j9)gyg5GtW-L59>|CjDrZlCPGj%mw3T@1&bIy4`HtIl& zfmz8?FV0iUog?bQ*5BdaYn4O$(y0KDx$RZmM&}~RX*w|8%#Ob*pmTMT1Pxp`aqe@F z)J4C#$|?WiGPA+5}bQ?V3HZlPNSBKopZe+1-*DZ{)PMCjhzWfij z^1od8)NWdsC7R7zlyrh#2;{dK%8SF>gCkRSEvJsO?+ujuzo>az*m0>yGmUINvuf$$M^QgsL+#!+Dfhv3f z`HTu+8}5-hH5(D)k~XizhYfjt%wB3zPM>jnDCTMt5-O?#f2`~Ac|Tac zE`X{LM>UztIP`9k-O|_|W!g$uEulN^Xb~Cfd;TX`AWRj;o`5O+dgU{nPk&WI->Ag$ zs#GB@_sp5`=)xRa(hDY9=k^9DF?^?Tq9Iw+ksu-Iu;VB~j7n(uzEEd$VLFYUr}2R)1W;T!@*0}=0$eCWPxy=#7K^QQapTB)fF`-z=0_Sup`@klXna7 zAJJhCfl#8yJBtJ0^^#NBAfSo~U51dBXE2fxT~#;Le!Y-VDIXJY@(Zv6J)2JYCDiO~ z^O*{iPdqi*qR-=(rg#ebe{)g);c-1-phb*p4=T$)0YRrJ7yN9w+JW z0tvBCsglX$N5taT7$n+1jmh3gM-$s#byqqxGZ~3TADwZnIOMO;tC*Y3lNb)hs3%3) zB&$k_Z1UuTt*d3?@CDw>kjYK$TM1y<9QfhBSB{ct4p9kaRg$YEq=%3YF|Z^>I?$HA zOEY4fX5{_QR`$nHBNl25s)1B;;0D{=&hq^a8JP}^-9`Zy8Wh+>0mjAL4#xT|Wj#`D9wkdDhbGH(q%?udDa0TlMn zwTB*;(($kSr~Woi19>So7ZsEXY;Y+T3l$}Wmhx(IMll*QT!;JqWu+S2mQ+g-=%2`M z8{)SbHUupBk#;qH!-2P2;|kwqoQPvBlzn4{naCgu21-ZNegKi#^gLNB0CE;K>R6Q2 zUYL@=NKEs^CVaRB9oNbwR6?5Ewxr9#lKI|xG!|a1rr0}z%D8B$ej(5d@Da65x;b4^ zAu5)yYh>Aw%T7Eg*#HC!%#N~)PB{wR@ z7Q_TcprqH&bmpeNo~G_yuKBb$QA@@*qJS(dxnjkX9l9sad8A9(IfT?0;=N;FrD54N z-GsD^2Gsu=!O%1;=BE1t@mUD@i?({ZB+yG3kvs(5%RFTD*Av|3O@^lZSgNrPiKG@l zU3%<4u!O%QiCk`oUQJ<+$UKdp|H!FXkFsXQWJila{+WY~&>F zVr$5!LoeXU-=22)NS&}9Y@eU9*4i-i`acab&q=lZvL4eAU>HjbefWPhdH>_BbZQC{ z`9nndV3)yphrF6Jn^p0^I#%kxS}jr*y6qF|EWHy0Bf=zdeaPF6w^QE>UWJH5P7)HW zkf0zz{f`B~+OAP5FmA{|Fqshy7V_X=>JKcsk|AOhXn{R&7?Ozc|43W*++YDFZ6vqSv{Lga$_T`$Nn;;s!Y=t5D5Ue4#3a5 z8!z2Q34!a4uqNo(GKcHcWU<$NW{kJ%moHT1lDPiA`w#WM$R6x4GdsQc$A?4oKOTGw zSY$GFVK($jjiBZHtbhq_#VhoABf2gib0^#QaA0M~ zXfQEAoX3wvjVjcfi@1p4?7bqaN=O^5SR*WMY>6c|i`{R=a6$g3l0kz0_iEA?mg(IB zNA(dk9VT<+el#oLwPzp*MILrchG5HoF9u~pg;_%#GcvG&8r6QI%0pd&^MPf;EXseH z_yiK3ezTKc=0W_XyQrw>dmS`)1t#i$xt`%nW7A?npyXG0dTyWj-0nQ6seP7KK?Y~i zupZiXlm8^kXt~xN%3CA=7e=<|NY4l5V>lomy3tqbB|7L?kNGW|`pH`;M^-@4aji`P z&K}r;$D?qi&lO73mT^a?3DL;b@b$(tm-1StvP{cv6-bDL*@#EkN{oMJL<=JR1%*C2 zc%GC*C86FN>^2$US)z?J{; zs5P3x(&wQw+G6U4-^6pjInTwSNEM+yMJ3(DYJ;BJ49vrShZ3GT*R?wZ;$q5V1m?1s zMmas|Vb4eX|Bd)&IKHR^IH4>I!oueG`4zdoiUU%s?MtC_wWoOx7-!ClYwP>YygEqj zTz67R5s-z}&Tb`U*rr%kXV1VM#r3QOcckz^i(n>1hyT!FU@}fe`LPkj!#=;46X_Y3$ji!F0KRN zK>I+GeK5t#;I%!XaNF%&=T>xMF_lw=8VeW{dHsZz1=c9@xj*RM>IW{s{p?sV>0u`A z?QCHeEteCDF;tUUxWhkSmSG9UX<&QRy#Kj?D?`6mp1QK}OcjG6cNMx=R{+X^F5nBt z469hLrSSlYX5oK zy3d(ThnS(Zg0=+U{>ww*RP|DJSM1)8zQ)SD9He6c`5yMBNB@_IkwZm_r@e z^fVJj1hLu!!X9?W!)rKB#-KG`f+6WcIx*VBJ z-<#M$n4AE)_|zO|Bm+AB{T=Hk>U!bKdtR97I_8XvH<5PzL3rLP^l@7`@{u`fVf8@1 zI$5py+^h z!P_?4eGuGp9^unbjT!a0amuS2Z%eLG3WkZyq?Eg?bym&52K0Gl2ElR`oNI<2k@31x zmtTqo9f{}cEQSgk1l*7HF(kbMiG@br2VP1H??sm67xYJZeK0f9Ti4>UzdBi%sR6UH zTNJlq`lnXLjvViEUap%|c>8dnD`0rv2u_}9)5lWGorG(bixdfH-mvS*rHa1L*UWR_ zwcD+S3gxOVa6qzQ=>O^vs)Z80_u8lk*4UG#z(O6=2iShDJwZ(SGX`=Jh5>!5eK02guIyOZw!$sxpl&wj!0zKuuI*TqZ@ z&CV^4%gj%ILQx~rJ?oGMuE%&Q<9I(Od@*YRKB-A;)B2&xfNAHD)zU%+PC=DbE)88# z)ZI&AMlEhATmRPV{PRN7u?N@}%@%+hkS zkdTo3G(P}u$|;!My&er?&N)zByENuwQ6s68lwXyU%sNYlJ`Y3YjCIXQO<`MV&J!pG z>e0h&16ghUFpu^v*?61RQ*2S4N5zGrnSIiy5Xu7fvCx?<(|X65Dm6#)X0LgA@p+{| zI3NgQ9M~YgZ-IMANev>HOLtXR*kS=OE>-W`ipra-N%mJosEIwpSiqGR_3zl6J|poxpuJeELR=r>7fWRhWfhz{p^~i1g*0EOKLxVb7@; z8Ach^PtLc5f6tq>Fwt`*##A(0mZkm$_Qje^SiX_mB&p1K=v78F zC*HB0i;9#a0>n@q!B(R3srj&Bg-e>74($Uqwv@bg|W zT)?%-rq32n<{`;mewxZ*xI?Ws0-#HtRd#9RU$J%YSa!0xY8UrnU_jjKgw0q~wRF#c zEl)v;qWW%FM@~#^)IdUg+)hx6V781&1@Y~K0nH&4#rt@R>T{b~oI`2w_}l41!X)w! zKWfx^WYaP(-a{D<5=E~=Q0EI5f$Av-I&u*M)tVpnS%aW^hMWoh?ZD4oUm*X?*} zq4JieOC)I1{)-+z3WT8tnM)Ilj%A%%GMwHF;PqcnCoJ}?cw!3wnfgF4;c9jtV#>(r zUIFBdQ(4_$flFMCit<-ZcUtNP<%oU9mqkf)ULaK3b2E8-Ui+t4nXojG|9Fw5cAg@n z&R_ByE$-bvuEtqg(07zaWUREdez!6F@nz_9;yd2G>8L!d?&oNR z#5YPQ@R&~9%cHw4t+&P6AeME8Pk=X1Z5OXnQOh1~ikMVP+&yIRCtJLjN0@;wYQY{2 zpHvi=P4+Ppo%6bipM{hhnaJ7e(vikUjRF%s3KZ+^RhLc@z!g^$e<7r|+vGAB5rRVY~LIU0g}(Os3KL(Ou7bW1^o zr^+*PpNQx%q(BTvQKO3j>2pCtLmw%Rz1DLMPVaA>;POHz@W@}9#6 z<&7x#7xLMkB$-n6@2QVDd>@#fKWRjwZ_O5&j=HDI6Vph^vrSL66&Y>|+AC09Koawe zNc$)Vl2T2_M@l+MAEtb-PNPhs3{6#+BeNt^5?j~eLy0Fhh(5KDxaS~v_Leps7fC*` zqQ;uvG1;;2g3g^sXLgh(>cma0*0$9n%y~TM+1hRxo|IT8uOjo7WXziS>G3ChM8iO zgRv3k$Um=HQX;erqWh>{3e45_ism*`MY3;nz9POqAWN6c-9n)~-o&e>+ldchD4D8Q z_6HTk>F%n5xU&+g{lDHkI}*<4c9i>%d0XK{r_M?`6#V6mEk98i9u?(XTuh%$N%d*h-WE z#><8b45mGtg+3MQ+wF~Hr$tE0D9t(bl3Z(WIZx(I*>;qIXSCuAbBDxby8XO0$4;li zc%Y`(p&6DP1CTSxyN>nMh&Z|;7OwUZfrrcCx2QNyiUpr%DW=|120 z3;Vz#at}ewyt98l*@(AkJl*nOqVbUSD+?iJ-M#bhP9n5|RY!`|1pqh?aztw^30W!4 z7SJ>Jm6FOrX13`{6E`11pODMGhMl!)9yv}h(D<@{tz#MweWO|1e;ql*N4mR@Zkyocxq6=?FObMX%Yownj5Sj97N?ar27jUcU~=*8d`C$r`?$ zNsd5@f>>BqI?FLW6MR+}dl3`IkhBti+YXh{KyE1KmID+<0T|&`rbr=o1Cl!aZT=SH zO_iG-zYYGKUGy7hrTv0Nl=d<$LeayM7y)bf3EE@YE1Lg&m%KDc(;dj-;w_(-y3(p> zAzfE91<#nYr#|CYV1C`KY z=aJ7WOyZiGnxZ*eXukJb1+B3E+b?L{WY7tYbu1N`gO7?p;~<|W-{mh_IQG8UAMR(z zap>i*Z^~94H{CYLiefu|g9;C-vcI{X8;(FDpRa-KY)fBV?yHm)2n}>76@-l9^CKHy z2>4Exb5G~1w~@~8fFf%q@a!7)&*?ct=GaBVU?Li$<%fzp&JdZX8c5s5Wap^FhbJwwO!QpIF($;h;Ljg zx&rr30rgDvqXwJYVHghJ{b?dKESGxb)Aa>Zjr<$!i2ryT;Yi7e){=R!k1?O}#rpTU zGs=kASGy^5qEQ*!bayg4?(0iRHX$t@@mIVuZiesd60?A`FIoCC34`s_MnY6MoV9Q- zu5Q$d+*rfP9O}$?=&5aiFA-eUferHpj#elkkf}#7$6kkXu?DJPB`}a@TQZqUi-?fC zzB88}uLEs1>{ptqOl`gY_q_fa(6!hvN+X$o!;dY^`h*-T7*9509B4w)HxIjNYLwEe ztUi&(X>YY>C}~~$?sZVif2#>wq}HqGs>yyVD|OzT&8N!IHzZfqu&Jz!z@DH)nVpRU zN~HzF_vr!Q8ruYw-?R1m!QpYl^Met&)vW3o6%(Idp}LgAD`U(5+H?FurT`-;CWWNXXH-Q%y{=>od7 zT#g_(8#}0%t1DA9%PzuB`MGX7w<1f`fBs{Y?()OWW1JtNzatWdl_oH+<)0I$n0(^w zzgn%iLl9JfmwK$Tm)WbPkC={CQXF>34sU#l&(@cNU}iiEQ>JwyQ`{FB%|die^Cy;{ z%#S!3n(kq+BPH9j>i#X9R}Q4Yr(+*o)XdIsm7I>O4ZhWncBulxaKL=VAk*%+!(ntj zLhFHmxeIMgA=eI=if?<@X+%t7@U4OG3?um! z-#R>nb7#Kl4MUK+;peGTRiE={;!^`s6p-^eqAcQA|H7Azm|f@RjG1OeaQ?JGFcsma ziGqD`>$NdMFy)JDzx2YYx2HzbSrwaERvixCoKk;S4o0Bq;kuP-WXB7HUB5fJS>@2m z^5%TpHX4r(8nM0Fh~bve3x;_ZRx_I4v%y z)fKzSu}?Q=sQY~!@McKDsJH5;uL_m61?4uq5BGeYThrvpXx{LTqXh~&yH(l)(H?(% zf{gl2+=C<|u%`1WZD%CS{|x`Wd|au0IWRD=;t;o9q^+Tg;>pP(ca#vL1|XV9e_?ao zpxsJpA9hQ8*4klZxF^8bZ?~4w6TQMgWg*6wC9RrJj7n&r)BYt%E%gKzQS$=IXfWZQ zo0cpKoXlxB_dlXF8#Z-waR`)DhL1fz&@kpHo1%q91U+UbGlcleu-s3^JAY%HhfnU* z3@kT#NrlU`O@c-oQ)9#NZsfh5D|N-kCHZ?PS%@m>G73Xc%dF)} zW2FnUrMyYQ<^8jm#J-F=pRm(jOkNHgqw$qJ?%v=Nad-LYWF$72b|bA|oR#;EsfEZOaITWtqS}?}CAYe))<(tVns;95YdCtGE)d94{f#7v zFxyyyZ+5TF_E^V6nl@5FA;Pp#|NWG&Z_*Um7h3y#&=t#Bg9>C^aX3N&to4nZ9pL87 zt95&|$#H(W?t$K#nB*|0pU0K^1M8vl8y~j)+2zO&INE}-*-W4_fQ)jv0fcn$Cw;Ql zCXa$(ds&NRctiVfpT@@}Mvh~dT zDwCAPcq9J%gHAH`b<+!hDUS6w5&IvD35mWZ;+p zPxNEMgaJ9=2EciOoVx5zmc>@vDGR1qF6haV>EzjLrcbq%ZPDI!U0by^CW^ng@xzds;_U@I(Blc{G1Hs zbEjx%SlId;*VY7nll(6c5}>j5jy;A1fyrr1=yug&X-q8}Zz3B7S&_j`1K`?zKVrEt z)E}K?kp4i!DXNZiq`zc1p;{lQgHNyZ|U1B8R+W&D_1XCYckUzGrzEOQNw!6O-JActz zGW{(QRYQ!$GzO?)k&y-JpA}*idYVfQHhrQ;N^&LNytDTy0jOdT?d(?JG3M*zxYl8E zsjg~U2Cr9<){ZYtVbz{mEj6914787f2~C%rF31t!k3A$Y{a;1v4gmHWyi9Y<)RGZ; zjknm@Fn@ujlt8Xi8wxDciqox(@6L5W$47b{oF`YxWp0(&U&+berQT8|qq9V$l%Vil zyDtPpq|i~8c<8L)-Ks$p0TtHXdprx5>xT<;=AX0xh>^Pn52v?ak6Rphqv0Q=ZZhkl z1Ko`%D0@Xt6CH#H{9cfS1`>N4(EFB4k_+)sEvG@1Vzm@jf;t-pK7mfnT>!o2lc(2- zeG$yFj*<#oB3|T{O*j(xQlGwCV+S`c?<#BL9t}x-t=~%|=Fi{S*KpJGxoP*Tt6BKm z1KxTC2Cxl_{{Hn@qUvn54HR(_+~D!YuA-c2wON9Pxx9J$b#3spRu=bOLkl|3^q`0@ zX%Vpv{TzSOfUB}%ZggP{<7d|;6L+R`Uz}Jod_{aSb{=Jp64L}Uq|J#pmJE; zti&YEC8S4LHAU!)ciz=~bVIC}knGJYML>;wER~1Lo1P{qwYf;9RR_L86T|zGIsWL3 zEC?1KA78aH8S>=@#f^H)Ei<3yjQR@0?T?VLqty*K&hz7EcP*?Um2p32hE1L4*@ZgqNrDUItrb!0VTJTeFW+}uab9I&!z13^5Xrw>Q)Rz-6qeU2C7LrUIP7IvMBr*blB8^5Q1`k)~Cx$alB?CkCJzFIxEn7T=beExxOr_#gk zEI*B`-$+zjo6Z6Y4xpq-^#2dy_rDKY4ai&5n+CSzD?a^&WT&<&cy-p|q)}8@Ns<@J zp$G52kuB4IYB=x~xc=PIq=%)rJb3cM64mjojxoAO80=T<3x=VDjxC-3DIG+>x)mP;BoVSb{4^1t4`k+BvnH}r4WcW8@LncYZ`bdmWknK+DHyto0^}6& znH)HFE)%Rr-N?Q}@QeKC`3pn5Vyjmy%>}=DAH+?^CYEW~`U~9z7pfK5I|AV~zFCNo#K>PO( z*SH5ZL-fB`0NE8?yrVlbZ*t(KAK1UlA&1|fbQ|U*^(=2Z`EsXCQH$QgipeEJr*O=I z_daB3uW$GSYV}?Yw_-bnz>NiLnw`#EL{F!!r%lA~sGeKyS#{fB5X@%~VI$D;y!jV- ziyX-;#Zy;%QYT*Qcai3FGra1mVjU@$_yw-KNd&rw@94glJg7Y9Z_z@>wNru!Y4th> zSnks_lBHVky{8@x1LshT>R$9)IW{xM!+D+4h+uw5b-A&0>Fc8S5{;V=gx31q6!VQY z;lcblP|I40x@o1TZw^?bGpnBO*1tJ5J~fkItBmxp3$Tc$K#;2xbX8=O+kJZNNa6!g z2I!vMY_bL3ml~C2S>XpK&{`%KXq@u$PG*FOZx^-SP70FnPJ=>&@d*(H7^C`d;+^(uJ)9x z4#5J(8%~dRnRe4Y;|-MGg^!T9aR; z4ZNk~>OcOP#fuC7z61i4DSW^;+SKc~K+MR-hZWyWVILUKGwI2OpbT`xR-1y1)(r>6 zhkEKU;NfZfsPkuI8_g#uaV`sg(oUyJgr!Ycs-23zb5Z=*$xd*7@(K!@F`-1KahZZGmg1Op?Vtwt1a$T_p(uQPay=Sd#2S}q zO#RmOHaQpgB`|icP2DOZC-3!XW~UjzbfD8^?jqh(vKnk%oGS+A*$atf*S$M&+sLLv zND8}5zoFMGLTwfQ{c9VuhOI1t?~t>fUzxGn$@F4PuS8~(#8T;#;ZVFUm+h7im?7!1 zNwg=&%vYsQ;8|^Qe|4SuX;@!4H6BJ!*Xd_J@JP7Vk*Xks27X=&xwD7#AEo>wAanl( zNJ%F8B;mgTlA!#1DKQ@iijB;s=buzk${1evp3n@94P`g3EOr^a=K&R0?^CEVkB<3# zw!cs!dk=pPP3KT1Qj>nO+B~avHrd#6LrbeEo;r83R%F>5NUF-l`h2=pV$OPddm3(U zr8}^^n44zMv`M?^z@8N|VxiJPSH7p^y$cza zHn5JHTI(FOP+w6#b5Ub^t*(%zy^vo5@sk%0t96?CVli2Bxw`75f$%xqNXZml{@B6Z zr0>x7^D*1mx;K5`C-d2S3-3_4(jr|+^jh5~w;G?XmeHUcDyM*3w{dY4n4mnXI3#`N zMt24*xcPkFr}6uaJkKhXM|)XrBzccngC%G>|HaeSct)PyoH@8 z-~zKafA0qr{ZK9K^s5~DFH8B~ z5(iJG7tO~Zmic{{;2pE$FIZxgdz&7>nwI->i2r^K`z_%)mXb1r5}bZ~L9dG?ohOC3AH=Xo)SROD=Q zI0*rUZ;MXuf1P0ki)JZA#L%4w#oymLiO|!w5C5LDb~2W1a#!W56AjYg+?%J$g2#ya zXz_I?2EBWGIkA?S7~wrAN+m>1Wf!-=>fO&5MgKrU)CZ2mSEV}R{rGWW7gH2$u)-aT zLekqeoXjiA%IkgUG=Jn9%u+C@!W(n*EtOW=l<~M~w@E_p{8wLJL2qOJL4nYXf2AyX z@3&zV+6-)scJv86lQ^*gGDj-UR}yXQ5!hWbz^j}RwAY~ zzd(~mMEt#tEp&Xn*uF7_n`u|jZA8zPi7oEbi*HkM9o~_fe5aZcZ~S0|?<@A*UZwbP zJ9ou2WmfN3bTruc1eP$I)iW{2uEiQU@Y!`?XExi$kALKxtY2%`TLXijng5RI{u<2x z)x!N3)->GzWdv_aY1D=Oen~3@U3yp~C&$m+%1yz)pZ@RB;f1~~C1?RYyo3!NL%<e7=8v=ieKjg*xG|eE28Is0{r|#GjeHbYBc$ z>hfNm(bJL$kTYzRO^%SyFaD4O|Noc~tJzmp+5a3vhUb5WwK0f__lvl%GM)jrxJKb^ z71>`Nbr7M99??R~jyJ2WTZAv4GvlNhJQLuZj}uFZEZdYNaKSp1g`IMcbmm7^F8ecF zi}l8_A99kEbh(o(Y2~NZ+{rrnAYY8y*FJ^EQi!E%;a5~dD|5?7MU`0$Xpz8U(`(DR zmP?4YECWpPbB1~<3y_%)R4sW!W4VmTC)uuss! zSTy&NwU)5c6sow*XFdHode{_L9YMlPc_&$u+9!1M8vclei`9hE0O^Zjzc7RXB+91b zf)pJD;Y(?euxwaJTkO7L>w=R&vYKvfp{0C5Vl$&FL5T*UcV|Y{@|K(K5oY!~35AJmpB{GI?zW-nz8-3Y-G#e<~b^0<5jPfzufWG>!lTivowtv7U{rHUgxZaR1D* zz3jpI7y5adSKAfcz~)rjM9XaU&W4e~i8_#3Ewio`v5r zEG3I?$fOfA@N=Ng^Ga8R*?r~#3BKF55_xjha$b1d#LOL9Gn}jl15?TT{P0%hzL}m` zZd~^Yc#os;b4VqUNXzXB!pg>iIS~E>6(|D9t$33PZJ5N|Ku$_x4xbC_sH~*^)tVdH2*ZtjQPFsI~$lwlphY zx7l&N&#zajHpfJ^z?EQBj^pF`XdsWqaPIwrRu>~K?)#P$6&+T^Lz*-uweu+r-c*1h zh)!5aos6BK54veyj2u(brd1YGXBW6oh^DW{qUruM@2aEmuc`IM#$!`&YRPp-S+}pk z^B4v~jovxk+3m1Z61;U9s2!fJBpjHp1rDDW^0rKT7#r2xDi_|BYvPa!>-PNb|s4I-RtwP36tdBurBBXR~s~pH$u3iosgp|15 zjF>b79^qHYE3n5p?{YjTX4cbbcA+)J7IAa#rm0Z5{8a!o&&J*v^PfIy&dX;cCWi}2 zEFyn37Pa~>SOybaIG6xGv)`3P4tl(FEL`Q&Mxr`T9$ZtL)F^oAGjNY@_bwXmI??&*axVvombeh44i-_Cc_Dv_rpR6S~ z(2@^3`#azLPfRUEBor_fT!`@KT3)BUC@2Ub;`yez=%&8>0lC$~lz7?8NOeCCB74QB zKhF{fOMcXwR!(JPeJB}1eau!(4j=t*73pWYL0*Jp+>3zoH)oIg`Se_Hd&VMd3OuXL zwU7*EcD(8zzTVpM0*&U!uB2|W1spo_7ahc5T#O>>O~bo|O6&Qv?+HY(RX|e^MO8j- zX*<))K=+X{^jn4PHx-&W)g4!_C2J|D8&*#_6}8*_ zlkHq-M0+%vr+|UrzcDWII4S}y*eGo&qurK4l^33JZ?M7&CMsIoD98Kvpij zOneMF>TStVB>s(MfXZEV(~JVFuQKOEKj^?&C+k^2v6cPQA*CcHySN<2_tYplvm`33 zWYLRTJ*WXB1LKkRHGTDJ;ddpqo36Ort~mn@zhJNfIq`KR~y zaif{RZ+Gb1+ZVn`1`X^|ZY+}m{ZWEsSDLZtEVcb}k|=&28s6SSC^^-BwCijPZcGY~ z`%o`qV)D4z4Sh10!6w!ssw8eq;B(gcEMQMz+z!f%25IC8>ObbVVlI3So&oPDTZs^& z$%~QM2Ap#DY!X(Mp+~LchS7I=IRiMe<@`}p*3^FQ7My1InDL+A9+sIzuszF;*xO%H zkS#m@g*(Gq&es}|dTr7tZl(RC!@J`I_Gv=WNP3@R|GKMzw1}x<8|O&RN&kiC@1OM_ zb{#40G=OWZJ74{Bylt`#3wMiui_u99vh8|D18;I5#1YNXW|bJUyxuSN2;2)R5OLgL z)b%`{QC6a~zj)L6Q_u=e7(s!s|?SM_EZXOx*bo(<_&1&JPtbBrAr+-L6m8#=vy z3&b5ZmekU}41B&M>61^hB^8%GeV_(pXFe^Lh;y}NX>dx(lTwB>Z}E1G)LjWsVF;kI zROa0GDcrwJ6pH_6c6S#25oDDd`j~%(;NzODxO%VIvt}lx^EOqJI6OKzv^be$qnD14 z4Gr!<;>Apz*aH6;DNoR^U)aCaws!XAONm))zE->O8cjPI@O1P!0 zOh!`(@_l&pmu*VWkLSp>hTxb%RkCqqzMcClYYsw@o3XPYZhJlJRf40DHN(68?)gZ6sA#*_*fL z#EoIzS_q?=E%e-_+H7R8;zQYT-(&iuS2?xEz?GR=`TkM#DHa#zt(o=be2~sr04DFx zFc5j}PCl^Dq08?F=xSeH-w6sx(!GFHE09xi^gA4gu(?{4E+$~dj(#50a)(Mv?S zc36pP`DtR@lu6HJPdc95YqR`0O(e2s-7Azey|8IgI3M}8Xwgw>73P!@qXFIQyNelr zxu|-bED`D8Tp!#K)!cK#RiKZ(7e|qn2bV2>a8eTeA zgAKfut&B~t&wq8foPlzvN)(%ZCltg#Rm?7Rkj+2}MtRJQ?(aPu*X2L`E&f_$Y4J)t zBKA=*{`xT61I&Fn!yIByo{i&M(F)f=YW^J;PP`D%O>eCd=$|+2$6j$#(P&^vDrav8KzfZq{Sty`o2pqOc^-&=~E0}g$y!eABuq1R}V@>-47lMfxs>j&y}snJ6! z++;vXw20*c;TsE>OJH>Lz|7- zHC$9b?mW=lNW4DJDv?7|q;1!ct8HpW36SpruC!sxEbLV|CW$&gT2Z;N_u$_)!AyjTwH?Wzros zz+>xtVv2)mZ{9U~c}HmUV$fAxE1t7G>8|P({P;q+zd+O90shoc*cd_lGw+P+|7!FO zvl^W;)QOSOeah&ncA3>pSScbuZ}MI#1=J0wDh4$lOG5{nFI>=y6k_?DHc;pOm<3E) zK2{?n2-Yh$FbcDQ2+_s4Ow+i1w!|4e&O3IVKuZ(oHq<8ZO`s>d*Tv2ZE_^XtlPfVp zHbdNaH6OW?`k|(iUnO0jmw5`#o2_b*T(OtPTpJ$><(iWlZ}GWXo=i$tEDF*yhyQFGw?t7sAOA4Wxy z5rl49aPWB^>QP|*W_x2o26;z9KxVNIDnv*O-??T49RhPI&@rr!qH0(t z>w>f>Jc>V!Dek$;_mVb+)gs<9+d4XZ4H;4JJvW2t-XYJoitN;VfmmDvN)At+H!PZt z#w7$iaPAdVIRHIOD5_dNqaNJ*qe8!=%2zq48zgQ^RYdmMr+!Jw_peByE3?*P4h+g_ zVYg@;sqe|!vU5OfoR9sil#3@NCB>%ARf*p(YK;VwPjCxVwZC3}L&LBMkyFV;m^g1K z*>^qMV1iz-^R1@lX;0=3f4C3f=PM>oYbNI<1j>-CCcwh-Eh$jlKYCU<&oMf0Jsb>Z zPh!#i@(-X7I2WEcjm;|V4LQjbkqtNVEio9TYZY%I8#R0z>(-Ko(pqOmTxK0{kb5S> zhI;m-M`*7gn?bOnhU2sHVT%m`Ue=F8I^q_DQeIR@i~GBN2MzDdO#_yCEv{x{7X`AC ze3vAF`f5-d;S+4~>M-n4Bj~3QwD#Db|LX)TIpcxch+}Mxs0Re)oQ5a4s5%``ol>f9 zwRNUHVu0p;RHD1FyCFUg$FM9hBm{3dgI|ncljKMI)?1{RsC&I^HtqfVyeeXO_#lyp z{P5|ch5=oZ@Pf`U^ErATnXs7x>)c$Cb(H)A4L5<z z{+?@5=AhD}@)j3Pw&Up^u~-}THy7klyQ;zY^o}kaUHNIO7p|sl}_k)F)k#+HJO^CEb_qds}EqX&y2e`%%9MmLa2b<*xuozI++yVb)B| ziuEW1kQy@Z>>GR+_(*{7(Tr^Xv=k-5KvxfhPI_4!UEWz`V$+j$Rg8C(X|yJq>Ia`Z z7x<+kx)*H-{iw^u&SH?*d-Ofw&$iHv>ik?NPw7<0>Tq&f(sQPDqvbowLjC)J?(rN> zaPIS=Bzz%GRy*ZG$lRm}g9OmsO>kaNfwFqDY}0}AR#BPAbb8@MuV;!u{|Q-_Z-F(@lb-!THv-&191-kFV0gn{C>&aFhu3;+EdBgCQ@eKsOq4X7w->_e1#@K?qsP7z zhv@Wv7r^90YBmO6&X3=r(2ybDG}rdKu%{RlQ3cap!yFmoY&YtTF4n@vO9n$5+^aK# zl|)Db9S;E3I)s`n++IJKE;ApXQCFL`0)Q-bZHvx`>2`?k?0PCY3+74)40?_VTr}99 zT&^q^11!yFTZZ_3Rx?=)VrI=A_$I+T(YWRML!Alvh2I=55R>BU2#LHQpRVde(_pPs zG@}4DOJH=HM>jOsUAsm1e^Qnsn#OgL^ zJ;9(z>gN0Pl+}>sn{xw2EnUB{7OHW(#`E>YuZ(^IG9`k3 zmQiH%;(!+vI9sNOG<*WNMZ)HggTZ}zb)SvW4;k81(o|j;y*mDkTf3-Iwu+;|@Z)D{ zUTRPN3xsc%vCn2N(tkc0d#=K2 z#>HDv^jP_~Fng&$gc!Vs4u0i@s`14n7fv(6@~AL|V9Ij->sSosQWzvIyn8@i7}s9k z7?vG>IX*gvYZldLm|_jqFL-9(0Wm{H9$TteVYi@P^lqVf7ZzFlb3%o7Ap^7*`jgy` zF+A%}5!T62>8qrh;}{kU6THS$;O&z3pU_x<*m`Hv<+6X26n542y!Y~j@<+tOy$WBA z`MG9w_z>ExRQ>v3t(FM6QX%M@w4OHaVZep(J>$L#D_Qscpj;3kqRZChNh0?v6VZ0u zg~ls>v}S}_Dtyi))`79eMKgwP7U6wpF>&f z&iH)b6s^5#Bv#q|v7P>3@(fiQM%L#VZys3-D!43d7O7J48_P1>WE&h{y}sfXrm~V! zqNG?{kvH^mMVvE!5X9Is7xC8ur7tJ@etRpZNb4;O>u!bn$&{*=pkVa%>O-zL{7^0u z(4aA`MOOu#2{g;paOP4gcyO>1$80GZ~vtLPW%I$vlC+g-=zxfRGE2!L0m{i_=>J{d4tf1Jk zPu<_3?#b1DVCFiZLSwI&z6Yq?zeqde9&B_N&$xu?$b+AXbJrh-t5M$;qIP7B0*) z-d=X$cn3$bE=^EmG*dqrF{INO(FBID4OnzECK>)Z@Mj&ZrR-^Z2#@uj|5B?+ls+6k zKEG6YFVJWR%H3GI@3*1%lawev{6I`BxVMla8V9l+pK0;o#}bzpsUQv9(g08xjw`6{5@Ws3m~h zTa&bzkkf zXG?i$Lc%G?kFAv;u04J>FD5MyC-Gs+0iEQG5OIK?`!%{n&@Da12^ZIWJ4pUF5Qa8H zOyq+IO|gpS`EOz6e`MfVeX%*?mVw&&>}48%<>Av?x$t^5%HEa4$mlxzI_frKw>sp# zxcnyW_8neHRc_Me75*(R^c#GhYzm;wO3wd9uVi^%1sAnXxYO-{mnH3No`_Rd3KPhB%o%fdOo4#OM2G5`wetiRc zRNDO1^oCY~pGdh_CDE3Au1LMiDQSTh`z=z(n^>k-vABQyx}~MafU?bZil^rDMyzaD z@aJ`9HJ!~=$Q_fpqxw@1WPo*JSN*WH{<~QHC2+&LMke!ZTy{(3Oo~lxty8mq+8KUd ztNQ9umJ?FDW|xb2WCZthpo=)AhM9QwEw?#ibh@l+@}|TTJba%voPp^gx>V^>LU6%U zNZr&m^ESjCON8rF`C`DCFb$vh#AHt6HLgQFV#HA3rS@2;lmyU=E-He7m4`0n$G}+B z%udy%^PtBTDwNF=~^l3JvI?e6j%>nR4 zNhv(*G`hM%EoJ$d(neRW-wf8$!zI)1g<}vj_Z`!ZLqD)|4R|pv1}|0}8;ShFR-I~8 zawAZUn5#q6a{$&jKiIsE>jzbKdlC|q9dzFC>7M&c1KSRq6 zY?XnJE2bTj=-hf>kpznlX&Nf(-i$Bgo%7uWIC+o?B=Kw1~NlWP|!}&hD>KT!@2<)kVV6O>#_B{%97KGKGSKL-vHW&At+c_-WQF8>50wfh^vTLD6A#!Vo^GoBep`|j-iZ%0!Dk^%20aes38BAd9iC{CMEGp5 zX;IK7Iw!TV4?GcZc6Qz`u3y86m7?!WtHW#=fNOZaAMR6EUJW1!eVqgQvfZ}o`XTe2=g6<6GjZzU z6W-u=F2pT`1IkxSulyWQH zP*&wqaTUE}TjH#0>_Xr>RxOp>$ zSrAS{ho-|xe)Dx5K45^uXiVK@m&f2`&dbL?dWW$vIryQpLuso+S4X3kzGXVIfNH-g z*FLTuiafC?#9?~hOiLO6$^f<{)N$XjvX-3eM`i}pZsK*)I?E(BoxOVB7tDNCDj(Th zF0hW$NNU`)H3}r2zry^X)VTO~Y7s+6o!nERVyc0F*s{X<9hAAIq`Da}!qYp%af(`9 zD-=ZD5mw=0u%aY6Kzthu;CGpHV*GH8rNZ|K-}2fn&g|BI#|V&T%)QhaajdVo7f}OT z5tXxd?e{cX!S$&}PoGkI^mOA_vZ zW%6^qB|j8e-U4Sqh4Cj2Fu_5ZvyM*6FptxhD8=cC<;Cm;{NFuCl+)}&B?pIokP~}W zDYo*==TU@t(I4NduLP?WfC6TrYuoBAvvVX)fX90@JwL5Acq4}h4-4vRKi;C49B~EQ z@Zxgyd5=&`%bb$Jy0H!^E~@Ic$@ZG4e;@4yeXZ%xLV4g)St!!-%zcA;NcMz&v13CNmjX$8_&801z-FO=+Q{F6||hZ@c%hwpwi;9u~fxd4HUJxe9$6!-z`7{7YD za4ml;#(z-gvpB{BcKJxc_C@#XnZ+E~)n*ng8`W|$*Wg};aG@)uxa2e{3Mjam_hp3K z!~hIwp}CV4vJg*p%q*t!&Y~5P2alG{E>mes7965kuP*o09V>Pc?utt3i7Cye@{{OC z6A0|k%w&&G@jkgLv5|cf)9~Q8m()^G;7a0gGj!`gC!Xrz9)9pQ#%y9koigm(HcPc> z43<}03Q4O3*@zNk3=BHe;I5qy=9!aQ@v8_`bEVo~#%9(Pz8BhhtR&*q*=nK<4Xnzc zcX>s>!*Ehy!q@uZkX*-*eFGNr(wERj$GlP1IZ$-IyLr+{iJHj4doT@&c)V2Kdj24n zxar6~=54KVvCLyzbn;44vv37 zI2{$P^NRV24xUq};#D83P^!AJWWdF+oH$1&OJv!`PvOI`kOQ}LbZ4FI_Mjqy>9~gJ zOHY<4lbJMbSEp2}Vr!A5*On<9)3i#eGJS=+mt%yi8^N%7i_vJvd_%Ptv~tjhfuS_ebLwCm4ltj7f_e{y&zA{xXzE zYD7%jAwE+Ec1+lXYKi^QiAY?zsaR)1sV7RG&J{Z+-yz-@SZGSb{ZfGb#Gf5)+kb{; z;^rip`j$uH%KNI%(6UX}@pfps4pPt!!H$(O)TvRI&9Ec#%~SRdtSBzy=;%IXhFeu- zCu4CU)9q@8s+MTp%f0M8Fj@VqSNVk-50+(kFl1%T^ggU#xTmmxWTftUAga|*X~$)O zS4KCB)6;Z@Vy+aZr8=f;zm%&{?xIN?64;r0-EBtLQn*!ixozPu%Mwb1F_kccAO_j7 z$9DuajNeEzgtxz@lV|ri5qqAq!}6E)^iPDcAX^>pSd=B|@K50olTTO1iUf>>mSYL0plDAgar3`DGQ6o3qJ$aO(~ zVuRODOTRL(@>@-oy!20{&Qc z1986Hk3YRZew_pZJo6;tVXMmaztzz?9Lia^zKx-@Wd$i4YN8pQ77(uK!Gh*rJoYyT zm55hAZpt!(NAeW^x$fxy$>E<&`ad3M_eF0w?8F%R!%+WEP5b{OuM+m_wJGK!aex1{ zZ+?~kP3tSVv_IC*|MQ;zk&!Yyus&6bYc7dC_y5A_e?0Q1vEr|1ec8f zF2D;J8ooodf(76@A4`pt{*~+aH;LNl{nCp1UYi{Jn}>dpDgRm!n>Vltq$k*xqvckt z%J;Hh{PaSkF2!m0mKsd`Z~6XnLbhP1+!B1*RH6L$Gl2~U?xHNs1r{iOz|+4^PBZ0K z_uwptH~%A-`ESb*p9zKxLs#%1#ed)F>t8GB4^`LRL;Y*E>i>tutqo9_VtZ$NeX${Gk^93-L^E@OE)j7#Mxr@l@~1&H9r) z(N|C0aJ5`aBu|FQ066eKZvjQ*Zl1>{m!n*2qps+G+c1A>azQAp8t24SXb$B(sdnwC z#in)_o3EwKly8E`bmagNWTw_%8_($ecE-owfRim;h8uLK?X{cx(qLQ$wGG1NWW8V* zeInElOhr+ntlntFm4~>l$>6g`*sdr5nB6ub7IKKQ@TD^k^l8S_sr-`C0k+0$4t%5>D4DdVtZ zw*&6HWwJSy@dOXE%~UJRlG55PoX{`Q2R}WZVdISw^dvw*jdHrMrT9)o;5cEv@q@fu zX$xE5Zu5HsT(H~vlZF3N^Zc{d_|Pg4@je_$erOP0JZ{%olM4n9sy_?BsyWN}7k`1T zl9EM)@14?cnRzsNNkXG@r*hP9PmAicMdv@e#a6wm$Z>D;QoPeBV&7b4gPF4J+oOO! zE?h9=KAZ3!Xb%B&%I*GfrTZU7l>e0@jbDpi`>)e)#;E&Q4sl#y%AfYA(q7W7x59tS znmVaSt*^c_L9!k~tGzL^^dTAzX$`J7YFgrjov()Ob;tDP<@P((fkI-sh`Sx;!FMq^ z(+asSZI0~%J{YaF16;&JSCW-g?8F_Bf77uNUCf|9%C2ZVzg1 z`t+J1*0_zaOIZmU8JL8~#dMY2wvI^-Y`>2+YzFDS>S}X`5o#rRcz%$1Dm+nNKDVW( zxEzM}?wtxA70fz`xQSwkSam&6A5JX|9lw1(D(_4bTkYL= zfz$1QH;d@FHgv^odmc`9xmGg*(U4Kn0vDq^QX%I?->vV!h}k}v7bY;DXuWfKb2dAc zE6DKg5b#H=>w+!(SWZKO%IMW%-)QuKjQ|5FaKJqCvwMlaw>cezjK@mnmP|U@A{~(@?Z<6$ z(eV~TrXE6+qwCjs&S$ePu=s^?QX_w48vcokKjpJb(}!w8U4@~)XYs=M`P#Di44#yX zu_mst5n-9^qy6dJR0wC&I8!utv$^y3?;!tqh#{O{?p*n zUG+&qW0D}wNEA(s-tu;yJajY?QeP9)940E0bWZZw6ecR+tx{XPJit_z7hfu)vg(>YtP5}CG zEq1`3l7X;6U`Bbiduv(fPlf+CCV5>Fb^WR#z8`(essV??d(!7XO-1i%&bxQRT=W~j z@H3#zQ^4vm|FfoL9jSw)hNH6}V!u5FsZ#o+@`6CaiwrM7g6C$U))-nrW!3eI0NHf| zsPdryskk^IiJ%+4UW%Xbb)h-l-MEm7LD}Vt2~cLey6E)jMeZgdR`8QJQ5Ck&SIz9Y zx>?-`jDpYbbu?Ddwl>|f1Jb_t)M(H06Dl3DQ%>Cz+}Op6JnHJj??Yz3Y965ke-A<; zCNX~trs&AFC7#y+e9o?Myc|cDWa-5lT({uk5ieRzKxsg(zRy5bnU5_ykSIj2|MrE& z3{_BrNuKUDJq;ByI_ESY)y(w1THCU#Or=F!fO54iSdK$`Gx4_p&{?08c9#^auY9v@I(!krlX)sG zU!qelg{R80d+e4iJ?Ipxzvb8)$dmT1hw+OjjR%j(b#u|8;HSL`j7Akfu7I>mcihV1 zWz!_vrK0g*?jJSny_suiST}Ey)KTY^1Jh)hu4K3l`AetB+13m#Uhe3+ShFnI{&fJ0&f( zwJ-E{h<_i1V9tyl{s;c}-GTlb!-xc6VB$(U6 z-@aA+Pb~h;bk_X@n{oSoy?&erLaQ`%|8#maR6Kk*gcDHe6BnCP zoLBN(NnekVWztlFo|4Ny8ta9DUIWF6Al}xrRlgDQdKBPPHyUd=?}2Bs9gykgt|x-f zHw*tX*NRzr8RYMypJ6{Kuzp22)?u?GIjaFcFHMb()J`yx;{}52H7iK)YNqmp^L&U< zcg$SSV+2;(Q*dyAivL><|)V=A0sl&G_e?aGFaDPAG%O(!k zphg>J#N$d&p+<+uylBCfH(`@MTlUs>_!#x8T)J;zUFw#@@zH?=*LCT1{QyK5YY=j) zAjPIFL>ellZZE2{ehedgebxr$%XiQc*=x2LERAQn3n@RzRkjL85Rw-Ow9G$Rx9qKO z`ttObi~WRuzzzZ>V^Uj?u7(WUi}tPN!ytpLgT3PMf>e@TZ$Ezc;vpJ(K^Y@ zhcvx6kF4EfmR48&AikC9W9bhFJB^H)X(v7L_*C6GT3ha+f-m`A^*ZMgED_8 z<(|Q6bbAJBz);MSDyd@virw{*sSbNze$1TXnb7nX^Y3bwP)a}Q#^QBjDqj_XUddV% zDfe*>9@%1>4UQpcL2Aq)Dr8}S5%HHZXdpr=wNt{B$Wi)-3w--Yx4}{vjV!T-!sjE| zxOOn8`U(AJGdS4DC?E87QQj|{z^l({fCmMO1Y*Ts`&MaQiXk!hO8?OcKJ9&dR%V`K zfIc?a{2jt&2W;>{O{luk5i9K%Uq15J*(j)4?D!#^uptf*3R->a$vWl1-0l?blECPN zj{9bRhDK78ccF+^{3S+WS?Aci(S%)d?X8O7tj$;`r@W)-8Phvcp5YaH7@&}-vG~TW zJNXUsIRnjZzchmZ+wfa5v^0~*S}VW3fCbF`yEDGFY4%gj_39(;J8B#cXL4~qM5 z>o*oS&^2WoKQaUV0VI-0!Ouf7q{viT!hc~6=?fHpLUd4B`fJcu z<2+?jaf=)@{P;Tc7);g^41C-T*rINr2F!!E@fP?#+eeM{IHVG>kag0E%0WS@n+3>8 zJNl}^0~at7)x?lN>pfI)I8U&(4`j3ix1S%bk3L%Kla6>aO@W5f)8_}-RX)azDRbaW zDEL4ZBRVwgmr=?5wMX@m>Jpdi*%UfJu$+zu0g=J+=?zAWcLg?JXwY*j^PwuV-xVm! z=|^t2Xv)~T)62s`5D;Bi=c$P%%45;YSmTZZZ@Pt2&V3`_B^k0P*ByZP>a|X*wR72I z1b>=J@9D?Bac%55c1d(Rixm7#*Ye(jk58J=}1JsgI*>;!^Rf+1d?R^E4Z9zB8sh zdo3zY3}VK^wxgm3Yz*3ZKKoJTvkGPX6HX7Z8Dn-z+4#`~S05-?^p^~lEIE%i<{~Wp zmUeGc8u)~R!+(3;tbkXsH(C1z%tZ=<87dj~(J9BEvp1s7Cru@dd@!xOD zRf_<{9UXfDqPrs^`$a3?!^@g9s&#A_Wna$m)^h6i7ThZgaw%*ETY-pF39KUCZLcW_ zJYTsFo^%!*8zB6zxR|7M2Qs@0}I@ ztzP2iacePlYM;g{g=YD(hR%hA&+6Y>mJI9bYm9TGKcEl9>}~8upZ2PM;C*;|qs$q; zPA|IZ>8!h)LZXv~sk7mlkK#X99+e7;kq58XTeYOR>yQy2nxZfU+_Z#fDdus<%Dk3~ zuBx7;<1t;K>>GL(btB-a`Z)JxKMam}r?HQ@DFZ*W?dAQ=s?tXC z9V>YGAd_sPP^UPGw-?DX)%gVDSd)3BBfk9E6>h(BxC3nY6E5&F9&EJt-wLba<9c~6 zhoK@qy{!eN$cMj6xw^{EsRGnp&Bw+~~92Z>1HYc#VQZ z2eBo5#(>XsucOsX+|fJQjMqr+Y+xNrjmLi5DNQbB3ROc?T|go zc20kL_(N~^#`RFBZDSTnY|Acl$jBY8QyjJzjW%X>hW`7iv5~1R*ck+W%)G{v9$MDakHwQZw3x0vgpGZCnlmoCU(k4@IP_9rxL;70x!!z7f%J$G1rPBLc-LX<=HYuM; zEe10-7=(!q`9l0xyp5#~(Gs8R$3p=(<76UvC}(Q-9d^jp`2N#cb5RwNU~;y?s~Mub zXA@?H2b@9l)fCkNo6trQ%P^_yn3vzaFGT#;P$V)vHIRS_;$+0M+9j>cFib>m%mYcu zhB=RpYC7ge_u}P)lZ%KMV_tR7=k}8Y?!86}GN61hAdm$%&L9zr6@MHp6l&8xB#@Z=W@g@g zhMdKOOw#1CEkuDepY$XwJ)26fvvZ6xN~i5v3lDDHA7wQYVAU;=09Tj_eByArqv$zi zbfqB_mf1lb_Hcg_70VpD*HAYUM4~cM6O%fNa}3ybzgb~)Ea&bC*Sb(SyOZi?P^7SL z*?;7Q7Ft;^V14XHjvfuK0qEbAj;eH zm_?)Xo};DI-jKN5azphrsSF~pmJPPC!F`B+1^W)xr@zQxWx7w1gCOETvOKK$D=3@T zBKpmn^nG}^WBFQc(D>1iLXA`ADyE%AUsGT9BrcL@rP|t$Kb7DGmC}}+NQ|R3tx8UK@2y2>(yu7CBJVK%eS?v zCU2m)hUvB#8*uJ>5FdKiDEYYyXio-ifjtdR`-8ZUAxcG+zAUW#^$5ojihWga-C+IGqA6hTUk&JMf8(T;nIL_#UBiGiymS zg~`Q8x62w;(dq1!1d^xG2@9vuz?;Id`hL5vOR>X84TkGtph6=-x_S&2PqGmV|F!;$ z2u#Os;=+fRVw|V3Fqt_rq~!bPf^_us3w}__D>m~9hkcq5c3Dw#7s5w+c#vX#xRQjg z03`;Ere&FFzyzzyUqPSGTOy>>fiK8NUqpSNNYVx8d{!_vWML;w5&+!u2I)U@5*LF+ zXZY!vGaw>Bs(1jzL9NF)e-BjXK~JZJmV~OT&&*#{X58?~;7UklqSwx5H!HsA$vM4C z)@;>ws%GMaxryh0E%VuYaRRLcN*|t4>|-G5^GudQogMYpvb1o-QGS?o(1n@{&ijqD z2FFObrmru?vTzb!Ns8xa#{617fm!WeHN}8#Py{oN${#=j%eExN7JSA}=ZI9z-i0E_ zPN)cNpUs=UKF(a!m*}gTtISJj%Ljp(Wecm@6&ry?2R>Dk?1JN7j-BTnlJ#0mT-u4* zuZ@VhA3)5L$^|{{6TBgvly}Qn74glc)OiEKVyn&iQ~87VoYZ}Y@kShiJ*SU#8aJe!ORFy#wm z9;(*czC!{pWCH`A7NRO+J4mY1ayECK!JAN<9W%Y`gA)t%qzZ4LToxrUXh|iBdd={l zz82WQtd%Hd8{xdL0Gj>f9!LmR^T@~oHVc50FkPP5@wA}FX0}`X$*4wtYjhzHt-HtV z=#7%=I>qp)b6Op(O-w&l{*(4X;}ZBNXR-{G{bdIZ{xnuXp0bCswY5H}w2vx8vTpo} zPE23Qn00Io>Du;CHN&Ua(yzT!pJjNDKaJ{M26A-$6zsgJ9*LQ5rsX+!jZfITQ9YF8 z4Qbc87lVW0l=o=y6MBjh3Yp>6-76kwWcP@c%1*j1#op&a>(jmOT7kzcRP9+Y{Dm zG04JK{^bGfT%j2cJLsY{n5%MpV^A})4nx;;gVo_Tbcre3Q*Zp9P?;dQg1!Z#pz|zd zK~FE^!evcgHM=gGSlt_M$D8uxrgE`CE;;xD9!yrzAMeW`_ZP3z8V}nl(DDU9LS#XV zYUJfar*TpaMjdb;W}mXXr1Ou6?ml%R_Y2<0aX=m^4>Ie8Sz=u^uz&7Xs6}VYJ_85v zap*7{_l6;@+}hVg#*Dm=h$IJ3_7R}$bmEPY)GE*&8JV#O>Bj~6PYmTLqvo2Hk2e|F zdMJ~|o`m^)ya=VK29YTo1h9>ZgA`qgtK>^}5EEgdD)tA{P5mLu$nwnc>J4Oz{t)Gs=9)U<6kN54+Pi}vg#og>a{c}=;r zb61jskwPNrW(l~-uphqfPxTHfs!3)ifAcT%+-L}5+^Im32pDHGyk1w><1zLafvFvzlEEuYjG~?Le(0L$Yjx(say7jd! zR~7D|Z7}WLWlrkd_zdTYe842g;SSUOs@zz&<#;ZM;FlxT@0Ia4fb_JT5NGmW;Jg#kSTHR@H=|fG#+SaSzmr?fjyA}czZGe(r8KQCNR@|8q9aIWwX8l z=DjO7*5Cg%2yd51v7zONbsFn=D)96o6I=ofSk9~DT9QoEoW7~i;eeEw1aLk#urp>y z4A6s$7R#CvQL7na;E+z`NlfN{(_qL#?Y=<@-Q9`E#6JDxx?dy@iQ}~TJYm>g&CWdV zHSf7EKXWA&dpH-k>yx4&?Dh}tK{){H#08Uar)S1*WDD=LnUo&IG)BbO&^a&mRnPCq z^ih{=hqK0ZEEVghq?lJG(N>_ME+Rl%ItIf6b6E%0-a=bTj9Ud%Q48fX(-~#Vw7bu0xJ~e6P8$fwm6FRRaaV+Mwr2{dG_IWD>Y)z(^6sV1CO(GA>3Qi z-TC`CUU^FCg-7iO_OT8dmRWfD)W_5a`tZgEgHig^``=YAO_A1)`Za7S8y{#$VGb_( zscQL7KbyJzc=Eoli@|SxBxi;9LTILPW zR`}lJUNpTu3p#+@)?vb>XB60y@_MKmntme@B;tKlyQyk&KOe>^r1P-jYlNj`8kUQC zAw(x$_1zIYZ>|oD=B0ONCXiwV&_In^D|R8-PB#hV2iLsWijv5+Ud7PQ)zh0pv( zWcO~~gb`Xn8LY0>h{0jpa^0z-zU4}DKAG2_U1`C}=jl2+JvMJs4rO@>G={LQ_B-m=9ClObX|wtF3MNaTmkdJe1k)TW3hcCMAt{DjT!S1Yte9$hX32RCiKx*M z#JGxWHkNX(LVwXPOp0Lk)H=t@HP2n}ZFiW!7e4ln$vI`8hE|nxhNpj?tz4&P+}qGh z2Ehqa4+Mv)>sTY2p33_s`9;Z=lc=;zcx{JnNL(MC)vQjN)HbL2py9rzPN z56m%i_ayE8IlKJ*LJ>dRD>>Ekfd7J{Us$X?-!0%;e{lBrfTv>JOS}Ul$XFjxiQrRO zSfp-`O)*xNY)%a-KHn0O3+h?Ap+JmKow|+4n|Co$Iuf}As$}&N)4e~CkVAB*dNJG8 z)Rp5u=YoxBe}x=Sz|B4;IL*{zP&hD!Y9{DFtD=?cg^sEtxmLRzz?b zm8xT#jh1P!N;_M(w4~(a1jnOEHvQY`6$|f>F{y~*W;R13%6V2z{%MisJ&QUp#jh7G zn^YqpMJ+5f*==)5NhjD*Sq{p7;8ND%x<<+i(Y0rmK0V@)L-YW>AVN>u622=>n%t*{ zY*Bx@c>R9zj5wlIAGWuc2LKRPQqO8VW9a7@-SfoD@7f@I4Efj)PIW{ULW5D-B12*x zwcBE5;G6j1sv{J|t-=qREw!x{%UiVY6C@fhXk|Z|M)DF=W!9y9ociTZqoh#X?>@Y3 zx$i>Xd*)j2&LGy^F`35wtD)>^1?@3)9GyTMP?+?$;fz4BVdUT&Ay|tr$Vy&MAv`w+ z#J-U+BIi`s&V6vd3cZa;xvOFfxg&=M$stdq`=y~K?_LTq-}#!m590ogA0j2&nY-8( znVD9|#^s>!qNg9zUEUbXjUp4f@`T0f1dqkUruF0|*o~+jFPTaPY%+3wLCcyS`_LF#gl>7eto12w>ooPvbm2A+HV)D4PtR)8R6Yr>@#-n2nV>n1e_OBs2S+=!@f zA4OWT9#5=Xy%@kr)K@>q8h=k_^bD8cc~0`7^@au~tyEa3mQ+Wjth4gSgj7tyX@u%i z$D=jz9KXHC384+dw4p7Db&p6#0^Mb^@Lkf<1eMpd89WCyzZHXvS&(xj5 z15Rq@>?5|?M_Oo~j&yqZY!2tow!nMYK?Sj%fWx}1RKfZ+=rsxPw!2U5@{2;WP1lYX z%v^h<<-!j^UnGk-41kBE{37~jI)NXq`+0WMB8o9R>1WkGsv?jcoqAg9>eMD?T#-!Z z#zRlc!Zl&XYxvRTmFp#F?#fvzWbRs<7zASti^z@IdShiy4cJg1U8wF7!2+oXdD3rf zQxNjKQRrveKDLg$t&&Z!P0LY^xxpjJJ1!p?M~~m>{!u;>i)xn|WbP}T$RfIm-gxr0 zxKo?iF5VL}QHKjMbqw^@-6QCwBV$n=0+9X6b@r5K7{K&YIT)LI?Wy@O5bE;|Ce~0m zA&UBMI^i9+k9NO|(Zl5ILld@t;X!pv>0*w@rKF(B;$_TQm>fMG#prz$aFol>j>Gwo zux!i#=XdZcO81)PBy?VG>?h(|pPWyI@#xu^n(^BztD&jUEVhf+uBNOM^~JRE=EzsH z=Ls$gYU&Oppjya2;V*J)_`{ElMka4Zs1@xZwdsn59oy{oMtTy0?WX;;Y8f}zQGuj- zyJq5PNu@ONj(~Fw4~Z0Fu`I04lupGEHBKF#O(kSQz9-tq=v|^*U6xJOlO>*`$}PER zWbuztbkfR(DT6Z(W6&v_)@&^K`>5Qj(0DJP%SEBqX;&xeTyNxRJ{x}`cQ3`m#M z<(8qV78~lk@)lWClDRhhMHp7{!*d6<=O33Pa2K=t2ob*p0E}ZI_;~f@*eJ?PMZWQIj5AX+?=H)1mGUw`gj$<)2SW8 zFpWaYXE33EreS$gs64c+7s57x(A@9yOD6zJhegMHMZ^58r-|m0ScWw18`~Np+B>6^ zvXHeI5l0Y{tOl)p3tPQpQIDb0dmV z^x`{nVI=X#ReW*AHe?yMuY9;xeRc22`_+hWoMv_nBN0V6TKbc+9Pji+_vT|l_(2jV zz24;T^lcixZfo#<#PHZ_G#wp>G&5ON13UZb3SkrXlr}B{G&*+)^#mrVv|<-c7aB1M z#+m^R=V~%yGH<{~X5lNvdqA$v5q7P@(MyxB4~(_=Zii?O_tG+61`$D2DGkILW!0#Z z^%cUomo-71liNpA6Hsk3_Q;%Py%+SRg|usjjFC*hfoi84ak|> zXd-^Omk?JP6vD?DqV<57lk3*Z6coj2$fQv{we#9VFJDCOG#>1QyVJe?? zO2=zadTjA7EO@vNX-nVVkl}fo##%Se1dE@=>WOFMd!*OyQcI_mhonB*3~+*CP&l@( z#`P+n-;bDd^U5TB3vnd@6xHOpAS+(T+)Eu(vx3kbH>JkLV-mKaEX4u|A2032<#^!n zy*<%t)J%e{x# zK(s_AH{qgE2aCG3Dn{6@SXRXk{9w=Lnj2%u|GFIrXjDp8Y5q+H6Zw4mB_TtCrg>NMSVj~E#n3@xwv8K?o*sfs1mXqxtsNo}QK-x9`@n2f)?3HuWJFsMJ32baEJ z!*OnF;7mnSA&?~H*vAhd-%|ki>GWDtd83SR%l+&`Z~Csyc0=Dd>oRfyz)k~G@450$ z1(N8c5?e)uoeI>z^Zd1MSwH~QOYh?oxJP5LKtz7->uo9u*MOW2t@MSrU994$7^)Wv zzY=5ryl@{!*C#yGZB@`Tq1Kn@F(s3%hZyG!wj899g2Z{xkLc_lxucAAEzF2<=sY#( z68zCF4du~4gGV-^m7_Qqi4324jAV4}%(y7_d0c33ta^7wncQ$xsGfP&Y;h?@TO^%A zs5qQMB5c1{#%_Yo`Foqv_TBct0-+{BXpN0=UP|A3Z11%1K(@T}x$@a@61(uWs^L7P zQEB=VvCU!B-gzO=Fc3#!-u^*K7$T;1+cqr%}f)*Vj zoYyUFx`$OFi$OAC8RtD!M`=4nI`wE`t2`v^T>YLK#D+W=4Egb^!3XF=p4N9^7a_EQ z&%o#J1G(g_CVs@NIcWL(w^S@K1_aYXxoa1)iUWwYy<4|MJf{a~`vSiGT9-(&IvzeN zNzMwgM6lS-yn%_hail|$Frhz4!8Io1fzN~n#`$v_^=bbSz?Q?kEQ*qxumpGTgwTVl_2GLtWlL?oRXf@b9k z$t5yKOtXk?a!x_9S~N$HpW>0LLmj{zZLPZKWkCPjO|Ah#Hc`XW6Ycp*wzT@HDt?95 zVz?>u{0C%FaE=rv@mP)W$z;sT3V6I|(P>}E zH%q@sxq^1J9e7I+kqX#zb0T!?8AgC12PxOS*anZJ?x9BFW^F;Aw<&SNu0S z54(sC?{C7%_mfd-c1o7_s_v!!dklRCc5f50aZOxGyk*!)mt!2_JjGU*%hPuIlO@Cx zYx_SHVww4FdE6eTk}#>M~Z6k4zIA$akZx-unEoh_MHQOJSo1 zeTHnlcGKUFNu!f|?$8qDzpnWkv}yim!Hp$_v+BYIGKzk`$7p_ky^pe2S!jS1Kl#Hi zqIqs+{@W-9V$8r*0vRl6m-po0*=p$;ikibLtr=3=4x^PchQ51P6!LNaDD>tcYa6v( zZ0V7Tf(zXi8=KF=8S}n-iDoBDM2CFYK27nWzFsR_O0j6$T1#zA=CtZ3_LPRe^3SEsTHb6mz$Fr$)noUQn<_dFwaCJhS=lU_Lpj!( zsOdlNO5XWz>Ii$IxM+J|)q1Z6msHD%X)bbxw0>nrx$=^`?0Wf$;|bHcZB(Uu+k}17 zE$niSU3Ne6jO-%z7q_8{1m^o3St(dL_tSMS?e#&ZM4rFhSbz;g@4V4W(HFhG=}M5? zJaB>-Jh*Rw+7#}O-kRrEbLA*}Wf1p_lU-i!lg~PrSCP*fHZ)jC2+UFSYtUz{irD8o zq)xD}_mVP%%qLcTRYQmY1^pqtJ6AjnPUl>9V9j$CXVxeI@&!58E*CEVe--`Ey@C2= z!&g-0S8tOc`kaV>R~KA2FXOq_Yq=g!SO-#+gSl)=-^3brU;H)c#|jb3#`)lnIKEPe z{>Wi*L(bqe(3}D19+WMAT~*wQaYRRW8EI-0tMULFk<4VEWT=V$%v6%DFK#VP1m*Kw z!umK>j-;@gy&F{Ubq+{bdj$qW*)EKNgh3M?nQocA2$4k<&qU-(0Q>n z_LKb|JwcRc`i@sVXE`JgqGipU2lHZa6{eHpH|+hmz=Npwmx#mLBK$t{&7b6r3GUYd zng54eFTevr3ud*a_2N1#07HNBiwXC!%YRcMcKHiVzFGUo&7LVC7iF^LcD%YTtwUWU z1wqH)%)Q0;7whm_L;DXUZ3fc*Wv|E&6e;LWe>LsD2vhlx0>l|bA24` zactNHk2U_8iZHoP2#NHHqR+Bx&+F{;R#R%@)(0UnXb(xLzX*!I9MeBk)jpX-za~4r zb9eoZQvJTa?+n50T5UU0LCiq~nf7A^@{;$ma%4K^1CDIgq{3pHZ!x@?o0j>K~WJKRr-%)>= z6S(IHx!<1f?|te*p{Uk={W4#;d)}azoh9@yCH?QE{@=gvc_^^!>B0Y9P8n_}ZXnfD@IWD@G(spr-d~-b0CyX8UxaZWXp$y z#{*=PhTXcxt~z)(@*~pLL|~Z<;r7b+VUd*U1TH=@&1*MQ9!hkSHSR~p{VcQzN2YB) zA4ko8+%Nn5A6yTD-z42VzxAp7hWg)5q5VC$Z!~6#)c@{BfBSPB20Q>fg5+(L{4>b^ zKNGZi+YPXlt|uEWhg#IwumpZ?3HdF#ZF7?w6W3Km)^){JPLB@6Higi3V5`f}cuzr4 ziA^1Vy-#!?@Lza@A{wU$-B~e?uPnfZxkJ9nLI>k>95tg^Z@=xe0l4omy@L&~D5_nF zFH1lh%=2I}#rgyhlZZg<0Tz;W1-v?n(z{|#L<3U3?rH9kz_Q$HPEg+^^Ko6fMY+gIlGu18FPygA>)!Ha*Z~O)G_PK#l)D|oiaj=>)?FwH zUJz9;THambVX5!EC(-cB$Aqy;=G$2-5%WF698BYD_5d$;YqA*3_NUc>uaR7@#{%J* zwgP#W2czVgm%PKnbMbm`cZ!-`E=Ais8~EL(wXboH-Sr5g#{GT{J!5Z4!F{?jy}&)# zd1ERv6fCzjF0^A>R7pZBEW4l|0&rqSp?4iTByk(wy)|})1r|5b$_dI@3cF-NJAvo! zBd4FCWPOay^MsD}e%cjZ`Sh+n&|R4L7xjI=9}n0qS`lx(!|T4)xAeyzYd`UaS+ZDqNhU{&>;6#IzAt9Lb#f(u*@O=>_FC6J0ua zd+*J{Zs9k3d+wYb!Gk~@Ei3j=GGzD;6LbOff`-Fxs25>#D`nN;68g?nsABropYKxH zn%e2aU1U2C@oqcM>p3S^Z+gB^oW2z)o3Z5;tS4Tdm>duGYu7=*X40pyf9?hFTc#Wl zK6wy5LVb?I$yK^`8}1Z5>&bc3ZEd65MPa&xjs;62d=ur}-f*->MExxR&~{UJ%O;eh+|b&uFjuSpMSt!7zxIyuDdSY^MB7 zVZmHHt`XL;wfyRrg0BY?RF-6NJ~1QLzwA4mo)9`kp#IMJF-G-BU3Zhv+oy#3H4VH@ zKpyeMU`visOARhiu-ArVj+D1`JIOzh-+M~FU$fIJo053viqYUWwii>Ur#!6&6?qw7fy>O#gu;;F({>Gqq;V zHVQPdl*<+i2OFS?FVI>1b(W0JaOulx$V~XDUvL#@y$7TQnLW?RKT?uHw&Iz^rEN_j z#||WZWz`qcS@eO;s@YTOH!t|;X%x!s!U1!J|1c`Q>)zG5Z7HU-WBj2okox23MY5ZC z946)MTi%Za73YIxXdpK>jkfSg-1wj4Zg1bqJyG&6RbZbRg=d-u#NLQXyZYR>@UPXmS!I9KPnm*O|wLMJiNyuWZ|Z0i`3K}SN-wh z<%Vxs?ZQ7$=5XmE_d+G4a0~>I_+|r^clefcz|hM&F8-`F7;r)61#M2+3C~bdtm`GF z`iRPnEGx>_BzhDXmxj-!>0q(;{RhW#C^Xtu4TpE=-74CWZZp2#`m9Tz zw|hSBDWZ|&F1rxnnVgorJLqekSLWfo*PwGz1|WLut95)*T%Hi{3X|7q!mZN>mM9js zLFl9n!c=NJR}B)hQ*-DTh9N@oAX?&Do`~&FL%9}Rc@|AqW-i=T@_ac?uE2K0oh7Z; zMC%b^aH(v=)p9nR002d-d>TUWT!k{GZ{zVVmCGgA=N<=SQoUT|ef?z6=XF$r82QeH z$gV|g84ni+-u5e<{2u_#Kr+8AXq5#Ank04tx@5Otk<1RPoYwjAqxrsJJ3gCrV~~De z4UTTOTmJrlX)pX2GII2lY5kQ?`^%qb=0C`BkD764Z)B#{@gXepC&3tt+(MlX3rB7d z%#WeAPfC`}HO#q(#vKP_BMD;@j&`)Zrp;@5blLOK*SBMWDD4^ZeeFBmHs>yiuSA^d z#i#bVw$l^FoLHom`-Ttj`9=+tHioM~!+Td_`~`hqxHfb$mvGwXYMIm7gt~PlD17r_5J~3T~VT!j*M4m9st(Tr|Ky56Go8e?MwNt{%i6gwc0Eck+>;4LR$;F0X#mF1g z*mK2y!z}fNeRTb@ytw^dF3VGpHo|SF0o@1jJCfR;fbn)(S{rrevq1NWE`=1W+p$|! z*i$@9>;3W8j(CRE{|FTSez$X(C*6eaNOO;`4%N5mF=B2V3?NfCQR%O|>n2RPjVuDP zJI#Rr3VZRzGVXQ6w4wZV?0j^q!`9tx)U_Y(lIL?BsPE%@=rGik{Rdhs<8H^}sJ|tP z-hD!HvvV71@&sbDhPfs{K1{esc< zz>5=)I2HDaku|Tx(GRS~?zNTZ(v#O;`Lw_Mi8ks(8{cRchxU)m)U7;x+MHUy$Ln5Hbng#l)J*& zgX^&S{zJ@ei0hahC`Q)fu)nDu?ErHSeXid^e6R59>&U_C7HQ?)rUTm8?UL+(Y35tgz_4+QomXt(_i^USmuWlI#L0WQ$1#a%z93He zOhaK{c~Pi0z4Rfx*UiwrkFqnRvQjUM%g7TfFaEF$Hp8pD03EVRt5wxIdC|R?cE=p` zDBNyk#o+@hu~%Q+2UOtLJJ-m{zWz8m%?ocsBb5E_0X-h7)SXq>LT#~vBJ)85EYl8_5qE>4c&nv zF;eqxTB4fQwEnHqH%vl&hB;9X8Q&)02a8xsUw#aupWW5ipqA|zaT}AAidg4aH(}-7 zh(!^0#0(CxDN=r&48rzhcsgwTZk9+I8;&yy~U-pF`c(ELm}cCRxg$*R!0w-u=~&AL(b zRK|y$GCowU*pw{EyQ(qpKe}6`Zxm;#V*@)LI@`fuVU+iv^NsyJtDA@hx2CO>1I@g}D(O zJI#couXqg878Yr;g0+n}zWzt3HDps&&!Me#K12kA@bal4e-QUUWH;gYW5zEmldt~J z<|LG{eI(bzVWlY|9QFJ?tQB@+i}#-qs?ghe7SZ~koO2$DoqDt&YWo8h$)y1$L7d!6C?A0%4K~2 zKz*kHDff-RLB~E6R{;M)B`eHAR|JP5jUryK4t_mD^ z@k&gbUEqANf9`$lvnI2MuC_UX*}1G^44JZ<`BrNNFzWYUV;QU4NHh0l4x+o}1OH^W z>RUL*Y%af-EpVd@iDl2@CEJ?=c0;AT0YgiVtprZK`HXGyqTtD2Aj>C2k3(Y7d`!K6 z8$P~MS*}uublw{4elRNwbBs1xj7Q(wi=3h&F&#pa z+W9vfD1}j(cCcRF+BZ@6i*nReSv*l36qcaiu4TwRHxFI4&*FotR>_Wqh3KrQLDsUz zF!?$q459J+d$9AKP5Lun>K2(0$_pfXQJqK7KXTsq10u+H=Y+~{($kO1R^%yYm*bT` z){j@tSy7JEW6#LWcvZf5Czsxf!dt(JmM4FTJ>`~xK}jj5ulXTlUQxZe_PvZVV>snSsUwap z|0?!Y7D~C0?e1&lqVUzVY8YsgBP_lyD`KTA#Y(E^pYqLMvd(9qSU9y9@uE+ zhmzHpzSL|hVp8%jX6Z__Y@DwOSK?)d*O*pis@!dPkv7SwZ{_XuB}lA zznm`l#W{(fJssG8dsAM3Ml}BZH`semgd24a- zn(v}cc|CIxrk0+D`X#H>Bdj$aWii5&>yVMW4f~drLG+A$SMGgJB7e>d_1@p_$ZCIl zkq*NUn_^Lhkjb4<@{XZ|5|^H-O(ZzF9PNMKf+HLL2S??&CG#pq-hFpq+?*n{{B+ew zzvk*y=3_Y)Yu8001e>=SXRVztZocyt%`ms{(-Qk=GYF;a81KJL(c zo9KcaUcW&}lNX+MDsg!IDqGoBo+Qt^4Fx~A32AvUcIh$iDsM#F)sEvoxdF9GAGvT5 zPFweFH2>mS9IP_akt}V-w11N}!%^N%7bD4Z2*>ZSz$speqVnr3i?Nk)?;On7l7J8A z$XG;0Q-gz*A0zYTBI#$i8T;ny{eH7P9m?^`V1DHU(dvj*mi71)MpKY+-7Uxxr9JaV zV(ZAd0Vf`PK*oof?7)a?_Ob_%f89me0wKdv`iD)w{SSP)&f>PX6jQFx!=bXZYCD&; z;wemCs0>Pm{u($I;njN@0i#c?)f3b1Iv9Hzl{Qhk+_xb{J*br6>@|;JY{uWPv+P0LKjo(w z1r=*lS(wE#VvKI{mbc~=9K5>>N6o5SlDG)dq+gdY#kzPk{4!}{xZ(k1;!W(n$I)+G zwFIdVbzD~(;!;HsNV1MmPsC2S2zhJfqQRK-x9sw!eDI#xS?VS|hgjRmG4U^IvQ&8% zNn?9Gs=gn$c?!}JhTrCi+-=7ee0J^KwsCFVvhywXMpOdJox<%W8@>$xUk|(@$~LZ( z$C}gyKiUzQ85n)@^+?`$0tqD(#K}xqYaBYCYVzoC#=oKc8594C%lQ>iFlwwt!=sO2 zZ-qs^t|rVremBy}-*5?p^F0l}!mIm3k3-MzrOecbgZ`T)i)5;Pi@jy*Ei%Ls*_30Y zY|4@4aID>GO4yBP5M|ZRB;TvDOOdxve(o(~4d(n@z}N-1jGuC*I)2ke6Ys!t%!?#h zVAa0;?SL;CN9cAR*<%@t6eIOigG&RBKYTxGH(1KjyrnqxJBir8SRY@kk@3ZR|M5lV z-uF=dyK?Ne%vq(&{MV%0=O7`h`XgYJ2i&r2#~ZfA^=5#wDZ>jg_SNSuO?$=UMeB=C zVDFvfKG~xByl^fi-Sy8Hb8bPz$y^fB^pio9MV)FzpNFg~t}vDM>a-d&1)4e~bp1>-N$Hks(C`V~GBrBFp@V3hn$z2vakjz{Yl$&m}>HZs|TZcpdq zWIyRAC_H)|%lK1^L>sqr5L$hvrzDK}Ak=(p%)IN+UHMxJ7V8|+Y>U0q=x)5^e0w18 zskAqcbm_N`KV&QGkTf~}8qOj+4B8EaFjWdLYR{H@oTLo_tjHmMN=#&YW`1)P=LJih*V4YS_ zA2n{pE?HG-On}>BSyP35rB#Si202qoFlin{2}$#-Vcm_$oV68Mom*6eJ>@+`nJ>Qd z0i;KT!k!XiFFFs%iWD7>ao%{>aWfy)-y;(Nic`zwskj^44INF3J*q0O^WVmz=n;L= z%%hx}E|=nerk1a~d(CfQeN|m0W3?Crb8pLw5;gA^W;zURT9m!ACy`w0;{(b9AoYr< z@}fma{kKz%r|eGoX#>*E%|hE7uW3)sNA%)&W8 zo`r^=>PJ{edFhn;V(Rr@_6dybfkoQg$SyBvJ%MIRzR0`rVqY6#eyIZFZ+RBuQ^z5} zxAUg##_6)Wpi?_oSE;WXHmo+&KoX?gTV*j=u#{By*tY2zw< zbUZ~Aq-GDp^5RHrpxXd#gmT_RtD4W05%xyx@k3WFlO2}^@(p!&U5mKfmyj!U83Onn zN$pR-1oX<%uUCUMS#0r%zSzR02?L!m(H z7v}-7kp6AC!(-{^>=I?ZQ(yQmr1!X6*T8)nN30_IHtc>w5<1`cZ|qoXRTwan=Ba!T zHI+wXF~kZa_KTVICxnH|23+mS0j-&5b6)RgsN%gCuq-uR%%+(dBwY=Zip%0`S&W7Z6fLMfE?WST@{`EwiwDTwGDlk*oi6RoN0pg}8>_tN!>WjQV=@9R`NEQPI_>P^@Chn5I zg2!J%wmN~|i#Ehk4<224V}Nj~MHV&}2`^OluuDsT2o8poi@bI6@4vx^rOr;BM6z1S zk+ioHD_%m5GJcTDRhhIhyhrLe)d$NzYQJ$03Sa&?(hPk9%i|$4I+g$1_o4A6i-Lnh z%$Vc}k6jW%x08`Mw)f-jB2Pi2q~W)DlCS>$TWl}2%QYke;?O!vqvwlIV7Zg4yy^36 zI2$sNI6=~<%cwjXZpXF_g~%?w66wV=F>23+=Dziyz%cvJ>++{^DB~< zzW&`HN;lg0{UnSH6fDW1u}1>oeb3pdd@=QfYniDdDOIwob(O#z>CaF3!i$%_&eX!!utofd%M1PN2iT(RV7=_*o+~@RE8njZUB_#2 zbnPlL)ZA2yj3m4?<_9<#C?d)d>k{+3hBW0M`>rw^H2V0}TjB9uj^q+~sY-cqGn#ha zDW~yCEz`Oi$9%i;NuBdXjhsH1BYCDBEiY)3+=>J@d1AI?uid`-B`U{AO4570hMrw| z4|3i9(oYC)EEAw%?U7-mIq;u@u*EQ(zD0=n*o;$Bal+=IeiO1ya5r8G;($MSaBKXnPF#UYGq+rNn8(Xu@*T`lqDMOYKRo$8L)Ar{L7KObMOuYXAOkHyuk}Ymp zb{>(t42)U+7}72FpUwFvY`@}ei;e7zar;F>rch#uNZ3a6J6c~!vEIn1U4TjV--{fB zukUH0tjN6=`3o!$s^3*khX{DZNRCXnV6J0Qtq|!YA`DI*eBI{0l$XHVv}9XTVw5G* zy^#rrxV%Vt%`RQw`%cHI_lvn6&HALKx`XDDxxVLyA^h?pTiq7U$Mh8zKQ=AJ_FJx% z9Xbu)fSmh(Cc9D&MTKF;!HvoMj@GwEMtMP}slct&_bCO|9SKYxad`z6g+0waFgNX4 zpz>n+CMLbeC@V#kYs^=SGNj!P&cD4YFKGV+pIa36k{2z-G*Lp#n`fyrD|g_e$Q*`a zd2#z~)cR6{Nm`!JrSIZNcG+r7Ube{3yZCu!m@GTYSljDk+myP@|BAHs$3KSsW25Z& zVOQ8IMs8V|z5T9Si~Y|E-?SWL72C^Gd~pe~&K!*x7kRgTVjii!6e9n=a!j(dht>G> z(YJk*nc>E~-@Z+_0Y-?s@cRw2W73~Yg}p40 z?zGqLBWwH;IF18j+jP%ze7aoz%KlaUWZZl9IrwO?RqbLfI*-V9ipSj6yvM z`cU8r$-xFXFx+HU?kLGay_Yx0PEKp>3VX>5N|C?B9KS2ya^DU2qfK8|>&zSO+Yn8* zWAF2bgy0o(LFZuSpHSh2owsBc@59Dqp5md@HYo^;OlW!8G424B2p z%E^51Op?XITkglF8%%Ar^ko&YTeWryh0k3?8uF1Y3#^Q*;bS>-!+QY7s|=e3$hmo; z+0U(dTWzm9K7QA5faJ^c-TK~n+ODvtjHS!(l~bL3PJ1*cGR7(8GA6OS)4AdI(${iV z?t2WVzZ|rFjCx}pNBkT&>wKhs`BIDgWFwBhXYP-#1lv{foLym0PR*JmV+re=Y2D)A zp<`4Cd0~IW--SP&6}MyiMVDavtV`wZa@h&(R@C{jdpS2+M>g)Ag^3Frd;dilHAMkjNv}iBN z*1J<4Uy5xv7y_4SpHiUWVDcY-Xr3PAA5@gu$X3F?S)pzvJbFS4>aShvOUJ~=ACY5- z<$KRN6-}GI=j;3g&v!}4lR=$($nzdprP92hCk4AWTk>#k#XD*hB;7#go z8J>5xqvMJueP>GfFlU~J9y_JgU&JMBX|BeYP*~^B?S$Y)w;sCJd??`BpHj0yz&y9xbZ|;9bBLa8)e7Fmo{Rb zA(SauiZ89bPMuInZrO!T?$O$mgv12nRmX26wLgxi_2X^pD}S)~wgA&#mG)cdmfd^} z(tr5_5zc7<8&*Dw+_%?a@IW7%RJYn{jX3w+9^!laMy+`#A04 zacJ!vi{!lVNXQ*0M`26}k#|i$qS_?KRG+RMQw@xMAsa9c@wQYc_1%{Em$FBQTK~G+9=p+ucYV`SfCa=LVVVeBRtPFSmR%@=Nkm z=VwmMz&2fBrv90C#eP28-hSTns<0Ga`kg+8o^w7DuAYlS()WwJYam$64!hn9$fHDd z&MjZ4PRwPO6ydY4>${oDIIsTY8aWOt5+-X0f?KrYp+3kP8h7HbZvsgzT{1tN5kkQZstOqE*s?RF4c~`^h=t|5A2h_ zUHPzAde1Lb`!c^Xv^<@5ei`8(Nu=jCpCNF*?$m8=bwY@AN>poXYG_@PA4HT zJ6n$D4Dc?kf)j8liwn1Wg&K!;?f7lK$xO{Qc{UO|_M%3gpYq!!u22wcT8mH3Za9*c z$YvCh-=pJ-L-IKn$RQhACFyy87uqlEFgcc-DGSD@BU9ErH>skm&9WQcKcoA7Log`q z`Pq{tsqMA}fRH}&nR2>>Qfx+FdOHlZ+)4PGc^X zxjjjLxWWcW!taor^C($5vC(S9AFq+8Ta{_``oU`@|*RM5>a7j%qH9e@%wbGDb3D?(TQ6 z*~J#PsbhLK-|A{CqmF~!F^e|WW?t^0Z=od2Z_Jj3%%F2ftpQNDlM(taSPI}}KU-00 zub+X^yFdg|kuY6)7`jr7vTbm;C{_elpbF$i@9gsb<=aS5UQNnEf;!pTJAeB0mM0a- zMW>4w+Jx${|9cJ{?-)Ju1sF9Y3zoS+Ur=!cMHnrACu_H(<&7t>Z|nOwT3O|L?li;| zT_-0Gzl)l3yPNF$p3*0Foo__>eepR+R%ExzWvj_bkgD&jfKIRdk>)z+zugv@#Lx+W zM|gRWuM?L$4QVBE{AiWj$PS6eo)hIEF)CMf@eC=`{z`jp%Zm)3k+c3GoREQ^bwvxX z05g`EtDv+{hrz$Yw!Fx(1*0dZtK4t3ZrrHAjqto~3NLdv+^Q+;sUC@t@rHjwvP{6L$JWbg@-Agnsd0w89$2?x zr(SOJuJPV5`YpZ^8LGmbmZp@;7b01d_6&aR6sP+0+e8=R@cRu(NSI)z&9mu2?2$k9 zDxQnHZ_UD}Gh`Kgiaukd9*bhSH}kG(7tN-;1(qY|Gji131&lTFW}6YsDePO^yPJ<# zJdjEaL`6dWxX8&XZe!xeTkRihHGQb5z>!DCO3ic+xSw z*2X9cpY<#0qHjwdZ=*w2W<0h2_N~3J`ZV0kiRUhqV{os4=kIT!>GjueROBrlhrFd} zT`7)zbnv0dTdiS+#h#@V@8j*ydcNnu9QFBCTr1c7(H%^Zpk8F%cNiajo&ckg>ePQ`u<$h84EB~U;O{cop-*; zO;)~+5+_Dq%qV~TRqp*m?~P3Zjz6@~e6e^kVp?P~6_M#BoGtUiReI*AdIC-L^W@lJ zW#HD1jss?YA8Csc3F|rp;*W>1a=fE~J8EKT*)n0um zCO`L<90~1pmBucEpozt>8M3m(y2SShdS}R)iSeT7s~wg78d?u66di#ohhH^lsX2s% zPfcaM>~Csyr!P+^Zw#j~l7m3N+refRq&@YETQBEZ-Hv7a!W~u?F8UTqx&w{b!u?*# z`)nS!>~cbvJCq-dUxlR=vLy zKby2~&I23*@$|E&`nekp7Rg_m<{PxcPB=rIG}M%@#+se@s7&ZUl^0|7DmavgC@)I% zlE)(Df)W|XRmn{i4sDPh8P_SA^r>g$BWA1hI8?Eht2gBZGVsfmBGY7Iyk9$#G)D?ceUx}I?ghRFf?>&V30I!S>> zFunGASiBfJ{#4(XPg$_lwEN$YVwnUt6DdObT8jj}xkKHlaJc0cGrj$5N-ld0GjEo~ zZ9BH$(`)ZG$JUkqgA+G@6=Mob#RFT&)f;`gY}@T$ho|*w-`G-lmNs*O%}K)K$Q)A- z42zrG)q6Hlz{xgy-4L5*^qCwEhwye_xJ64I>Vv!?*`i@ARhV<-dHnbg6ZrDIUAG|< zQvJdh!fX>((7nxD^>Q?BpzFxC{j#%rQSZxS^%wS?aFa&$q2CxNhW%w%g2GpSj-fT=?vj96+nsGJWZIF2lA{P8RImFl8lL%I5^{!@59lJ(}t+E?m!l$Mh_HkgYLO3;_c5xX>Wq9PupV}P4E0nLsgbom-u~(;1>$Wv!1 z(R@Fg%Zsv&jkRxfu<|{OJ9oBSdAGd=M^=hpQ9fe5$WfA)J&Bpuonrf>uFH$m?U8V~ zFE5h!CF5Ha_LAq7V*I=kq~sjKuB*!ev`HhYy!8BC(-3QEXOtDT#x?3xt9F`UJJNqU z;jJ?n#CAH`8kyLzs$vJ)DbBF#P2M#_9wsIc*jNRcXm9S3U+|Ro-?s@KMkGFQ*>8}Y zq6*%B3puh=#)-fF1xGf@v8Rq??fcnZFy@hk_HQF@jJ)e`9<{gKekqUIqzp{N9ZJXt zfAMEzKcXF(ZVj;e(CheIKW<5}kH6qlwJ&2^J#EpAQ52TL8~a*A8NO!Y$oYhsb8y<@ z4cK;rq3lwFy;uJf$#1UJjtLjtF{D!u72l*yesXS70F#3*ln7HjypOSUwrn|9c<`kT|P}|wjHM?-q5y5#a+&rbM z26xlu*&R2ki^+X*6-Pb^iO@yfw2fgxdK$eCy&pY|9i^hsrd}nO@%GR4g~4%%%P9~A zmAn3b_Ra^c>gxXgF9pTG|9?UY|7EwlIW*I>NE=oTQ`1s5(;}TY+w?naE@>LrPa#{_ zx56eXGfTA0iztPBM|8OxA6H_vcP?#93-}(H#pZgE)hkGyFKd;Z@0rz}9 zpY!*5&iR~w@AEzw_+Gw#MC3P>)rf<8p*_Ehe?9DDMo&w_)#L9%&EB{1&FcmDMm<4E z&z*vYapvw@(8p?<57FPz;F5sp{Pd!J>EHJj&zah^>B@=m_B$!c%PFx@K2|-k#3i6W zdKhsDI*X0|UmMpAW8%+&I2#sHqiCY@3TB z`X`F}V)5tx`rdK=%)QlscvHLQRgdFwHmeZlX9-0oqVX|ux^PNx(zQA^qE2W!Z0H(? zsQ$6&(6dKi&AD6I}adn!Mx8Gy8 zac}Dvfyqv9PaW{IbiLItV;P6I1E9(=ex3bLaZCmn#9yg= zv>!5D)>L8_zr?ZVC>}U9ye+6N5JFtXXFp=+y>|2E{57bp(eB#*_dBte=(j5b>I)(U zqj%b58>0N>IQsH?wz^P%0%x)wvYSMSy>@_XKP>BuNcne%KeO?&EXS3P{1QD9;t^#$ z0mWuRsE!Vv`eKZ>E3;qglL*i!6KZYP{UCn_7W_x=%haRhmFH{| z5<&pdl@`o)s*t4Vva1Cez`v%;Wa&D5eZV#?T=&-@KEJ#f(v`r0^G zq1y^9gU?ua`Q1CQ@pU_Pr?ZM}prpZEfL%W07DQ#JrSlE#dk?a03oQRVs{VXL&&}^o z?8mX3QvE)%bKA|Q=#gk+LsMm|<4|dnqIy~8qi6Co`Pons7iHlyYv+xX%?Z`zj=gGGCzH)Ye_r;xd8Cj|3sS22!n`Mce;e?!?{ zalF`OqsO4$wj0f4D5qYHe+~ZFO$d9?$GmXh&-ij(CJrxu5H+|FgEsvEBTL@F5B|78 z);^|1)qdf|_B32g&5c7%<)@eROCN`NF=y&sKHi0^#JJhA$Tk=B<@}da@pSKeZXMTA zxM-|lu2~Co=5^4kR`dXLP1Ii4etV79HvULQ9NW@=>i*|Z`;F3;HJ1MwpFdS>bL1Jq;tE=!heox2hL zt@{RN^B$Fj?=;K3XI`(5pW5{wfqv2_K^6ekY>4Buhgxva2UAnLdYwhf}dAj5({)KJ8&nT)bkLXup*cH`f- z?M5e|`nAJ3adePvZclz2cXy(^YvTLvedI<>UcV5Ng8$BlLf`9qp!OB9J4f{?SbjPU zA19(}Oz3393=k7=B5_U?F`|>*u8}jl-awbmBXHrZ7jeX(l|dUNe1Ggsl&!F-tnV~` zHTu-Qgk!4Rvsw4x%(d@`qk<$!$hN+U@!5?d-9}>IAEkJ@eeLHLwvPYV9XUAmK{PJ9 z^8xgZS9O~EIdt9o7SRV`YHh&3}r9s2}Fv$pBp2fjfJV50}2r&whB(oki3 z++?+@7kWBO))z7Bjc?aKWH{z75d$B;7RMi0r_N)EGWyu-7!e?9`_GH|`$3kTT6=>y zj$*_kx7a^xwYvbRFCx7c-8K(B#A2#8&HPQ4P(zE z5@z*9&}b)i$LUvj5zg$t2A#W9;$+Hl!(6UtFXXkh;Hocry4)C=uTRt5xb*kqz^PBZ zPiX{VZv6?c`4QFYZ8$XcIARxB@!wD1#HnK2Y(~pAjYO~(H9Qt(>hTJzvY$6aEB0o85OW45fidG!ag>hAO1Bs6<(lq&lK#b1{xGZ9(!p|h5fyPiUKRRevy7L9#Fx&81ghq~Om3Vm{>+2#kf z-H)#){T+S87_RH2L@}NymOZLk^dtN?P4K?tTbx|5op=P7wAKyAT`S zK3-6RK{NZ|+ZB)K-vHU80RHC_7Q+Ub`|dgNkl7u>B{%iQfBy88ft>|?e%NLMeaBeZ z-TQ=FFqNT}&NuYA8kZ(Q*AR7T#SEM;osZ73^*EKY&Cvc$Mr@Ll*S7DI9q=%^aa*>A zj+HkypOsjYZx-L1dR(^XW;E{Gildv0bwY#Qco&^6HrW@^T`PtY3x9;~7Tu4I=Rd=j zVj*0=?)tH^Sz3P(K!zS;aM|N`;nN4Sd8~3AN&gdu3$4bHX4rL=2DG+vXq@Nk*(re1 zYa%Lc%aqABuYLdPaEYj<+95W_Qf|Tp>@ zsi6&G!$UuO_wRC4t1*^-I&?OS&kfJR)!5uP)H>0?tJ%3Yll7~KacIltOufv-#6hyQ zxu`o=pi|C#b)(^CeEH|87mw8oYpB%)v-jsG2O(G;+qZq%lYUgY4#M!4U{>T z^G}I2o2PXAE)9J~3Z*ULD{3l@#`ZUW>Jq;tj#=FFIyWE$SAS&u-t|G3I}&jstJvm| zWgI$*Vjbb)d(-{e%WU>S&J@7xnrKgbkqzd@GQJ2kg#B=vU#`~7&CLCx7l~b4Rz+2GyM9-pkHN+He4#b&0U@y0M8DRkImzI( z&5aS4J7^9QTEpaHT0cLDe)qcCqN82F8qiDNi>)vJim|nJ*<;6k; z`fi;1$Z)2g?XBNUEvYZk03iB@#>S`=U#?o|QeaQh2IHEu-;?JQfPFlqp;r&Yj1x0m zhO4XfQp26mXrjIdXBYNw!RH3y*B)z@pyw@fa9L`i%~PRj{P>$T%>lL-0cNVYAZrq_ z=cmOneCi8?W}5+j-eC95lZ2$51op(y>q9c`5H&@*%-0u7F^8CMv;u|g=KN*o1X6xc)JLx`KE_I*SL$uOObv-$b>Ccg;(wRm23n~QQ#yl1I>}punErS(yFq*^yUoO7rn$mJrx8EI4(#e+9a^ zI^M^Xdp|t$J2w&MU@1cq-j>ZbM5E`!o9)J2XY)7Xgq^@15SROazKNhI?w+(EZ8hF9 zQv{nzqucM35S;A))O=eorQ7iN{P}hQdy$KtMGr}++Xr>>>zt?T1@^?a`0@o~_4daW zk+w5A=5^F93o$U!5TDXVaH9B_*x+e`opA1HJLwp{A>>v!wx~>*Z1bc!x87^)+fKXj zh4lSa+fR*+Q@ngoHd`I8-vbm}GoCWcw@3k&i?QrBr+#3U)U8S#7dG6LQ!{fnQb_Ln zG26$yC~1M6M!l+4i&1vR4E%e=Q#N6l#MS7kZak5~=G6=VF%IouVlK|~x4)@zsHm&v z=S-VPPjj!&spG@0BQeNuI?6XI*5h0Noo>@qGTL8qg?#+2(N33Pk^q+N>KH%Hb|ag{ zqm!Y(+um}AgI4hJW(D?~zO{xBT=kvY)q8`pRxsA!+dpi!g_C$UIy(vMNw52$yHM<+ zU&Sxstk9&}-g-L*-=#klQ~LR$f4LcF4P%KuPbJGIFi8uluI39{YtE&q{4u0JHb&0h zfL`i4x2ZH5Jqo0{#BYf)XPPmO#$|t6CLjN7w0P0=6O6a$G-bHlXJ|Jd-mcw+<$iST zr>tPvIZ$u3zL~3c!T+p{*){o%F**6|YEItdSSG0Q(^YzXoB*e?-$Z`7`d*uvBlL09 zJ@Sr1dmdq;CEpN;LYH|@;s?3w(0jBCA4dyev(2v~eqn-P;;0W|_dJW9DiC^rtD(5Q$jz44AHi?9#1k4U34d4dy%BBGqMaHU{41 z0Lmg5J@t_7n23TBoPBE>P8d3{ak-0ek<^|?$74XwFKixS8%`D)EE?Vfk})Du%CiVW z806>L$WheBgrUkPWk4*ljWTv(XX2s(Y989MO#fDR zT3^neJ%+Cq{az~x0*xdjqU?BynUkuhs8-*8{lF%}5so$|tzt5yvB~hn_oW@{i;Sp! zzXV)fvKxc%cP#%3!QPA3;;Q}6p{rrQW468saW`9b#K!35&-So&Oy1TPaTYFyZeDHp zKB|mT*J9v&+eh%*58e-s`qCVBL$)<(RjyCMhO0KhU3=o-p>YRJ`|d8nImWmhC7@h)8U$G?fOirPnUS9aT)k0;zC4G<0eX=9tC z^~n|QjP+jB&tOQSCmeiWr+t6XCwc#`VnfJXw(pwJU&~|+6i0~0J87siKlTm!_QS}< ztyzqhd)XQfGTk*RYr%O(VL52|Md*!bfG~f76u@;#)AD6%JBD(4e zYfbvK_pm*v+=ws_n%5VFu*@~g_+-uu+@WKzb zt;0>yd>{B_NWv)mo zEnIyE^p~w_XnEO+O8ljmA6;uF1gS(_>Ki|cVGGCTDQR(?wA_XMqAjw!M8DCp=XdDR zqm3JGV_w5HM`ep^S0^FSQ}_Q_K3dTXJNNCRq2JSnV|!bAHB-yWl({xfG?Ak1jNiV_ zsSNgebOZWJ^M~>jvgqT{+v2jRB{u|r+x7b)e(?6IF29p>z7du$KP6h zfoQ{=vJ}1c1BM#XCi+72N68unZBZ8*))qbot27rjS{u(;(C_iBu&i-4f9?8PtN7=7 z{O!Kp+hXB&Q{9)SF1Cdd`;+V4ejhrGpKh3sE=SELUkZoO=)U}2T(QQmp+WOb9XrP* z_EQ2jjfemMKmbWZK~$bJQ`R+S{`s|Uqn*RBAktz%Cvh}z|EIQN*b?J9w}~_w<^U03 zbt&w>C3gK+#}Xjr{bcl8vjM|OoO7&rTKmBx?KTzm9yLUH8*vG`-lcsbg#5#- zvG#LG#&E@YcHobZ42jl^#x!Wp^xptt*Tgr*SMmvF#xgphU@&;F2H~r|g^t5*ez|E+ zP0bPNp|qNz-S^^>LR-ChwH)Zq+o!i$;0!4E2uy!%r_ajCf4@0}hK5MdffIW~qGAx) z)wvNf5^5Co|0}GvkLihsX#YbeJ{&x8@r(){BG-mH3f8y4p3!X8mf3{XpT+ zlDN0jjaJs$MRgVB(kO;t&PQLeGh1I!UxV*zglffBEH}PPuQ}2-+ihK6q?iN(2OAOp zn|xJYe9BI2ulw#yE!xFI;(JjIB6{}F1yx-V3Df!_9TnYhsz%n<*uJZ1TciBnwmR|; zxBB9ke#d)X)+a~59q-$cz5cBhQxHwoPrYsmd*WP;_IeR*{j%GZyd3jRy8phe1`V}J zJKsU{4}EocpEoUwNn7#N)9Yb>Nw+VO*{-Sai`%kmj!#>0F&E3$wH*(uyA5dHNz7A- z{5t>oLnvFdTgKBRb-P$-A8Hy^Tc_U`VY4jc`3T{G>+fD2&fI?kjutB+^#9|#LXbI9 zl;JbV{7u(BqK|0|z2)Y%W@wgMLv8SsA=C2>QZ_c$iSL{Et_h>-veELqZVi`hU7l=d zl=`)zzKDKtyXN}F!jV)Co2rxlEziTjA3vgaFEn9|cxoEj`KjO7_6$8|=kIp1?OE7L zlw>mVy|T>}?E@+Vid_Z{)_ab(5v|3%Udsk&;uF6`qk~gl5evO&sSdefILr6GJ zuwxIS_ogn$%rmymSLhb12O%)?($@rX@Nox165qRYbxta2)oC&}gZ(n)iUp@_97jeF z8rB`ZNF}mj;C3}>wNs4UU@{CepibV9K#Ar9H%1lfGJbOXdKf4OIpxn+r)vfn+*2=a zBHg&n_z;f5N-7zi*R!9Q9_xgl!@>m2s3ZAmR{a3<1Eaxf5C4R9q{*$Pz2M&=>Pg-E z0}IkDS>_6kfE6$!|D>$P$gzRBbvAAjjIj0m!h_TY?FV!1PQ^tFYCz7lG&aihRn-;I z)M4LvFxI;|SG=M@&O z508U(nvXqSn=Zyda23FANrhJF4)<7p79O?DXEqi>WO~_iQyrIMSu^Xoriw6oH3VvT zNh|>+rYt^#lmj9!J!q%Zhv1iN)MIq^`X$N*g9O~mds1@#rC;@rH=uaF&Dr6$-$m{_ z%CS@@g8G(mB9{t|?L|1Liaq)=oET@>N?fZFWogXOV}OIn`j)uaa3q-wMryuWKN=w`BBZLf!qfCiSuG zgagNcx(m1ibl#;xi4nMzQO0)?gx3lGp~94{2BNiXyO_mJ;xugfL%C(89(S(M+w6#i z=?DHLP{id zl$EN2eTb)fVM8|MyI1agyHegJN&VMjE`{(vd6C!-k&Nf8m$*GQw!^Qpe?tmI6vcNI zc0U6%&(qYa`ui4$E}%=E^mJ6}r7D&1tqY7I|5L~OA3jIi0LAyZ*NZk7+_vQr5f1-9 zeEolXV!V$<+qT}XS6r=Eu@!=;oy_CU;?e{o;$kMuDElNJo60)fpWDvDpg5aIrk}5D z^f9E(EUEX0a;Uz%dWzoe-N*gl)Hb$^ul%mRvo#thnVteV5^po!Vk6T*KWx^-410I| zlnaaXLMTHLnUpd#(H}``LHle(Y~URWr{%3?EWmD^!t|fQVUUI(>U=YjxIfLtVLvbZ zrCa>BY})@UAojm}fgV!UbR5RT;s349mGOJ~|Ns7f`rnshV@9G}^p}a*{6Y z8$4b1#UFV&P$J);BNCUJP{(9OA}o}dj=oaueo>{HugTaVDgQAy|0R`1s0ukVKbPIC z{#YvLDOlB;dikG%c7F!xl(jwps7m?o{OD>r& zuKJjf2yk|>o`RSu-5$ueU%HpW2zcoFdY8TZp7vIJw4-=zy%%q23yCaC+jBAFD>TBG zf)$ezd4irJqlrp&>CdocXE#4vixZ|pvE+KJ(^+`XVf+g8nT6J5^`bT|8tvo=_tY{g z{^E0kbNg{Mqa%F3)E5WicTW>XPs@_Q!sSW0%U+~E^aj2P;lr8XXqqjYKrK|FDmrx1 z1LfyueY^@-&Nq1&6g1v!a9*xbnBnOfQ0+!pf0Ggxr~C?JC%MI_n^j_wKWl=D+?F z?zH){NQKC|!_&~1&I9pkU#744>#gVanI8tT`GzN_jMzLU8H3JRk8s@~vGbb^J88Oy z_&Ljz-i~m!DA@M+>BmyntYTnX0X}doUcm?~0D;P$3`6rs<;S3FL-yF*We;r)#qgo}&T>)*w;3-STps ze3JqZra7($$&Mp54*8lNESGq-UBW4S|KmJ4q=eM{ZFpL544Aat_ps2`=z)?PDp1j*#e&VRFK%f;YA4$sr zRx*p>$3ZdW@GY01AYxs2b@DM)^3MrJ3SkzAP}a* z&7v)oS7`2>c&eI~9{Cp!QeM|YbfvDFlp21s*>N!7P|#lz=cka>xlE#~`Vp7nFB65| zxhVsPNzsoz&6isAvFo`z@1+=4T}-WmFSH0l5&*6e{Z~ZVzw02jP7w8n=jAzVSh@OMzL#8gh9`)+*VOWg-9JM zbLctrpD$l;)Jz`c{Z29}vj>}$l`lZ-rs9<}c9UCZ2?VO^-KadRj=NbG6n2awrEb6D z1(~nfZ^}bS0CF*>36|hgR;4P$e}+8#RRQQffQJ_$BB6l0`EXni3DKJ0y-VrUIwcMG zjU7HdFeq$XA7W$9!J>|1AIW+?Mz}bG*H?&OOEVlBwZ!Srow7GV(|5;r16W!g)%&a* z9WxXpBM`86t~0(mme++~?y?M7)ky#a^*=zl2yQIsu#>2q?10jWjcJl*t?Wj6cO50+ z`mye-v#+6^DY^vHLg+xcvf_3HfrZUYr7TwJbv3=#WG5SSbGN4DN!IN1T{$a?Q@$8A zI+hA|1}S?Vu{4v4b&EKYc=`0W2&~5_MD^&*f?Hr?G9sHwfn>G)EcqlZ*7K%87zKz5 z5yj`?dQz1#X;>yrU3t32_=IXoYukegi zd*%>>{|NKMlSji>K7AvTguM5v?6-`^leJF~L)KxEcp_eBT@q3F{g=x&Di{C;X^`0!f?G zR*P|wWvrkYtb!5EzYc}9&}_y74v!VB*V9i@asI_fciIL#5Qk}k{mcy(kL_z~nh((@ zQQb=vcUdEfMT%!Nx`$eL;d2ZF*LB7kts(k8uj*zA@tPkIp+EcVMxm$Agx1@K7UT+{ zmS^?W=Wgs=&McO8j};M_NhVL)JNz!!Hk5V>&hewqQV&MOU;<;+-~l0m2Sq z!s6UfY@tn&AGadsz0hFkD|dSwqgsr|h;9qhmxDs8unmbtIuCU+Hskw|3a#QzcNC+P zkk6HSn>$0hRUCsSv-rFve&W4@Bh5kdq=o6Au%L8l*B{u7_(D#Mwofpw>B0ht?`}TR z9o&v@#L#Y=bWw&o7+R4W9Uo8Xvfl!;Jn2d*sO2330M+Ho=BZ`OV;3cc@(*wKnmFHS z&iwAY@uWR)XsiRof4$o6f-(u}f8pn3YAm~!5QB)!;Jd4To@*wBLf@8KjrEUB|Dx`7 z*S~D^8wVZE>qvx%@OmLY31BG@w}ozU(^%4V)yh08%?XYIeP_IMC@WS;FtsraI1Rpk zSd~a3>L4-x@V-hx!x$?E0{Naz5B;_KqfOGd6@ zQtoTIT+mv%(i@?nC=Bd^ljb1Dra)z|l>x zJA1LMT|rQ+Yk3%{5G-zWB(-_`fG*`K>6+?*64a!`zHtz9M~b)vs=rm1{N+2mapZ9k zMP1G#WJ6)O+&~_QO-cz5!YsuF{bCYF=5daL2oPoki@oy*+=*=#me3Py> zJdvv;WS~{_HUS8>)jcE4m9PF9(^Cwy$fp;^EH%!Bn?V&t7vs|}HckW6gYr8kG0nTI zv=Z{<{WN!N5tBnxek)fl_rf`52EN3fO%&>go~Z_V%jERsayL;M=w86&uSVqVmmrGU z2g1~F;I~R*C3f?SkeTc+kNh_6xJzz%!V7AYKN5OM;Gdy@xFb)L+&ORkrYesL0F{@v0gY9VZEapq^!D_7|4~lC<(+wD8WgPu5bsXFu4oV z;;?QAh|cC6O5SJmEp8uJ4nRxzdI5TTN?T)PH%A_hC<6!;<4u&(-d;KZsb`}oDO~9rL9j16i=4}7LN;D{e@5s$#duT zz3UpY*Xk+HN9z>m1mOU|K-@}SGp@3`98L)uq>z-;ISA43_x`Brq5S8W8L3cI&~w!a zKF=TwYIGh4Nu-d%COO33++l|t+nJe2Rdt*e_zmF@NJ z+|M-8DxPhzm8Bh5WC5(X5bHjoY-D*hzOo!dQA1;yY{)e6y>3|a{|HbW$dI&B?!-R> zf4L4i-j86!zlnZNpLH7_eT6fQ#EalK#+1aHhk0B049>pwO>BG?>^A?Y$=1GC+Me+q zNoBL4(ecz7@adACj6w=&As%1owyW$drJ$8bGDOr#QG`$HnhNj*mA42s3RzyKQ(s9} z#txIq{EMA^oDrX$4t3y5MZTehW&z|0l~WOshQ>*f*32dWSAqknkr5?``;j z7{2Qw+1oq3|8v!S{0Qw#Jt3IkFHTfBDCW4=dO?~xt#w4-VkwK`M?CmAd{^aS8kTFO z(aTW;tWegw6cFr3j1)l%Rfqnun&E&_r`6J(QMz=~`!d?oO;VnHcWYwuQ!)25d4mb{ zww~GcN{tl3u#-ORyk3q4vigJkoW725q=$;uCBCv|qre@-3><;wSm$+}K4;1t*|=WR zLKB2gc2To?HNXL-ynoU7K#_poyoO08HuXo4oc^o|N|K#}8Tq>nL~RCido~rVKXG5n zJ>2pVTkd@tkxk1n>3okO6Gx!fzcacp?27YjAIPGWkkvET^ZPfR4%f&AF~NJC58mzh_=mEf2=jmhpvRWnmxa$|2fu+w}j z!F{*S!VMF79g=St{EXN^4=Dfjg*>GC`0&JQi+58tNx;X7uM`X9F`sT8d!)?I#` z{xYvwVfmUIz@WKURE#R`P#uIPcB%Z76G?}E;P#yRRDSBVlvR&j`a3$%R`~Wnxafd)fgE|_gl@sD^DP*f(Fbj zX^pps9RUZ$BE^pkP-lBIkwv3*C*7GP1l8K1`B_v> za6pdfz37OdsmfPD?TSTGSSq@u&%$=j`CqqoTXG>t8KwTF_y!WG%~LL1&)(WKdQgy~S37Ku#+rsx1W;SNpA+@BD)_J${rn zFr0H(Be8uGFapNmc4-{uWiYI6_M3%9J=Qp8=meHR?l8V`+`QXxY`LdI!2On$6AJh( zck{yf()z6KfV{B3sr)01Dx36Bbn#$RSUnxy>7%YOpdvn$ZH!ks$JV4U13nV<=?hvr z<$8OR!tvt((@b+K7WSV2LgqU-2w&3hVtskTu765Vm;$FC^pm+7c5gC~!5#@;Gt;ly zKRpnnoXYI}G(#|e`nySG`%}foRdCMA3~{zZd;^u0_(XpUXbh+>{@vu2VL0~Xu`|x6 zVTyOF`?qTi(Nv#p0moY7fXxisN~XDM`)g+)c+Vh7E;C)Cblc}z-NC+~*eLA8!FGSc z&yG|2rC*H7MW=ofyqGbA6OpQxKtqA-E>IHiZFi5oYTY}ZazFFCJI+bt%DJ0AmGk)C zDJp?uzz!+U3T+Ngs(e6Qs=dq&{gYH9ynvj^>fDRKJ#h|GLyePBcLgm#!QchvPCugq zrXFqeQ9`bI-TmA4NW2%}467^=8^gLyY%`o95aX(^Hoip^1hugZLV;sxf)XQ=(6fl> z&6MpfJlqgTj#CP6ekaIQhdgyWUqhIad(k96T+*QUC}4_UM35~I5$YZqX1vjz##g95aPGo6OU*Xc&W^*OR33Pu)Y=$t!Itz{g z1A&BK4BvYenegd|l}}x@opt}~)5w4BNBEe5WVG-vn#e_Y6I2Szr{^mzuFi(g^FBzr zF4BweZHv1yU@uE#^(OybN*DNqE$ojy`3t9+od}YH@YOs5&&U=-_iEh0x>Lk9Z025A z-#?Or+}RUAgXv&<_1o0{za}xWjX*&pOl2VJ#IIC;6FcLRR$Ssnjl{q1ay%=wacm8H z{gCS!g4W^+E9DeG5U6TIMhm|uy%duSE81!GTVI*Ldt)pcbUMh+iA40L{DG5p@-ls< z|EL#$$V-cAc3AA1(}vQa1e79iW&svidUxOaACM8Tos0|CXdG0953&1yXH|YJBAVwP zWd#uXW)p2h#;7Pp^Tyo8pR5P>)bsUI<{65d+P0FxC`X7qlW8H{5WFqGVNaAJPEM22 zFM;01;R_vJ*(2`fR<1(J=^>ILUVxSi!*66DL~l;vkn35bzZ^Ut5lDRsn80s#h`~UU z`SByC3;^4#1uSe;ed}Zi6nRN9P=xV=}LY?|}&3^Ix02Mh)J_374c zg0$$bMkRbf6y#FEN!0ntaf#ug=U7%nH`1wKp?H*^E0PS(AqJ{-6JEB?_Q}ZX;!v#c zkG!t@$rmUVHM2qsYGAfI^bPaT!XY}kS%~W9O^JcAz^orpun+egG!l5q0zsot+9MD& z7T#lV{F(D~zQq!@_I|4CNL*^n4d{AZ3x{X(Ez%v)>sASmo%)f7b1p4#c!uM+g;s`>~9znq2Ho=$gYZab5;gV`8`t*;8KMob}TRssJHWE$1;X$9Ux^agR zy;ou+zu|b0OZaFEu_e#el8u|?FwOxa!_DJtGpzQ2wTHKLn#{YmX5iH?e4v2PUM)8` zQ({hk!rkW{3FXnc%^vkmHS!g|M~6%bto1!71H1@hpoDIF%8VpHID;Z^pjae2HGtu| zU64ULfEbVJ>TG}&lXZ^VN6xcYXz7lq;&;dysF2fDv=%4iD@u_)FdutOp@fXSQ=Sbz%bWN0X+6PRZCe8m z8htHPQ5@59$ex2L%8gZ5bu}ej%pQVeYH38p=U_7rKAZ&G6al5$1nMOiE_AMZbI(JkPZ~ z-J!C(4>WVupKeES(=1tA4-J%n*?9`MwhftHPiFVHl;f11X9p5bnrvTJYapI#y@Kjn zhK)Y^PWithDhs#51u3Pl74w`L&4%5>4pU7&9<>JA0)ykHBE?`wO1CXri|ZMjr9Y_y zYe>lYqJ70X-bv{rDb1y+&zyB+o1hA*au>cT3Bypa)Y&Q)IPT<)qJt@xH19BMT@l*(`ij%x`iy@o`pY8xmQ?6Ft+8btA9bp=`IvE||208+xubWT`J(|E$|o3kqWOWyPku zY`M;~2S*fUE~TRrnJB$cyH!jgGr&?@!3K>NW$6wTZM$>T#llxg!H3Hm_a*U+WERD3 z*iJ<^`Cg-GmtH@3hC834l~ks?QYR+j1}aqS&>8;MLn<3LOanbWJ%PIc>2 z<$H6j7bXttIN@!hCf(t!d@p1I9N=Nu2$C3Fo09t~ljvqVFuP7KixBNJJ8Xe+lSGz| zqZ8_&@%lR=DUU0#bPtR_66EKj!+ds)Th%~q>1S->heq8qy95>h+H^ zFwi40SAL;r4_~@}u*(1W+Murgf)#`K;Vq-tRTjEXGT&7wDEvb?!j|kLt^)>4UWb%S z?{$R!PEjG@TRl}W6wY11bphu5$Vw7FRox@FBh2qbm>R zAKQNO^CenWZqJ6{1NXCtEx`YD7jQqzv}9zcr=y76AWwChYE+%*RVcpD28T zDT=&(%+I?YrzC4}kuy*8^nIpgzGa~!sSnU$0mdfVyVPZ070yw*gXs^u14W)#{*wvu zCs5x9@$;K5h0jE}9AbQuNt((1LKAvEHKeB}b)u=I_CRhYuQc`{P~T4jZ9zgkIjAi- zsV8?#7KYxz8yEe6m_AN1Go&;eK=RVK^2(e~zHr9JORjj3gMi+l9`N}v%$WNGBOL9> z&!6>Bd#9}h4Mo3R{3a^QAhsW#%{qVzgonOiP(~4_oA&u-gwF7@WJ1=DPzThot_;>) zmBDXHnT`%2wHC2&?_s7>1vD4jV0Fl!!_}@l7i?JN6+De^2hoCaj_tC*rl^}If#d$$ zVCg*TeNPugmF%hD5r{jKbGX;m^p>fZVC>BgEkVRh^SV(njMJ2vLe>}d5p2AQkYp?5 zDhfnS%d#(V?er~J>vHZ z&xsjD)E!!R)?uhk7RP7W=y`g-vVD7awkEbELSj$ zd4gB^dU!5I7rz@46=%UVWTQ~o$-rGAk10v7Uc8ni8=D?C_HQQNl&+yZkpt=4*OW5hW z6o*ye{EBy02lxe0q&)4sp{(OYC#+X^HWUw z!&(Ok6e>TzXP6k8ZcC!Gp}#VL@okIDWb*pk>qk=5(iV#v`Nimga_c93(F$WXFBrz;tYDy18E=`RTVESknCAuZfKn-6 z-UGL9Ck3K!vbN_$*lJ6sXgB8Hp#3lC{sC$b6lBXqa}Vs_e$;>b>#x+z|Nb@LlP^$w z6$3tH;LmEnrxNn|luO|DXlKYDP}$;sSYRyLfT{nsQlKGGd&@h+8KBz0#0fNaJ!$?Y zCoZoMymK7b}IdHWjD8Wl&y!S!?^2Y&dXW-`vPKAO0b!qh%3jW6{Ap?}`w6^XZjKLot z{yvxguP#KtzqnRf?_8^{9Y*p0yXOD8*M<%V0^f7hz(?8X%zw&yyLIOJN5mGlBP3tCPC<@|FG--)_(q*4}CB|DeLyZxWD`(o%X*qqkmt1 zF$3HGfBXK#W2%K(|K26b`Ihp^M6@@S1nqa3#% ztt#DbYAg}>ugW&dbj|5iN(=FV%ZXjO&?&nCH+}JdC0S3$O4}wI z`{5?UymL84W`%8R2J`VDL9H1I3k;X&h6z|G>QOZ>eX4Iu{v7v5xV_jqlNOo$$%#V~ z)}OHD+O09zcz*z^mvx#awreRcav6d5t2HkrMhBIAQti+qm{{FRBe%$?1h>-UK-k`b zJvWcm8>pqGRkai8Nnff~M0oeX;-r8SyeL9R0J zK1kDQG9dL^)i>%Tb~M9pW3&Y%oN$+=8QC6cdG_Itwx4}GXeC`g@cmD9rgi-Kv20$dxiNMiWlRwc3Sm!NREg|w3f_Cuv?}pC!*cv@XNlnQ%pV$ zTE#PO*zZo7(Qvik!Hh)K@%Y+COxMdFs`V^pwUS$^HR&;p^kX!YZ}ab+c>bnZO?TQ? z00a#!J+vE-O%GT+8|dohWgpK>%4G_4JO zvOw>Op_oahr34A$ctxC{ri6vM8yAyMUM%Aa)`|lQ-dtx2<{kB~F)m|Q^UPRhHoq1) zhI#hes05qOP`z-H{UQ+2!%W|*8A-5&gN0YGfu{q5>=W%Mbut_~)@E!P)DK&J;E6;U zO1;vtY!tKb`qXIeA}ZO^l<)5n(Tw?hKDQE_%C34y=E&QK2O{#7b-p)49T!GDf*!tt zkoF}MUC!G5E-qw9UXasIWW#QaJY;b0w|jcynJPOSeyzBMbmEG+ROnFn!|akyPn-a} z=aUY#b4L+lyr7?+1AU8miRXIfw>nfTJAI2JdS8QZyNT1>G^H30BLl^ zvitpGb@Rq2Y$!t)E~_@6IOsb=W*I3j*ys_*z%2kuRr1lE7S18S)w9WKRLMp5<`2an zr4(3yYaP!%jpk16Z^rPvoNgb2`Gy3I!*V@IEQi4paOQ5LJpjI%6=mY#F|~(sJ39-n zHMIo56X8oG`f%Esxlku+F^PAn&swyfT~Glgwr}yOHJgGR36l+gy2lp}EnRD4H#T^D zbkCC>VIcWJBUIjtJd=_;C;3ITf%gyVYDxZ_V_?&D?W3O3xaFXOLQ|0r#H74G!LYML zo`7@CS0Bfz77tqQ7vY^@ZAdA#MSBJ@(~-y{Ud-?;Z#XKz75DQ4fk`V2X^3Vm_XB6G zUn?BLGW%`Pp6IU*Aje;X#C@NxzN1y-ke&n~JkxW0nGp7@-m7^tHg7VUKQC`n%Vg04 z$<_l*_((mowNT}qrIl#QCL8r@1VEHveMRhhL5jGu&evh6p8ReC^aUZAUl4UU?LTyD zAVUg(d?cwWs+I`rz`jI>+1f4#tQXs|>vj!ORCx}WE`Hlw-nAiPm)FtopDr5DZf*v@ z2?7Hh(`i``a??AHu;^&uY;l5LU16CS@)F;iZw^cMuYFV}J2^V~63l+v2&bNEj?))T z$st{}yu6oz_YoKdI0m~aR4@8@K1wf8TMTx2Vr}B!d`v=_!_Y7@?N-$CT`kEL`n?t$ zYj{cm6L#}A_YPjdW~c68KgFzaY{8P)P4A}Uoj!wN@fGm-cIg^?l|@+XnQ}Q$rZk_I zoqig$yG*j749--j9J6S6TS*2yqL8_(5uVqbS?7|02=2|g3+@074T#y%XD*A#P$fyB?2mOu{ETNb2s*W%D;o+qFTf2|(y5j){h^>)aJ~}a-Vc}55pJ7D zPU1Z74+ww3stlMBiS0RWFCgXEwdJnnK&oaGpd>D&=wGeHUB;jmh;d(k!=0)V8W~R7 zYAb!KH{xPsEMK@S2_Gv?<{sx#07~*Q*?4JBPk?EYw`TD$dDX4JeoFS!Wh0y zMAgt6_Ul;Wu-KQA6&Q@p-H@=jO!z%PQk@ey+l&lWpy+(nI7N7E<{W`qI7|#gRuNU~ zUmN&@7PQ)bE-Y4PIc0Vi_6;4-bRRA|lkrYLU^y-{I`TaBYtcWE1@akYL%RE4jBm8J zdlcDFbT9rz$!en3L%?%uyRiQ+lq{O5vO=@J30de%8rxbZpiLa5okYRp&`>&9P&VDl zIbY%X<-gW)fxHK!v*Dmi*O#Bu2XTZ@hAWZxAK@S4^6JVlftxsq2db&>hQ=yF88KF` z<57kSXO;meK@Wkr`m}9adM!By6FlE~;{${o;`Ww4OxhWfci6B<+lsEOUkqU%e-GN= ze{%C_t}oMMJN#%d?1i{E(%&V0*Do9^yVKs@2gbu>Vk4GxSqm(8`-Q0y>E`Cf)CM;^ zoE^?&*0=6PMGXHtm$nLky8nyz?1_Spx5wJ`9@~(Yjiw{Vr6p~?jO6gm4;qS4EaJ7V z9?totK#Ep6O2^)C(#gyVAEVjRI~D6GfK>P%#00P0J=3{xiE2PsV3T*W-TrluDJ#j^ zTPVg09OL7hIQ>;Ad{(;>?GxkW7P}E!ytTX$J@WEml)QNFQMF3E3^Fg32k`AU~N~ zU+om~zY+LMYtKrimwH5%6o^H%=g^FozB4=VgqvY5VozZ3 zq{rd-uYF3#2lzsPY$}nD)qhkH#@V?nDbxJ@1)JHVq+8+Xsu$CS_ZG|9Ek*atx|GpY zSu%qaqPG8L+TXmG*RR~{`wZhDq-LS{xZn-cvqS?~RJ$7`9!h}ER;RB4B86qta+N5p zaA^5PZdcSk5+e_=lx2MNo>^h-m z3@Qz&#uFjAWxK=G=2VOKo=cMRb5c86F0=L~nm}TX39)h3eT_#NT-vQ*EYN4&{VF+p z4m(~R8>*FVQ|x-A!|UFf!APPgBHq}B=voI)KEydb<~Kvol}oD0ETI3iYpgG{!kS!j zO_ox9iyO$9d@*5lMJ-vr2zFd$uTFIS#yffy*aj2=3E2YJjNlbB7&Jp@3y#0WP6Zh@3f)Pcj5 z(l;r@rPH8@S1da+ajt3-zGHl02Kbv@7jh-6`_WO4kD@YrdLu~|s7hyjNi#y6^uxD! zQ%Zp<=)B>IN(x#>=+!>z37^sB5unGM@2DS=p14j}@k3_|@>CTN{k-BjGmq%p$KL^M zN>lMtBc6m&BnW@=MWsAI<{6(>H4hU0%o{nn!%gUN`C`4<2JTyQm`yl5bc@+hntp&cgN zTLYiT;gDI}U}b(nV|?|%$6@1+9+t6FinQBxIg=8eRzEgL+9#U(agOfiLW0PC?UcVq zeK|(^SnrjfeHz-+FlH18HVn==`vl~G^>**LDVk2y>B2bc>KNrYQM|D!dSplWpW+9f zNDUP_czSKUbH5H}7+h)iZ{F z1SmbQ;W6L^uzj2xLf^9nb<-~6DCb@kI$sbM{0`#}gKh$hdm}$(%|E3cmI0SJc@0g!6mAj6gfr;ec9wMsS8URbZb}J~OpRfN8#_~98|x;t zX7OY*%vK);+5P(N)okeXMwM#Qd;)lHRBF^py5XYs znM6TV7QRe>ztS{Pk-0`F9xL3Uc*zfd4Bl(CAd-;e@Gv5XpJ&34SWgC;E4ijc0_$rZ z*(C&zkM^UgvDdEqaZ8x2_!V&^_(c)Trdrp1PSG{EHN`6& zt3|t)B|n|b6of{cXY+_ZC+3?44k>x-aQ+b6MtU3(1@8mwt`cjF@!gqY_Tb>7BEA>OGIJs$;{|t;<)dr^ zioq{<7^C=!;U6{VPt*yK-)GUz_4*uX9J~cqfpGy=Qidie-b@RCyEk|ROFojsB{mBY zlUg>al)qP?myqnvFlx8(iz(iDQ)&E^VTgTGS^LrLUs?cvxvpZM>q@#6Oai*DMz!UT zrAkyjy0gMD)0T_c@Ng5N`;gtC^%QJ$p-?RAeuQ_(yXVz}&Mc zntnL{bKP>roW~K(uq<)uQNx2Vvu7A!_{*|oS~PnYqi6C&&1V)%iKT}BprSou5(BAd zy-WI%C{O`NzKBn+52X9u;XtUxOT}`Xj|E(ikA2wsjtq$mA{8;ZObMo6G@1GK@R2U0 zT67=^vBSjnI?wsiVd*M2FDv+vq+O(I(;_15Fj$S|hk>#n+%Q6Wu$HLI#!hJ7{YG13 ztWL2xEzs83wj#b@+0I*%;33bUzyt)ff*nn(TM7%{YP4uG)2Byc&6#pJFHxC1)z0b9 zK2$T5oN_eFae`n77|JYF7SqF%!wrx*^`6$lh~18R<$>B?)qgte7M%I8LS1*dTe<{y zq67FH3$!LQ;++juCYSTn4-^(?bUPc(9v44({y_XE)eeHj2wj!{W>BL=xiQ}=KumIi&@>yE>p_9z_f;!%K$;VWa*%O+uZv1q;33{6y z+G8tZC9qMdJ3#}*_9}q)v7i2j?x!8S1V?aqg3O@eVmI7TZ)M)!r54-Jv@ISC4J!tH zM$=yYxq9|;D60epBI3nmvp^-m|qk7)| zd*z9Nt%Am!hEM1=4z+AuCt^{*b_Bng7o7lQVLLJ)CH!IofEs~k*ili)oa+lJkjZcA z`Q{O?wb>Wo3A>(v{W!()F{!rj;o>tCD9lGLq#FJZhpB~9BK9!2uFTQ$mXg(QKKMZ`xzU5epd%DR(@(wLjTE58a--8NJOJSJ?S2fPp zdrEny>!V52(DVV<)-(z>QoXlRw2~7uRqQg&krqOPEHXh~@f5X%HYuYM7Tl5EJ7L*^%C9#gjvIg5W#8s;`G-e7bIIhBN(< zbH!xzhU1|g1D^$luN3f@omApVL)iAzJ1=f~e^7GKE-<&D3=vrNDJ2H&<(9$8f`>F9 zj%(vM4lnGcyF+ifP~GA1;nQdeBd#1dz{)gv)}Eprh`lHX_Xu1x+Q)*$Fo>jpNrsV= zNP!c6v$Hl_cRr3Nu)LF}&a58^gRc}T+=_G1XGXfd=jppc=dP&gmi*J6+wj;@&d2=l z-5#lqr;QD*sJ{SG#7=vk2Ntx4(~_GuRa4T!w*}<`_;nb+@m79qRV@X~y5!lM4-{+Z z)*q*t+cLd=skDkjH%O&mEUaz#TJbTGxQFD6v!Jspte%4uGgJV9@5JNl1Nqw@u-^`J zYz1aO$0l{P3CuengZMggSJc*}MlcQAeYT2MUhci=DjJxS{WU;dWg;cyHkUWJ`~bbi zbMJj`IPM=NtTml^DTZmn+4p#aLP^gTG+i$;=B$sjyE-3JfUlFrcBz)|ouIu~wYPL! z!A`r20^-eYN`860%imhyVL7BUvi9Oz*~%Nn(kIbkvy%f{DMh?WyYnRL6G+qW+87Lz zQ*Eft!3(u?GKk>T{c;?TrGU>nP#U5Vo6LF=@KLVg?kb?F(x}us8zl}+Zse9s)les7 z2K!x+w;=A_u_?Dpz*5+kPy;5hp251U=@&o=@5 zE{Wo`>+Pm$+bcuEZ#^yUnSxGx#H2sBc|!Uc7A3SXn&&>9mr!&0s5qYpi5Ku@REPqA zOgx2AtuK(a3>yxvShr|x{?R4rr$6@Kx)>|a34N|FON8-JW}>WE?l3mW#Zof$1u%ql~l;3$4e1RVp~96Rn!so3z>a< z=3LR=c{>u^lVxhM9R1>o^(cf)4>yRT+|g0w9A67*x~+6!l;q_`sB$3_b-{j=d@Y#D zcLSJ^;NAWcCEMx0UQea~6B5iQyhOgbq=+s8?iKm7q%}%e8*q*kuY)57wg8lyez!{1 z4@3Q1?&Jn5sx;tzB6*wNmDFW$-*UB<>fX+Hg6l+&tnuv5lLIcdi}V*O=q=9UtOUBV zyP7CyH0DM>wgv>Mo&P)!6w1Kv1ycI-38xTjZ5uiU!}RlCx7b}gmV?>exS9I0?>_Wx zX`JIa+StN+C)S}}07BlQpybYmoNW#n5K>Wr?sN`sK>Z1UGQ*+z`wm0l4N~Od2)T7k zcm}g1wvp}0e-aKsNOQ)vr27dolT3aQhl`Y)5uO;SaSiUr$w4bsSNg$v6u8;v;H$YH#FSXBES zlVN#3;)@jfp;Ild=Y46UVrERm5}i4lwS|H#ERlP4dwI~ESWPV zBRwC$c?phGY^Cq`^~L|0cox-gDnGM;ldG_!u+ke2zXisWIpa;VDy8MWV zNHw5GG7g>Og#sn_^+8#B+FC~|F8kse7G_Xf#_jv<^2uvbdK9l(5jT=X3;jBpwX8VN zGkkiA&Gx2*ZrNXFR5^O|^E0f@uz#BA3WJOq=3>^3g2j@vKy6f5E+3)P?Sh|cXi0DS zwepGEWArsnT1ff{RUDMjv3HQvxx^A@uI=`7%PtUc{oxJX2s$6rtoPI$rckcXE2e~( zhfRGI$kdr^@^Y)OXk$2)w(xvQII4pemf??Fv(=tO1L z&_)qurBtyL<_v`G5u5Q$@p`s7j@tFu4tz#jnLObkPZ98P7 z(*X?)Eo5+EGud(6VKmj~K8+FU3cZkWF8-Cz-T9M_u;Yg3bJosYASo$m2qf}HR_J=u zS;g{FRwqZ#9CfsY8?7HshhgQri(xHF)C7vxY7Wjn%T-&fj&P2h(7xY%Tt1pxP%lM4 z)<9zw7DC(+761t#-60=O;c`A3ZTn>%zY?&rvi=g1!ezfV%DK*Ie>R%JY$83eJIBPl zV$gZlK63ecoKtzck4EPuV}2d5x_(;w%7C(weDVg5XW#mIes?T*S=RJwC)IiZu4TVj zz4!IU!2iSEdxkaDZEM4dB27?1K)Q+o0s>N{qX-Dnl-^M~A@mxWpn`M(>4YM^gh&k? z6e-dnkVq(@hZYb5p$6XQK4(As?DL+zkMED~*Y_vaTGvWe#vJn=V~%-`G3R`Y(>yx$ z@!Yu)3_%gJ#)2Y4c0^}?P7qjTF;n$tA8o|4lL(a7jeIo} z9>L|M{G#LN^_b^6#6xdgYAec*TiJvCz~0`5;QdfkgIf%H<{2;$W8mE?(6&L0OHLrA$6Y97D%kpgq01urjhN#V7-__^XI5$&^7m(w);HQx}@vrUT{^M*zv zxPy$myzEK_Prz>58goc})s(YYZ?I&g0-d~%uHpKFfmEqyIG3hpIA~@;Ns(ERYev0M z>vQSHvRnBPMP}t&lRj%TmIxy$tXkQFd3&(0+kx%c5_(M4O$3k6 z&0!y-y0Iu|K_9VI;nFW0!-5_;0kl@AdIfW60wC-S%OB0PhP!7zy6vzHMmq|gg!j}{ zWV?ZQzy?`(QnTj#5sI&9QPO%zfa2T-+mP>jmZCpUch7~J%F~Ya$uGTKxEqr&y8BIQ zN$w&#--tV_B5$Ap;uX7oD*bF9T4{k?!|bE9euyTdvpnizEj#mDC`zYmWjCyRRRG>f z6o6X^612Iad#S@hdTlh5zSjg_DD|52!ID*(H?K>lgJ${7M9vt@d}p`QYug2aQ-;{n zC0YvyPW%Tb=##kuG12hb$U(2_(@ufM@MXslJv`~oE;R6&cZv{Sc_0B4(;yJ{RxB^t z+-L5i>j(>k>1CqD^v-^|C(1{i399{p#h|Dq)KBPvI~v))4cp4klp0*PiUFN2d^pGu zQ&^g`xJ@pbdGR6Fahrrzy={d4X?qZlnPA0Or0bib%)4i~;T3MgJ*V5iPVb4JZ1=y{ z7kuF)lg0n8)z$^S6u%ya&i-t>9;x z?XG(3t=F&be^}Z>zuxF#6`5nTQc$}V-$kxH_)6g*Qh|Z;a=fNLDRba!TyOGBB)o9k zVL7t_G@EAU)s6f**H7+S7F>B#=9O|Hd*izN-Y_&>6x$D7kGBgd1t{)cWv&H50y)PT z-#07=XTbxJ;VRSzNs~9+4k3NJn{0MHLGPU7!LPS%c%$zW7^4qX!E?w(xk3)v%0d1I z+p?#b><6uZoXUdYYq|7zrRb!vZgY@!k&@K;t|)lq)PT>!w`^O2XW7a9B`^{qq%qL` zr?24xXY$7GEwWG^q3AGn@+J4zr`-caEDYi2c`Z8xjhUUWfsq|Q`6e&KbM{gYPw#Tq zdwARCDqp)Hrg?B^t2XDCQTTb3R$JiPvbjwjA|J-tSt zrpilWZK@~RkAMX&q`5d&d8itjmPyY9er*mNr$d;gm}I9R&lI-gTqC;PwN9Kx2^+zS z>E{3|A{hSREyN&)e%?mfOWgrqZ-)Rm5#CY!2j>YjPQ=p>3hnF>N}Qb48x+~cnv&j` z-S+n|I5gF_v*-{W^r_Al<__hX$o2f_jh7VlZKua~h#<;~Jpg-Hkkxz|a;?B(s-jOT(kHk=A1Po|2s^{j#!wX8 z!UuCrS5_ayo+EiqVq|uvv~nk;W1<7Yl9kds#p4^0X2b9+c7}xN8 zTns)4r>eTMswphJyPAv`rdtS^r;K5;@oDi$?mC{VEgElrfn;B(@YSsyHN0ysy4f77 z51er5PH|?QN}|)6Se>#mPtz^G&>gcW=4df5M&|rIF8Bney*Qz#8Cn>fij*RyRF|Z% zpGu9ad0iy}{azKOC=3oUOorGL1BPOU0o@luawvP@P|j+xwg|i|O@#F&=OFRqTfGrx z?5;hFZgamK+KQ;{!Zri=%@;zLLE}syU*g1CM$oifg^|1%vK%2~7xF6R0g_WQ8)jXl z_@%%KLj$BTlP1$OmqnRMa|h{V(R&y9W$5ShnX;sCRCr4|t zBi@K!SC{$D^!fX4pT2FvEC8K`ELg(tc~xNA&TJ~6OVS3hXHUL1WSu<`K=848G-;~w ziwr*tvrEgDq4sR}jz^!9$8-^3ms3CfSJrZ zpEO8}F!Fu`jHwAfd?Y8cUYu(qGxkDcY~Rmfz3q!PyLva_rEuzD)QnG{bocw%ykIom zV&*tyQyl_kIovXl?4!<5Ps-nj4G)xkcs%50Uhn!6#iEWee}sz1X1`u8?5=mT$e{VC zXK#8i$7R9CMd>6(6^o2Vv! zrKW&=`Oa@bHKY%@!ttly>&BjET>{0~9RIS6g;SHGdvR$EGdzVF^=C~nH?9&F6=C-* zLY?*i-p*+UCGg|U-d@T33X*-R(^|{b#+L7!*(J)TRb@!XZ@xbFzy5Hbxcnv_KibP3 za#i8n`HTPYag&UcM&{>=c7^=mV$3l52^#WFGgVvRH2kj@w+mf=>M6I8NhRF}DVN5D z-MV1(_^%iIzOrni5*nQeLX+P4rw9M~_NKwb8xDd7c+tZre|?cC(JK`))je6Vh#uj; zw@!hEB43Il)+|d=)SCY9(*NyOA&=-S^IIM**;@UPLK^r#Ag$yW+CM2pxle>NbN+!Y z$!`vGhrFX=ENo%t#1?2MDg388|7ym+x;o;%%H#kI9{DW(_xAmz?tgWaPfKj5?qhzH zfA#i%?B3tsyQxUxiR!z0XXH=dLVoju+G|b!lDCFL=lTbFRS3JdQj>D)>8D43CS%e$ zF#ZKaCF?)+?Z3VaxlA$aAgK7^$(uiuk@vh5wt)7P!-x7W(vK-{AV)0_@(`% zt9{r0!J+*7Uhnxp>}uhQs-k};WBxDg^WIASGduloi`dmRu|Fxl!KiuXwuIWiz6eJ;E{LudO9hC*Tq%(4FohbcFEc{o0`jw=^ zJHI8gH-J=~45FZ#wWG4!K-f&!I9kj8xLO{o#^$$79s_1&6Pl=6`9L_pr@hv6E+KC6 zbfFhEYVJJhd>Sid`*zTtsbnND29z@|w#r1X;PmP`xyI};TKZbG`Y^?UeRvMX=d}~O z*P>-_J3oNLi0iMWibSc0%O=WMIRx(XQHgF(xd%3s=qlw(;?IPuHMHKU-~I!fJ|O9+ zgPW_%VXuw9vIDhc>wKbv@+004H)pN<{GRN$u!7lrBrjm#WiML)@LJOKwF5M$DYcU# z(GsE1S(X^(dD8z)aYb*4AO3}#`>Af#1XCnz1@S_o1n*Q_W&%cNzn;@N{c>08NWX~_ z{d%c$;zsbF3`(3IOu%@#L}>2>Y3Ye^&{q7uvHuwM9&NnDyfKdUe)Y_$V@sIWGWtE$ zfVhC*@OxCM%`#wKOF<{r!$$>U0zC5DrfE4$&M3XsOB?;mdm#h$C z(r$pydj(?vD-lUZAkjEf{%CEjS{0$*=|8)g6@Qv8z1vp>Ha0?himTguB5ol!u4z$N z#}tuD@dq|`Gxl|nflg*YhmF2LmSZ++Nyi|+r~f^PG?%+M(7LWI# z)EXwwOV^`_nImBh8q>CsG7dcI?^0tk1u6s9Dc1-p`wl1QWcz#-FANp%2x|dTiJM{^ z+5-@a+z{&{C~1^pE<#{^*lv^QrTHE}E`%gTR>jee%lsn6GPQ2At%p^dn>3R*1c*ds z8#&k;z`Zy<7jyCC-b>lFFP!(B)z1=3eBXebF)~;zVn&-b6jfK*J!6-3$Hp-uizn?L zrUL)gMnDwxEft{q%*rCYR9^M44s#yBq0~Vyr3M(->U+HCR}Gv=GSS`u&!yERr$OIw zeuWx{v8~axJ#6gAq6dwAFOb5-JlSgYT432WINxMtZm%DQ@_Coif@Wjx89Ggdt_dDa z+FYQAiaoX3dm?dv8t6>c;4LDYi@TTpVudMYC@o{dWqE!YZ=z-Qg`!hxyYdy=(T}HW z3#(4`$7X!$%XMk|cKnL}wi$nU?kRJFj+sv?(&o;{E^Y4n+|_>R0ebPFll?qSSZ5Ux zfP+s{8m!g?teR{z-6Avnb>r#e8`Q~;+z{WZfpdO3Dl5h7U%|&4+AU{WtvU$)^^sPD z>j%mvkDThqLC+(7GhYv)Ln4viwg!${8IXeV!mn~DtP2UPHzjnoy#;GRnMhaI<}UM^kTc4?7$AZ}u@ z9_i$FoP8PP36n2|DVpBVPN%!KFl~29O@_P8r&~b;g`U9_(AwSk_HdFa`W`u!ar$NhQ5QLZTA*r8w~AEn#=cDVkf2d)}lz$#m6;}gDLxtW&LFdTA!l= ziJtIi^w}fGNppA5Iov$o!bRo(H3mao-n%-T=VWL*V(sVy^DgSmz)7K8>x)O`67J)^ znY`FMe8sijNi)aL@^H;0s1gHE4e82SIGoA;etW1lEgJb`eKozPbck!^%)!>b!|z3+ z>nMvH*f$TacLyQE<{8ZbufTY(HVAHxPg*#Q<7{=6Fgl_TZG^LjB5$6bTV036A9%fX z+MXCqe~P2TyL==wnx>U=wD+Ufp#!sBlg`b4wijgYBrjni)Qix`9e_Orfh!_#oB{oRPmS2pn@&HA;ho2^yj z6TO@A*H@r+*UR`tIW5SBXza120(yh@@y$oICJ+}%K>ogj{rrz=L#8<7S`BQk#hFd& z%jOG1QOJ+QmX1Z)oO@XTThklR#M~`k3*u?d2A(B7VNp33<>Td!pV}-o(R?0tD6;ux z?^CzeI~)4{&=g|Kr}?1xXRLh+lrz|@q@*wr$-i7=4H<;WPRAH#6I}SZM$YW;<`aGTMGS7a>w}5cY`+8NC zWe?G4Gn8`+MC{}N7)=bBV14Fsh^dK9{@p#FocZx-tZ_ABz#z!4RrzCm9=Y*h(7s)3 zf+N|WcJo%SOB~);Tz-D|Q!$PI3)pG1B~0DjVJ(s{&MY?}`R^oLo+OvBH=|xgU;&wD$pSk0*lno5xGc9uU^NTc!&2RJ!}4 z&F}Br_uV Cg>3yA{h8!(66H_E~N}3e+G*GKexKG@)O!h2XwrFuMtYXtb$`pj$)| zbA1hnakaq)Vp(IlMSNtsSqo}5{i!xy^Boh)7JC1|g+EZdE9nlXzx$}se&o~eRNm)oe}(_Jf$MB*N(Bc zr&+8H7bAcT`tl}qqqc5Flt*g`dA+zjKezoG+}&rGoq8~W*GNY524*Yg8$g_)Dqgjy z(u7gBj~8ZT&2w+HP?EQW@Lqx|E7XHh_b{z_xME|!@5Vd{C~AHlw83(2aU- zDC2CmQ+tg;!1Uo;9o5mlhVOrKb;Fz_E2W+W8hzLs15-?65_9hE%c^}Q1F!6BW@}OZ zd~zEw`!&Z~Vwb&gi0;{msta>Jxp!0Wd_+gnM)YeMLg@q$yq&i_c%LW1ID2!LJ5HKy zEa6*|pw>8yv8?oK?<3}Hyg;BYFeOwI^$M7IsNd{EqtINEu;L`oDsT`z0Uh$y-x1i? zGvbg^`*ZG?UZ@81JdXF%>Q*iK7pzO?F@&QEU zI44HdqU|MIOQ^zNBhyJBlfw`huL5u(yO(Ij!?FVBp_{?m9tlM!X0od>J$A~}tf{^W zMbQm^U`6fP3`B}47_a#fW)2=LH)wIomnYBifz%X9Ghz`#j0J2mgxA|U365CL9VzYY zLN%5@VZK6c@qCcK2!E3BZ$R=-`nZw&ixx<3N>cwbT0kkJnH^V2TW3F_(Cp#GW1A-W zN58-MLGm}j&BpZqNN~S5BQRou?|L(!uf3-|=%s5T4=gHwQu&iKPcD|3K)> zspbP8G|m81d-4&5PvT)JdG&L67VN9&kHXxGVaO>Bc51llZLK2ufKPC0IkvWi+54~ z&F=47yav^sg?;;tX*zj1INlE-<<`TqrvvS30mSFkVr(a`|I_Dulwo zEWa1Ewn7@9Xjx2N100SG=NRtp)90B&i)L0C$AFK0q-LwC^-QI7X#quf{vFkzt*Pv% zUN4aOZSZ)syh#V}>BMX!attA6#0e-1V~{H$<^#@1j0}g=*c=`ItlzMf#+Of$j&zl|aT`nZn3*|c@U(zs|X;;1(7&ta@px7e2<-_{` zbIa_0nV2Jfp;Xde?N0WzhSnOkEzt0M8QZg>^`HJ2`!gAgY`a&`o$kLW|EADu5~B%( z)NA%9F~24dALxm2>5H6xa=ZF@PSN50Kp6}rFlx0Nvse&l^F~hSOnK0(RB<^|Eidmv z2RiqbW$#vaky-9?eTl;;-bNxdIF2~W4l0jad(;HDE+O_@#Ai#lss9lSmENQx`<$x! z|>Kc@QxC(twRlUc0m*vy7M)@e>ZYvs5EnW1g38AZq`{wiMs6 zRnqJzRPp7~fkc~C97;n+E1{fh51GGejP;DtgqfVuddr7xiorzdn{+~0KN zd) zS>N(1HoKWtHqnKA!yqKbcryz7*`TPDY45?3Pc1^gykwJ*j!2CmlbU9_pXK1=jL@u= zb<-*C$Vv*iru5Dz;|eV?(Uq*gfUMTv@i958mY?38@ZdR!I!B_YcN0&-uPWtp9}Hnz zmr8j1wE4Y9?Kg3LRmGcF_r}K>Mfc%3W~-MMU0PRD=Lfvanbj{cg^|oW{-BKK;dZ!r z&H4i=jrPPnO>Wte!9rW{8-|KJ&!yN!(}B0P&diuyZ7<cZ?2ClJ+W6 za-`74CABGnqMUL+vMX$SN|&ZobR}EQ;&i-W;*(m%jMw?J<0p{g)Yenx0)_i9g)%9$ z2VBVg%Nr?Agy%cvZbq0Rh7CR5xHPBomL+?1rVgjepfh&zk)GQr%P;dGv|jyCb>>0> zuTu~f={e?M_d^TKsLQi?jebMhRVxG3|CGjWnb1mF;L!S}Bu~Q(SZFiOYC^jFW36u> zG^cnQ%6M^=~KY$fy9+gSzQU&5o)~c=!JO(E7@w4 zE*Ij;;?==OQY8>#oTQaxStwF?V!n%i6~1`22N)tHqBaqy=UiIBF7l$-Ym@$DR^q6y zQ{qX*KqwXg_B`W$o+f92)APaLHQB-%pzxPNCp`j1h}yoQ^;%pv%zi=5?#9_AU?4s~ zL#CzBUgY@ z$!Xg6njk@)%*i8^c*yUsguVThDEQ`l^IC^mPM)fX3(&J#SHc3GVm|NpqDnytPDEdk z+TKwEu468}X4PP;c&BuA&3wORrgip`|7riE=i>fQtf*R@yk8k&%3A)DBp%w(}kL7C%M7>cW^QRsV z#!drl)PylQXoxpUxqD_hfJwntb4PhI;ly_6DbzbMM}E5ad# zN-d3?geqFyV?MsU?`%}#8~3ZZx)PWS%9c$GN&}ibsznK(r&+S#$IJC=RlVDD%Lhgq z?DCG2mga)TubChMM-MQsCuVNh`rE`~@42%YUw9hRa`-5wim^Okjn=5|z=i$?E<4tie%;JIk z^+MxFDPql{=gF!A&Xl1ex<5DkGus?|DVl$zqxEiqDw!6IK!(++dqc^BGkZ)J> z@}0@n9>H|LVwz3$DqV+qjwz{4A-uVPwO54gDehTO8EAWYE$Gap=ALPP9bvyO-BJoG z$(B0WM6*6TXd-}=Dl12rmv*EC7&QgER*H)?2Hag{UK|b%i@Pt1GwhsK?^@ zacoXx8P(V}qR*{*vH3B~Zk=p-ZSZicWJ74(G*2wX`!)c)*!|^W(Wy@3**c*D2ljSs z!&;dEvc|hImQ`vGH4H&e%CiFwV-C<&-@;g5w-7arYEh)0Ceti2xq0)o0-E$vE2P%n zEWzWej;7wwRu*iMoTl-)%6;%s1NXudy3!q)zb2i1S^jmBIg3jb7BmDrcJ5C9JtiKT z5V@IS|0|yfbKJv~CjPPw=|cNXTH&%jkIV<|r3+UhoMj$V{~B>05hrL+K-{FC9f95d z+vlH-vS8{cvM?coPsu(cfAAw`BSL{M18T;&A9lU-6-eBC}xv@S9O zgYVWYxT@a?P&yYvxH$|1A%?a$%QkD);swo%2i)%J191Bl!*Y^Ws4kB0RhGDZ9Z%Rd zeFgQapPl~MK|1n$!p$jYPFcf3rcX|A$86|G)oK@UNF12S>q$&%T>#agOUsuk(~c&GQ@S+5o&eryfExiF2vdjLn54EnW=52s$Khtxi(3+c)0fpGn+#}2TH zN22=K8NmswLa7H5@&sUdEF}0yutn@RHdQ0dSwZF{Vja8!;FM%wH!6f&jXBk^zt`r zmor3i0gWiF_=b5)-_2IU0CyO2B$UCk-N(+DLsH@ zS2I3oOBkfsIAk#`(#j0;2h=f*0Cq;D`|Yner^ZxsBnL_n*Iv2W?WgbH=c$26yQ!Aj z*2>A0A4_*KB)aljR3}+BhZ`HJ_XZuf*J@p+im`$f9@0riynH~^9u+Y)HQp>Ox`&A? z!w(*2(Nys5es@8ni5^g}BwFLf<%0SX)uZyHHZYHNdy2ZV=kr0xmzK(JO%y-|w#~XV5X7c4wri!}50f&Zv%_F*j4$?upBC6#sq*PNx|kdQy2()2Hv8=cn1Z`7!k)jZz?-gHw}N~z zh975LJJ8C2^I}6Z$+k0l1@wX?!lW{u+_un zbpuagsYD$JON?t^55ci?s#j;=BH$KGIbf=4H4_S|sHNikQm)fK+|A51eiVKB7-(-z zkjsX3eUn(GB0IE4e}=c8h3BD=e z(zR@ZFA|#6KFbc;06200AW8ssSWGPkUJ3|{QTS40BsDtKW-9GpSwvUL?O$f#@a7PP zJG#W?{r$nRp2lhKOV>CSqKzmO&0#k}V=NTPK^)J9X{XIWFGbvvA&%FRv^I`jhD`R3u zIA))j2f3PIOFC-f&F)@5ruo--^p`R_OJ%i}_kNT;eh_E+*)V*dYILFyEAMI%A?>`% zt5FQInD8IFMGq30%JrF~3l_x1FoC|qG_tRu3OzOcI@ALw}GuX?n30{k~ zZgXfsNjSgQ#>9>M*g7qvj0+lSM!igk?26TUIUk}ljz3vom&TRmq0Xn>`A;& z;4_Z5>jTRfo5*?*#*9s&k?fCqPM1b+v?vi*Nt`2TA@v5L><6ie!q}60vc1#0R%vjN zrt9$HWO)~nye&PxgHsM!XjKLz(m{_mJvqWdEZfUxYf*%|`{H9T(Y=N%T^lgW=VY8j zlILZm<(S!-(zOau`jAIcZUk+6U?rq1*)O|p892-<4wZo#y+U0Z)voZ2NF(q2aN1H5 zixOLu(bf9zJs>Sh7-x0cO5$Oj6~F=BVsXwi$VRsxAoD;@qbaaMgF&Z5BS+zGH!BBI zE{w3ZKkoLp=$P{7QJ$``$6MZk3mQ$22UN@Y?E}tovXIl)RLbNeCTi&D^CvW|LzTsd z$CJ@)?3dIJ2bSZe$u~-Lk-+Ko{->o}l7n{hiu(>mQx#T!q5S+H|qJ* zT^Wu|Bj4v)e#cbalm{#3AHhs_#>p=?D)<)9d)7_1R8LUrZ)zT;t@$2{N%-o#shg2k zez$zr`%sAuHCIRAsqbTc%#UYiXmV`67wo1|%g`AW>u1EJ?YY5^W1jLDKP^hBS$~Ii zINs7Po`(OlwXs7cOrW;krHFntkSy1pdnBNI9O=Isa>#fz2F}~I0~cP)7Sp1$LSrjhhNsz zSI$J{q%Fj|3lFawln8n9=ofF~!}ahDKokxmQL|TJQrmZ9EAl~|3Y937Y-hJ59&Yq` zffMWO39nk3TjHJ4g*aCiWYuP6MK)y&G|y-mH35F*wT~6swPdA1q1g-57Sm}Gz0Gg< zcOFO>ISZeCMVAS*80T@ZmsYKn)^n0felZ?D4fP*=;5;FLrSZ>wSGcgf6bz+>J3wW3 zTQsIC^?=NKPG|5NgI9xzLm*}f`ARR53YOn3C;|^UC=^iVKxjFj73z;gOyi8?g1LlFpj5d&VY=7t3Q1P*r(Zr zf|l=XD$3H2=U&UnWT3<_-^uz61;*~0eoI9kkL1$$dzW`|2TnM7G)$JHaE2xF448*X*8aH1+8tu&LF(c;eJm2u)>nrURVy$U&HtTpa-D@=W17v zue#z?*V)|iGfuiNyCp@_Woysi{k6I!YHhlL5a-M90Dj972$A!1THszstovFKxW{Pn zh{d1PH^m4Z2*Kset1dKZEPFM^$5B`;LH65R!-xsAPiuh{xZB2mqX(@OEN4;1R-2Gr zDejMyang!bURu5%=m9sABufRh<-U2wmd1))5J-*pIu6j{9`qJzSHAwYDBtK;l1?W2b)LZ{>NOKZ1tIS~rTCV%fk7>= zmXzMz?yS-^KwHslod$}~f8oe7LK>wriGnQaK}WI|ZEPC-2%B%1n@%WwK0dYc5&@xY z*U^v{1Dr#EjT%aemtgx(r^&vcZ|RB^$n{4w>LV0eIaTKCPdc-gQ%)2%w4Ak@IGR*7 zV2xVF1x$MLmSrh`yh)cXYS2O0bQJKWi4rgDd35;Ph9MR}w73m0)BS zm3^hRJMqO@*+LWIWoNx~aULrVtNYzJWCL?;C-c)WDGebni_YCQkC|B9`Y{9Ou3=AH zRY7iB?%8;d`Q2RG@m3h!U0IrbW#Jc?BDd*f+g|mwW)*^FEihv3v)#KTDM7Vp!QU_n zq`&dD4pD5v*D^?<@s+C3ezzF}v_ReH?Zl(oN)tX|Zc9@3gA0c5ri!&j?k9c%-(2!; zaeMGLDm#g!CeCKbnR7(Z_~vrgZdZ6VGPTn#LsOm&)v)_2(4IO-Pr^h;fH{x4Z(4u7 zeLMV}^IMa=mr8P3t;)jS6FktpL;591YBKZtFq(pjAC72lzZ=5owtn@S51(QZP)AOZ_`@>7ME4P& zUV$HIC2I2M!1PG3)0gCY9#-Gb(Rvx9mRGSk-_@l~gzBA;*0E|zh9jAoz_DsBbOGl6 zFh;XM7t(eNA+E_SSss6YVdZ0V?6x6tAjUHJ0;IB>>5D!}n!%J|eEH~+lF9V7 z%=JNoLtm{edA_F`-TMvcZH?029swMKB#&z4mi-iYS z8ux9|(O9g%h2Kuo)q+E3_DMT>j@xn#ZSLhM)etSYzgsc7F5_`-&?KdTtW|lkgS@!2O9jcxPW_8 z4$#0=v7;!vqzJpHcsDB^cw<=wBu%2%**LU_G`8>DNnOb0jm)75~_ zW!OKF*XzJ$p%&SY+tT=G{grY@x^LHrcUN-^3&$3@m}5BeUMPp3vX?7$WMR-?fJ2Sl z*dSuX2r~hlt&yTQT_kRf=v%7>;NsJEkX1ro11S}<8Lm0inHyP3uzEFbzk*)!2zHaM z+>w|h-BItB*@qX$arm0+v<8(@euX%h0fHcLEYnvxx#RSnzl(GKQE4R2?x*gbE;l^5 ze`WOLS&QMqvYqX$v#Izj(8*gd)ksjNMCP}X%EFV3)#2Q6&$v37z%Z^X z*}FA0cDN|i)gZA7F-~oE2er%hIeN~pCdaKTf!F}H$^%;eCvCu?CLEmx{X4jklqTnT zKOnMW?|VHH#(r0IJh!$HDO;v-rLE$tLwMQZhe|gft9UT`=ZC)7g8r^B1d3a=QInY^p}#bw&ec1l>^eq|%+T zYK@|nu;CrN^8Rh-4W*)baNH~8W?w`45D~PA0UYKe&XUxnO%OeQ{b`8TrcFZE;7KSU zYNJ~ACY{rG96nckMe>IB$K>oYd%_Da-Z@0Edhq08hFL?V=KLE(* z_>~iU++376=*>C#lH_;NhL3c9+N-ZO`h)$&umpue1L$~9fx#2zoXHfkJc*R?GEm{F zSv9IT_FY?_!Log^y8nI`b~Z z#dGCDLoj5P>WX8aVxE}9ZraJ%Gu{#&{kccWg^h&bqPqT;N2C%K#b2jg0EC34xeuEh z!EXC=G|f%-3d24q1Pw%_+een-(#MJEva}tOveLBfjItO5P)w_?_liAf*8xvB*@pi5 zXjo5T=T;ERFgysyPoE~~k2c>@Q)jl3N zC)~B{#yvzg)z=ShWNMk;42ZV{R;)^5eYXvh;--qV1hThO6IXRvBx|NdF8dIi(!`tf zORq7S6-8FB)jD`t@r;-(pT;yY@{bo&+&{A3{Q8tFpUG!s#R`IS$2~e1l6!MF1*Ffb zdFCtco)AZ5z?Yh~NpVM9$IGJ?ZlH?W~g~L=j`?MH-vi~E@g39bA^0M=p-rWI~_|I z;2V@p89f!rSj8bO*BBiPT?AZy9`b3`cF|l!7`iWlr&9TG#Xih7xpem#Ce;qw^3qYy zTw1DTELb|;=b56Fl*@@~X^D^WJ#n_{E-jA-+R6=7pXH$z#`aAy)G-8QUZRT)abmpj zvrgZ`I8wNqi;T{f=^A#6u$d1BnvKne-Qf1xeD=u)@uZqfs(BR}i>vZCYEtl-qB1B-X1a>v){}0+(Imh z$D+zGN6pH`s8nkZ-}YqeBDmn#GEYpR~As5$8-Pr;an9S1M~* zxwFKWQpkZQXKkZ9BANGC(qpPEFfjdI5NN|UDpqdL za~MeOGi(B(_7L=zxRGXDfRA(5b$~FXl{%?gSEMu7PH)7v$EN^}kf+ZVf)K^`=ccQ% z2)_qx0mMXlGh8O-o3rRIzU-8C<(VvBO#RJ)g_inugK5e6ei!=SEH9tdeQrHjqStI$u%r1^ zG?1kLp*bw_kd0Zjh0K>Qi!|vEP-l6}rHSYoUt_as`;EF06O73w`Pk}*?6W+zOfvrf zv+C|MYM+hfH`-EdeQr>@0ARJDl}qdM3<2{ECuI}omRrJXYo8f6>oHDnv!3fDl_c4v z(kCZRr>dA(WKLx3oNRWR$D4tiSbWUIRV)0@YdzVnzEC7OR>u9Y?xm>0Y^PR#X)O3t zf@9N*eWd2cV(R{B)2f9#SSfUjfGEAcOeep%#WS(xuBL%ECdm4{wKOt~N!jxdmKTHg zhCy8W-^T6XDKJt%H2nC@D)YI@ExOZ9se zuTSxIrZG+L9#gxD*g94bTw;wlAXWd%Y+Bn`{f~dFK9RHE( zkK(jL7jiN?N0-k9#@M+LV0#nc&Qvk=j1*VXi^uCTw#MT98bNV`WIY|~$Fp06DQlBkPG5A>C4M$g;4M7zJ zL)U%+(jW76ep_J66@c15xnsWNb|Y;yt!MvEQO=#h?0|9Abkygkce8#X7Jsa}!1E`Z z3ByNS-MLpgEr@k>(-XOVO9*LwN=MPTwqVHj>UBk2^I7$d_Ya-@vfESVHq5$2Nys-W zevRxwu|5`FcH7rAM7>V)>%D>?rO-AL_}N~wD+WzM?|XC=Lps$6*+`HU^O}fpTk~EI z$bY0F2wuvfJA~K;7Py^ln$!^sA{5ozP0-&|jdSYs5w#OPCM4G06OY|oI+#&>m;`nU z8cV0MWks%syjRaGJ1lL=d+{x$objo#CP42+aBLSk02Ui?D6cEo*{bQjvw@K+Z}$kC zCs4z}_Pw74mFzqwsL-A*AETCQjM7hg*jiA0UFaZ~W4ZWy?O?-d4Q|Dgf%_r(HuA|| zfn{nS2(W8AE}$!^^S(*zpoz{mm1(vS+fP4&2*8V<@Qz8bFzH1}+|HFOW{~0JA$w|I zCM8%s}kbcWRY{~$aZS~X7$^w z_YW$YjCC2iEga*oaXcrRN@6a{34_JNb36ZvEyBjwo(6ldy9R~7ul1_bpsdw z235+%z0|m*9i=FScfvoTaBT@mOt(hfaQ{r=4tiv?-N&|}jf`bmR*k0h@aUF#W~iU5 zP~t$l^&^MPXS#f^4>Gb@fv~g4UJxHkmHe^{ToeC->r*bv5p9g;4x;V%t}~jXNLBv$ zno`+7NeaD3i5@8pgs|BbSG*!E<^9BL26{(5@bj9_3AwwsGlD}z-)N$;n>}#EbC5@9 z3D(ZrMj?ONtL8(@NwMKY>ijxwD0QEp-|jefX&G*Tp2NS0d&r;;Rcl*{_bzQjEsZnp z3_yjt8<}kGPIQ+r?nG+HzB2d25)9?c{82X3ucKg7EZ#g?c7z8aAKg?Zhp+!M8%i&d z4rJYGUDMQJbI_zs$bNQb#1l5L|BzNWJ6Dt|I-8XI^2_rY#}x>gC#kf&8Rrh5x15shqfy{c7XQ zkDT(n9h~JRXndPfquX2Z8Pj=VK_Hu_(ZrfNXx5)<(3H+ zs<;Jc`JzGUV1ep_2jdYjz@it?({_wiaVfLx-+6TA=vNRd#8t+~pM6`t$#EFhd+D0O z&5GAsAVi~I;_0uHWxqDtDj>t{c|w1`Hglp=h!U|Oug6gN*toXbtU+XrTiP*mQ~MMX zh4hAzQ<{_k%P{C8L0X;k#4OzA(w8A|g4w<`SKzl$1;-F|1S~C@&#RZ>K>%r0eXFQX z241W}zq&jafsxc`#%=Z6S7eO2Z@n4(Dfb;i?ScykuOs8Hk<+^%VqRFHSbX% z4O4tQ7oyvbGluSb@L#h0ilta--}_@fDjsr|{X-l6qcsX26=n|LUb|64Wlfy+GkzxJ zp^-0^l-~VqNc%04f=kGYq-+Tl_|S1<<0SqE<^`9wxmyXPT9iN8?-1bcdb##O7eOA!|NbkRcp{r~@4qyO_O3Yz5nru9PDfF00OP026( z{^yhb#g)R@MPFIxvlDj^*7Qcw2$P-=<4$_1kX5`I&-RXKbzkiX7Yg_APJ*heJaWVR zep`gvxnKL*1j*q~V;(j4bStf3gVi5}J ze(m5w*XAJ<&CtP#vbx#&Qs!!DT2vw-QL@h|A|f!WK~`+)ez^Z>Leg8)!KQ`K5YCZf zLGc!iFdQp>e|E#HdtwN2!(k|n;pKnntN((Hzfkhm)k-+AzUe5WyYvsVK~&hmwH$aB z$NjH=vIds8YseL1(e2)aIK4lU@h9iipLD68$*uKo7H=YC*C>>jwS$yOc7&VGzf!b3 zpzy4@_hW><*TCE@Z6}H<%!2*RW+C$FL#k`HQp^u5EYYFXM

P>MvEz*RRy5>D5v5v-1nbOt1acu})iVzE7PcBNS|}(SbKzsS;29 z5gR^rxsF}K&v$qK1e)U2QyepXG&2>}-RI%oke1aNWqwm?&a3roqFLK-{H8W%RrligHt53OS=Sx#fiVY0AYri_mH=98Lmdh&CC z_;K{dB$apkEe^_DpXIW9-P3yef}bj29p=Z+(Gl}cPjW1~`L@qTljX>qtLhX@yZhTo ze`^xO5M1|~+PWrdWNlTVAcH#J?|J9IC$2x`ci>Z(nJsi^=N;eDx>uz5=L(tG3bCXR z2sW#A<)hmAi z4pCX%2^#gK<5igCT)o%2SvCWUzV0yVxA>n;+tBWC+feU#3dbFw(n&{Zs#o=mSM~}LH6uy6jtbYOFYf-aYAA0##WB+-(9EpV?yV>7D7XJ+jSe!M_9C5} zo6Ya*O7lBo7Q4SLo#BygGn0(0iN)_RrNrgF|9MUv_fR{gQsyUJTU;UQLu08g-qd7%V<<1nxvYT)Bd^A}W4^q&K%H`|Rwpq;&|41J$TcyCf z2Q__&blUTM-svmT{sQ;E{QBG8{tF#c*T?qtPwctE{!3GCZMmoUHLX8wwz-ap|7l-& zkJg%hRrBuF!B?H+o-4!;o^)=`lbf(-?Zk=FGMMw5-_nPtU6J5Cu1VXM-C+Jbu0x;s zu_~(FC(JW)PL_@3bTIdgXIoiwo!OV)*{C+gG>$RTHW9jalUCb=2XTl^7F3#(_ogQM z9^UE~)N%dUkwG`QH~rb%%d|4+w5c{}ulN35$H!_}u{zS;T2N8$KX-oJ%@#6p-1|>y z$~E>p{j;cRr!9B?MhDbi8GeX7#>=w?>;CDi_wt%Yb_=w^&sR9@L~+(|N0)cPA$~ zue95$?e>%nnrV)o@0sJrJ;(7zGbq#hwXPYI9A;=E?(uE!!&K2)_Z!@G(ybFx#j1q+ zD|%brd)X=D+>Zew_jC7S8rXEs2yE6TU+mpsucXd98jyvtS|6PIpXwY2F7Xbqoqznu zle2Bw{fwu#o@dGMoOFA?6{eHHfTt9+c&5`?H&vKKaksstwV@7sRs`ky{p4!ji*Y*a z)z;ag>v;0mzHFX0y|~#kCp#xOW@7f6IY0IGu21<2@kHf1CfYuK-R-W~?ObPZ%(y5+ zWvcNxwYRy6>1ul2Zg_HMg{D%Lr97~5GuI9=nQLBpjo!QewV1g+zUET7j>SvH)YPC4 zR=k@mM0PL3qR;!!Pu;j>-oI&&=Ul$e8y(Us=3e{Yfb9;B%f8bNl3m-7Jy%XbH%K-K zQ;_KTXrSA?5+-Oc76jFDUu=iHI#tc6k2#N9t+k6+s@q9YhR(^dvYa{wgZ?)gt}`?9 zJL|XI&~Nxb-*pK#SXg_U=QvxX_O0%#sXPDtYOl3g*L)nA*S+a4o{qNlDm$z%y|p6I zg(gxuX6U$&-f9NEbu{VIWij_Uo0h7zssAUo`@4=x|6IONO?N)+*3tc7O<4~55x3t} z`+AQueK$Kre#86CM}NYno3`d#uh7Sh*?*t5KUTfgP5ZoZg_=%VwqN^(?C-j!m)#Ja zk?t|!WWl=M;CCHo-9&$r?XOI4D^VIUIX~?~C6gN^uG|#pG^2ref2v9@y-kN+dv@qm zxII>H*t>7i`sL0^U4wSKa*+-^J~OWc>1k_~8N@{S6&1?&c_T8@du!Kk{;c37Qx!^v zHTqHb$$?bc={9-j?l6xQ78EEyP^|pYq4JbLs+9MAd4BEJbkuWa*dxdkRM=*5qw7$x zxy^(tP(f9Nr=wj^-K~2q=# zZLarY>r1Du=v`0umbSy1dbiZnyPi6^cgX!YXAmqR^@QC9Vx2#vRaZpTi~UX9pI+{^ zOXl8dM#(?Uyiq3n+Z)`Y-A^pl+9j*Pfg07ezD;G*yuasWk&`bonO|C!yljNSTIcju z9Xr%M8EGF(=NvL##vR9We}DXjkGIW1n>Tci%kEawrfEi3$gePEH|A7n(0Gz=XyEM9 zKUW?~Z3WtL0 zxYfKe@Vi>)+&5`_N_*SRlWDRs=hEr4W2e5pNxz)BX-qkl)%zo5?)nfQ^?ry%nvPwP zeV>uvStl_;bKTZD%&*(3r)Z4H@0M_$_q?}NRW&u{TbuU2u)>ju{B1UGI5{}{hLe43 z-;T^z-RD&jZZ5ZHI(1tb=WcE+;Z>8kC)xM0(G2GIeXss9a!usj?#EBOuB!PbB|VRi zYI}-v4|hQ7C~qQtOqa}S(A2ArF_+ju)6Qwm=Yu;;r)%>tYeTr5ZA+q2Dx>Rwp?HoS zpCL2%DD%ucZR_J)7u(mm@qh2*r(@{Mja<)}Hmx2rI&jAutF(6B(#VLL+vY7-^{dw@ zws|m9Imm89&v~VFJ@@H)?fmgA+GySqWq%s9_1W_^@pP*W9)M5e_ceZ7q@TLRskhR< z7Zj5pCoN`n%Io`>RrmkbzT9uc^Ge5S^!%H&XZd;BZbAjjNMd~R-*}Ziv?}@9nVNV(r8X{cIzH8E%2N+%RL%Zr`3}^Y zVYFBOUVB?-g&O+zt$Ee{Avg1_x4_2x1lPT&Ek9VQEsZe&YU@?=z3Vmh*zxLK_oUvq zV4-oVHNyutsp|R%HTjb1q3GHkyH)Sqx+1(xI^=0{V`hAIoD6kucuw!1J6~Ov&*5g@ zFE;x=xK5icy3oFes=QQ(%syP(<}M7sKGe#x>Dczj5>=GEsSVf9lj#>3`HZ{$5luLI zYUq2sxYcAF|8`*{yv%Ek-apBlfX`T@qZgkSDX*RY13J0y#V4X}ywyDN164&D?h8~j z{X89V=bL)xI_nsyRmtJSremj>EH=l$E#?>qUsFzc^nQ(YUde2pc-nBnSG7G<<}TH= zm#D0Ss|BReFUq$cS zVd`mnX9Y}|IntDw*qgOJcBTKIF~1|*UtO>2>DKSSX6YWrHgd%wda`*_QUg-aSakp^)B~TecEAOTmDY?wdK_dma1&W6K15b#z-V3 zvu{)FH~&g|9{Cr2u++H$YZlkf))6;?#@_s(##H=4@6G%F?7azmRMq+a|4A}o zLN*8_gdkxN28kLFiJ+)8xJ3zql~@%-ZA!6)Ux8Y|FPr|^v<6#&eu-5vE{Ge%x`86C zL_xlyQACPCBby))0wg3PBrt*hbGNzo&VDm9naSk2uU9hno_p5!Irps3dCoH$-?NZj zG+!LoXbkz*5P`#}URa{K$eNDKC1W)jBZRTTt?2(@H2!-ljju$Tr7TJ3k(&mpq)uVEIuhF6_#HgwKQyiK#4ad}~ zl7_6~(1+^j#NN79J)AqR1;0ErS!XwO{$jNI;Y}P67wX60qAhcvZE{;8)y~5R9Ee1HP-g*^IzO&3OyG@`SNLu&(CM4~jANvJWZP)s+t~+8?9#gGJDFxS{Y}YWjy3s_SN6ftn|# z={78OPL&9iFP>6{n6HqvgWzJ=Od1Da^MO}WmVs1-K`(Y_$M64CL2dl9qhB%-q~ zpyv!>Prnyq4R6Lb?#pR1H5bO_n8PJMl$`L#uQ&N1h2w2mc0bO%qS@~Aaj7iuCcjUbG!JR_j-tKsp6V#gWS5=afqIONmrv&sHg1ZQbQ+RwyG<3^a|pNz z_5Xeq`yXCnn8e;{=*N?>YHa zL~G}Z_0FiR`w&0eIg8easX?Op#5!g%2F|@vo|m+<+(M7G@tiO)r@vS_0y^WlM(miV z%yX?m&5sk2u&On%aOsNsB}FVvhiUD51g(7wX}th_ zH~tk(|2rOg1!mN*?pH7prLnUfLV`%NB&qK6!=5Px|RPJg~h zikyaxHnynQh97CyzOv}8dskk91NkM05(YiJv(YmbB!DD&R8?>vIt<^2BrA1jNMKL6 z1Ytf?{&FO=5`jH2(hb#8+J$6wkn_H=5%pp+g<5wN_UQZbWL8R(J=crQ?f+HGya!RK zpCFak!-?M!Ru@O`R-BPFDcJiM*k|jZJ!$}?38$k?U{5tR51)dIKYS3;X(_sIs_D2| zsPyNx(_m6?F24Rn*bpRMZSI>%kEIE(dGyUZM&E{jU>(Yr4#%Lm!%_95d>n-^2%tLS z!h5d_vB_j9&z$~uQm$zh!9U*&X=t(jLLZ!5w4`#?cE1_O^@Y&suPR)u%Z`MLy6L`0 z-OE5dsGqzTLc`i`V5LQPQ{|61E}Ezov`c`(tGbu|g58SNEzz#}uXfdA*jXJ*g3C9C z=QpD!)y18&a!-@v)oT4*xsVp2>|8e#&B6|6b5$ip_~D_>$Xp=^QaJNFw7x03Wcl!D zSL&!CKQN30$es9TUHc4vq`&l)pCDP72=>r6ds}{6REX9|x&miJecdH?zEe5-k%amB zZuln>JZaFf6D|>aI!&8mcFCJ!^gfs3@XY&gc=K#zF3J-pL7Mifb&J-nxZ7B3&nmmH1F?U-p*Y=a) zjR#QkiC(x&T3`Xre)BT^n{Qu9Q?8V*!tP(MLieIikRpst=@Nv6wkw+zMHA^?1ugc8 zmpEB{JSgK*O?33P0bx;zieKHKJEcJlRBDILLfNoMZ!~51*g_I`R4q$)Vw<=JmW@v% z-ue2ny^Fvip!PT^=qjgXBrA64Q zac?~%PXgFkYq3XP<_C@RA%z5K%ZE4Zk5T(A2F*Y~A@E(==A+7~uFGHL5cvw~PoAtO zLWZEFCX_C}vopheUs)Rah0L6SyS(r_8>vFJe)Af3g%%{|-;RW=47AU<5*goKr;G794}CK4#y7(}lvA3y zFf_*;F8QHD?bCyI`5*|O7+~iv?|o@!wJ9wjEx;vO1L3{|`^NtQ)5f=u;G|zA2xHei zCkX6G->@*}o(BR}7)0Cg}sTZlaU2rVK;m&;M2XXIUeVMC%7tV$#;( z^v{o?y`={AU&?fJuB z;XsKJY8EbBJqa~;=F5Ha5}ep_0C9Gjv(6p(6gzL5rmp=e&gr+RkTysb9>JyG-s_=` z(l~0N+tY_C>C}K!+KVmpsjW0eycfq7J*I}98mC!Hq7B>2#$%7Tpyti!j8L6vtL1i#(dwczsHW-W@$`F5}$UosBAQ^TFI@L_cT!?RkvdAMT~j!w0Y_h zpqE3XV}s3h0(-(6clj*wliVxclWJc{f4F}$#KP@@FILjuNg;S1I3-^xR_qv&SGgOAf7kGaBk?Y*PN6 zZE%t_X&RF3qt>_!KWhZ`V#ZBJ9}+@L%T+ug|JIh_EX_l-g>S}LUOAL|(Ne75DrxrZ z`PAb`(O{D-SV;TMhn{T2Jr^v8m3Pb@)CU0<>kKfScouyBVUG@=*S%R=h3Y5KCgb`cJ~ zC-lRj5ItmsonVj!3E3ok6{GEKgeEmztPGVU(qi$=1KEF|9`y=%Tv=eEQ)dD zlfCZCuPy>@?Ln#=^<|buN1v(!(QS>0ujxHsEsP13YXlPdJ6S~&Uz5hY-kQ#Ue8jfm zQu$TJ2+FhW*Im7=vlxyKCD=Xr&)7HZ&-jV{$|n2<+lG$9!4fsC$#)Nx#%-h_{)jT^ zEWD<^La!x@k(#aAJ(&NhJjqyt6HB#YcdA<^%|_2@nzm6M{<_T@)$10x3k-{Md;d=r z*du1Wi{3)U>LPR>qn6uaGgr_gzUw5WOq-4F`Pmu^B`9C9#c-Rb@xZUQwF_tCr`hx; zmLrrlq|-PlEdQjxw3V7fj#1v z{=__We4x4t?p?Fz2MzWiou<-g`(xC;(?DC$Ff_hPhcOOm7fqi(T{mYb9?^c-&4=8A z)TE#S!OlQmS+3%OKBwT6dmz{d0afDI6MCT*LAz|P8Jf1wX?#>ep-uEhyP^6R%)3~0 zcyP8|1j=!&XfgIq%E$M^uf(egbMB8Vtt>KBukHPk~^kaCp~GM>cgk-Q=uw!n=qrurddjBH5Bio zFS<;PRyobv(R|dI?VsDsl17b3$}QQp=bADaf7Ktga%OG^j*Yw|?{lqpuEml~b1qt- za~+%qGos@)A=BQB$bN%KoKC(KPgd6>9-uUJEi%Y#;-Tq%ws0qARe!4 z-v$dO)bxI8HVF}Hu2pMsT3&0=n6|3hhy)VSv|aVo*GWV|j@ljMnWC-3V7O21rh z*v3o0GQK7e>io6+>hR2QPE|+{IQL7LrXnR_phWvEdVz{2RpfLu(50l&Cc3c-XB&gn zqL(9Pz(i_Di^bhJ4PWlSncHe*?MIaM0fQ0K^=k(R?XI1I-tj-6VP75M?z~slk~bI; zN4~;tc>vUKv`%tIYGiw_oR05oyLd`<9pPU8`jgnmI|S0o4TRlV>7rWUdNh5q3I~)9 za`tpwylAYrU=maJBX%uZC#E#IU9`&-N2XtgUe9&|2LDFhMU{;{%b!MUP-M1haC&oz zEw1gm3A+Va;%YmkYl$2PbQt{21b{GoHr+^5 zilD{QT3LmtlmLX`jhCJKBY^~Hj+F}aT`B5zAIAC8O*WT_v^%@5(Do?Zn7g3|3F)2w zaSyDWu0ZOjL1JAK-#ZpnbToma>6PBmWR;o5CT;PXu_NZ>kyNz5EFD$X@q#*3-m}PN zr~AqkbWAbzJ#zb^w)i);zL2Z>Jha*&TQdB6Q}Y9kXoRDa=TkdAN;)1O<<=`{m#uPa z<2qEVe*nEkxoewi43%I1CL~mRv!K_%R+83bp|f%wVmg7eEK7YJ2$MFnVZ+EViuIqq ztw|Ui*YX*aZK3=8z}-k#_Wx+IQED)&9(@U^Ig8L?;v!hO0%h{CkO^}Zp>Li%iq`lP z^0x--!X~-^1E|wl-2*WH+a%q#?4SW_tIOV(R^Gn`o=t;INjK zs2Vry?3u`pywJOkWCL2DtlU_R4r8TpK)A9?cGFirr~M{FWlq6G zMg*i>nMg=m!=U@hPo=RlZbWRC7&?M%m5bQe88^_XxN(&9^(~*syE_dABfA^kz4$%9i1E?w}v3Z_yw{*%{+^zQlMUP45Dp$(27b&;uua@D4Zm7L!vMd822)TvVC9Iv8v#zJj z(ZymKt%X}*oj3$%wtZ!n&`l%^`6rl;e520OWal7G_ymvXt*Ujav$;t3{h`x3@r~xL z1sNOP+Y4;!QxgVH+VbqNPN4KUjF*y-)W_ADgtPu@=<{cxy)Y=(tsw~!9WM|yPe^?a zI&!jxI`*E9>+(~=x=kG^od<7DLUd4sucU-TrS?I5Hfi)Np=Z*vbK-pxf{4;Zv6Rx>J(r};}d3PtOwN|a3w;O}1ZvuT6hvK@We% zrY(Ilv}}fy#n0j-4HC3FGz3-K{b#7VqWsA>*naUh6M!5b;qYYnXfsoaFd_)j)ewi6 z%u6V6X@Ga)qxW&fc1*(Eqf?YB(T7H`u=}>l-bvqJ8fjudbu{Y--hJ^Mc#zrS?YS6G*ojFP^ zd{7eD6TPO%i}All3<*~$exH;3Nar={WUW`|%1BwF-Ln^{g`%PO8SJ4y@uDMh(ry`! z_LmNTDOQxYkdI7J?FX-Ksm39wca~}BC{NT2@rdr$C8*H}m%z`TI;u$u?KX8Y?mbs6 zZts@tYsOC+4?N$xtd~7I4ObhzR)0bo6V!fSber3DLvNzp`OnzTH{0Ha;*BpXBGNi)l0=K1kJ6uPk~Hp*=-9mv z;;6r^_Dwa`sZZWX6xfr@^lc(Se<36HaSWJ$E28A)1C2l9REe6tidR=5`cMkaQAs+# zd#(E7R27-kCklNH4ZGVi~*M=2> z_rdxuyZ!EFbsUtKiIi#CsIr^03_)DFDEY!pkf8UfuN*3K3c8ege(;G+f@Tmw-&-eP zvzvUGhm9o7jH~SC>6Ewn--wPrv>SS_LgK02<#}`r?rmuJ<_pwr-h^W`Kht10KT}(f zj}upYflk6W&}E;@$oPiZNKJP5GNuc+{P4X{c%Ki3*}2I3=C-qrY0XEXX51Fl$<4UU zu8azPb-it<+mnDC!wa%WR2RgPQm0C}yHx$b2Xyj?5~mTg86*mkO_`$BSaA*EyDxAu zRXttIbW?`d-kq~*W?rdoOVZYDSKUPJi#*hAUT}Aj_eyMA`q#Ss6Is|%3fY`$5;wIY zzN^SYSp)s$oo|sOAK`hvnvR%NA}aK`M1eiYAmPE6FkpfiQOcYO8T+e_QiAmDHw($* zE|SM14QTk~JJgn_AHm)`uSc)m5CNkeeSXP+Y?G1fM&Ij2NQ zb)uLA@q6;$8f9wk6>6P8yMx

Xi9mr(-)?>ggso7m2O%G^{AZ4bLUv)g@wTj?-Fi zO484^blZ}+>c425&>kmk!?`ht?>h#EGs* zy6y_J?~#T`WwO$T2t}Ggq?pHaM(mJm+Kf{|sVTwXLi!PQO;9H+?$U0E*rs;!$yB_~ zS8c=o;rXhBxbniyZ+s4Q0_-Nh5-z*Yjf>X&DhutqDv}bhh`lCT=1Wream~zkyD}HN z&W>_)mDEeOlG^#EDZ|vsP9@uFcHq#`^@zD42l1EoL%T#bD|Dh8y96r!Xnxz;Hv<|Z zCDUh9Ya56Y4C(*}=O}GWMg-)RzZnGJn}Lg?I5D-+a_N0E<8Nh zRnx(1hdjiyxvyj5RhQJn+3y-Nb#*ck?2<_{%C8!Fc-)m9qJ?_lpj!Dd#~YjYQrwBZ zCPbxb4D64%jZN~wZ;ihpd26IBMbXh!uONBVeU3sjr1|2o@V5}^PH|i2NC^+SZ`x>n zUJ&Q--5d3S?K={P?Aq7X{tLqUwYwidtY(2obwLXUdo`;7r_P9iOoq#4n=RMbuzdD3 z;!TyPUNH-M6)lkj_~08~pu=3(n|>;a^u!LAq*qvLwPJahYyY`0>jy@HGfXme)S zhxqB9S!#cM?Ta}1z}2LIL={}oWqRoIux<1;SMS&8lyI&(tJogYMV z`jA=*M%y2w_O)RJ?LPFr%d@BTBhbirWezWBkX!^DD0Zqaw~K%!<4~DXa2mIAIT6m+1Y7h^8J4k=S+qN(x_PHlJ@ zdkWOHv3~ba651mnf4jy|?b8_<-!wPPc7c;OVUr)tMdIMj`+P9W&JMql_swl*9oFGU zyRWz3p+mCCVBi0)MF&}{mb3SvXJ$tGERxzQrtgf_>}ppPsZq?mM(&^YM8|A;S3ccom`TeYv;(R(FdM5Fo&4+48+%Ny=4@?HtHWq%v#WD!XqNeTDh_z^g$JZOHc zt5&0)PJ@UOx&qq5bwQG@A1fU7VwXgXFTuY}m@|V;{t^~;)$82hFY)uO58Bpok8k~% zg!V4br7J2yJIJH?P!ibl_6pwjpb`pNb0mtSZb99@rcy)bt5!qK(+6yU3G4L;)1DGI5JMyZz5l$LcI3C-PGCFB$!^E z{`d`46jP_AcojWmq2HG0rGby)j(#JwIbTz0>-=e~*xk0JZ-&&?m#x8Gt-zi@qj(xB za;4)=?clWXNB(94^39wIkzzsdh=}b;M-I}$v0W39bbWskgtw}@809FRqr$Hh*o(={ zN7vkJ#HO6Uk9SUUurD}kTi=Z5yp3+BsjI28W9wXJ_t!7Fd-lqyjbOGzP9x;2VT0y_ zp|Wb9KI@UA9z#&vG^hM}^t`oMpos{b;A7({Xt#kVth`H>6-^bY%}A?Ve}EQ}n(w#j zZz$dwE=!~-CjHwlkU|1nk=iBfz~4Sbrxkhn{2@EW^2E;5aCp>I0pXCG zkV%7`Q0<|mA0TooAL81wUb~P(pR*j+kxDSF?U-k*O5X)ADe=XQwq@i zyX$R2RMz5I*ptzljtG}D$euq_uPYwi=FB73k{X-HxHxGp;(TeFZ&8_CA@b)(*PaGwIj&Z|^~czFv5n^T&$4>jTjL#641UH$Ee zVYAx}Pae16`=*_%QI=1F>Ee}*{#&1xHxox8DkYtclAej4Ulv1Ilde5jO*Tnr&+by% zI>X}|JAYg~dNaoy4*B6K{YK4==7TTnl)M*IZri>kig2^n;e4%^sJ~W8o8nXDlSz!S z3x@sZ{(~u4653OTOJX#Vuc0|r@s2bnUPA&TBq*Z4EQsyhNix!P-fG0YoGm}15;YsG z^gh_0n!(+c_cl*j@TO`z4%)A)xD37oC$=0w+>KOsX*rb~5g*&h?#@^4LiP3&F!xDE zyUtx;@~EW3Wq^TsamGI+8qa zl^}Od!oD_<20A&V>jO!$&>m@*7mgpNF^*9GbQylB>XGK|yzax%ylC~eO{iI;jEN~R zSyu?MyW$lybU6J|ol9)03YC5mF}m*om^-h-Ug9d8fTCz^1aXaXcFJQj>Q*TRj{lpz z7DQEtw(5h2@x5cn;bmgDPVhAkTLFLF8c%p-H>W0~2t@Tp@*L{WDp&Dz?0@Y`Q8S2W z)OZ*t3TE4OA_!)wSM-o*7W`>4grd zL+85b5b8=)o84ajUvwLLE8=JuT|}d5uDxsca2XD0+}PfgF2TOncGw<6;pQw&E(h{5 zT;aQjOoU!W&V{YO7@M!p6E!v|zVEwjJ_+uri$7<+c@IBnwPWSphW2u2MmHd+q5*+? zHE!Rc?-(dALh!dGz8R3P@Gti)a+>->`SVd%$)0JY-wecs;6%jc zW~qsh@8UCPhf$N`q{7+VKdN>u({a(FKOt#YCd?gGW8GaV`(|`;ck&R(ZwkH+B;@!8 zvmJ7R%x9y>pN)43}S>FQNNjs9csV z#<6dd!w~4;gpb##6BV?X!C|+&+hoVVJV2NslCHxOU&d)$pR*A)Z#?hW?{?1fOz|^V zzn=P?hRTvKZh7e+I57KvQ2xYMNPK7!GQZu1fuDVhJ~L=0PIEc*@9CXVel^zkmI?Sy zz-&SCG)?aQymmpJ{& z8ZS0o*{lcY#(yl%eDfg=EnbM-(;p!bj$fhIinlOm>*u(1^>R8USbJ#;t!z#k-Hv^! zvTOMCT^&2!rctd|LA;O39E-k_G~X^QNc^z;Q)zxEmQJkDyh_a-fli}@GL%QhNXDYB zbSKWxP(P|uG6{jkB4$XoEhWc_KT~I1sQ-RUJ8xjK6{MSH&JX&d4;iEs2PxRl_%07W z_nxjS9xo_Sc_^#%s!534`H!7Lf<~|v27Y7{AEybWhz>O=%sHJl{ZSC%+UBoajg8>l~W^g%;AzB z3JOoZ{v2oe%1+69G_#%clvdC0$Khfc7h5K&a|(y@UzNA4cy=8MHX^(A6*Xen$0eRl z$x<%!z@f^m=J!UlmxV<1#~c^o@ET=Jt_Ic5KB1XY(Kz?<)|U4^`qLcer#HQ;S!<#( z>9}!7vHMFSQ(DdQ!{`EKPdw4Q_`1L64c!)G;*+DDwP12LBV`Z|XM_J9#c?B_!Bx`QvR(?Ff~Zo;|xD%7rdLKMDJQju*8(ff|_;`Bl=WD|CYEii>9b8m((ic?iYO_}+tG{MwO#r~GOK z{6keJ*%3CIw0Z*VZ+ieqG*PGw!K(^LU2s7bsep|qY4Q?%`m$!DgS_h?qGO6!L5owe zw7C1z0yDZ^m4v!CXt$2uRfs>)@Qf=V$t_Kpum1JKqF2ro2@z8E*m|c zx&psEyjU&+CHV2l_r$Uo8WQdlpI;4fiM8kI&*4hW%B3TJyx?!f9~AxSN6dh0$Ukib zF#>TtNCKHEH~WrX1nt1JUF92T$K&OQPdJDp~6xqf! zyZ&NEp9gQDO*o{9aOZwCVm&)bnzUS$-=Op@)!6Y}7!rv8&0mnPZKmP%045M21wIDG%GGR7(=E%*Pz0MfE)>OWN6fK@Bl zEHA=;Cp6;1NjIQr^DEe2R3f>j|GnP-0x{`5&?#F!YKzPcPrMn{8C0*I+DyXiYQ1iA z6pAnF&EgKoiq*Y8h*qfS{Sok zaLEsEv)`ZNOvCJ4p(e7j!ohqfyX{X$K zwY=Wts*Y;;d8dXYw63|tE|GSRDT4$-&nP5}yd7AzSb8h>iPDC2pd-(VH1nVN=pbki z*Ore^`ktD-MrjAd;A2eB4nSfWE}WK)AD5Jft}CX!fKFTIAV&Qp4f^UI>z>L$@)P59 zag6SN70nGiqmJJw8Yx-5=zpwUL!%JAaX0JS3P#~|xu5q2#vtWpZJUvTgkNXla6yTR z!Ey~^gypJCL5O;tYEE|@ueeb6Z2ZA_T@z>ABPTA5jqf9z_`pW?1f1tra;CFNBJ}x( zh8%)DV}1@#0<`f?wODp(D~RMv8_=LosQMdzb|F)V`bM zt#s?K`@+FS%NXe6A&7z{kqT|45jPLmY1gr%x2gq7 zeGz(hO|Ddo`|+F6=a0;wjeaxsB|!O>?9GGm9M#uOt1D@ArEC&TmDfxNNHJ?SaIkG+ zNobP~Ou7eUB(SFl*h{w7q2cHHu={2_<&E-9IW6~^g%?l$i0_@S7tEdd9b(^R3K{|R!9iXJ&JxrQhc2RzV)wfh5 z9+*3CzB6r^3rsB9sIRg8apC*+<82arT1(>@yW>SoW9A`q`gM|EDd93a=5Ax_{Cu6jo@vsvND>7nW6^QyZm$7;ll2{{iA&uw|^gs&-QWlM;+z!cUekDffbOzo@OyR zX4*`2aCUT65TOQwosb-F`^xs=2N8us(DrfjN4XN_$dl*m>ptC>Mkp_G{bwJ(HiABv zPpxM`y(b?VXszGEv!1or?kO!w9Jc(@BUxpYDzoItZ0P%X(?Eul ziXPS27(k+%T|2{4T%VVY9z0eZe^g`T!>DNpdh3c(ViSM26bO3XR%nV}L&s%QuYO3+ zKi#2XyVP^reuZ56LYOses+1sM|Lq%pz7NTV$$RdQW%LA(8J6CE zim+dOC*h`@lF1k|=aB9P^>zcN&A$V0e7vl|pz4;TkQBz-rr)O|2JRr3$V4#*3*q~+AO=#>843F8#e*$#EZy(+*d`?% zSQIzPKIwY338ppw027aJ;)h81_7=r>^t;wNZ3&)3e}l-9!~&#a;vZaL!9g>ft+(K{ z!l4T`av|fZnuN~&9@#Y))FKy#T#CybAv{mm1PGL>NjjgAAP#*8#U^c3vllPxe zuS1f8v)=JqT(FoQLyPQGJ>M>LlhOAB7fJG-P>L4lq{NzdxX}@aV$CW^*!imF*Unl9 zDj4PW9kg-9%0R6cU8NjKUWgv&7E_sGsAXr(Kx{a&n2U5zv>T@7smG2)i!_UkyVg#z zG@byAfxa#frALp&kU(R2h|N^l!cz`4%1EkvKpbYRu4k#B)RU%m1!GVoQ=IuOH*@v$;y4&dR9j-|U)ApI8t$&2ReB&U2*F z)W3P8a&q54hcjRIZ6?1yQC1WFWahE}Yy1HtyzY>bHnneA!$UmzD}Ye4n6Sg(*E!jc zbXYZkn4fIwbFE7$+Q0ToG3Ir7N2gLd@6^EdW_OG%dSqFxY_sy%4W@og=L{AjD(aez zY_;h1oDDuXj%pRzp2_fbn@1oYX`Otjt}nzfp0E3EL_DfW7l(gkk*ynwY0pi*9U-Zw z;UJxCxOmz4Dlrpm>3XDf*cQRPj){}4&~}@uE3r>^ zAqn`sbjw8%zI3{c6g)tO&Me0)X?Z&68%ta=gsAPHb&Xd z%3{vM%*v`}Ye=*to!iW|uZYN74oOn;X5yh?bfHx?v3J(-`AUGyI}35ioGYv35B_=B zCT~YvhB1~CeQgLAI`+Y2a_UK7hs*gh#ZQHH5ojU^Rp@0W7E$M+xxti=@TA=lm40nI zR`{Ax&7OJ3Tv_Xfc+wsGWC^&e^h2wBYe#s99RYo*8ij2tuYvq}F#HU$r^12sPEmPu z6^CS2Hln&`m^A0zc!&8#FS@-(yszxa#IJ}W)=u`TabGBb9*WO~+5B}gUF&d(T%gUC zoU!7?C(EWMcS{a5GS*#3^(lLSolMs_R1m~%Yv8a;t9n zlU}EUn_I0JkXqniJ($Rn9VULU0-x`#T&;46&X-2n2d(F=PYaD_e_`7 z>~r-`{lofN<&dS3p65p^NyU3SQ|aWkkYw7ISjRqVch%DY8C}J1N0p9e^I14ie$%0< zy2D)+tozaFmQydi8o?}C%7t)>QKTgzU$f~geO? zyuNCz!QKnB-}mBYYjpX*t8iwi_=pCltKIKLu6MVSqRJ2P5X^+skEspSnsC#QeQO9E=KaFPf+IHIHcVHVoekjdOgb zY5enY_?x}F!_!<2*yR)*(bd*V8Rj-jKO&1Cf(sBjQzoART@yFkTz^z7B|5y9cA^G+ z_G*9O`($r#Tv!|#+pBmvCB+Bq`(&DT!y58tdZF#;*}{CA$?##pfZ~CH&G&eSg`wZ{ z1wx4Wz@9QYI|D%OmkV*)Nu{KrzC7hQD;9E*Knvrs*P2i!*0?@={6a^J?%dY)VU9I( zU0aa;2KmKqaY;7m_IyS`;qd@|m;W%`u$M6zHF<-YQ9TZj;xtq-d}$IQW!ZohtkbY$ zZg-qqz@E7vco>J6b1}=W<0N%c zC<8lqTY>uYVg0iS&sd*;N}6EFo~bQ2hU*8>80dCThUGqq1pB%7bOdku-o)oOQQumuhJu9E~{YZ99bf{c_#F`S$k` z9(224nWQ+_PZ8?*ReBU3VhS^J?ys6l`o9|~=y-fb^17bYm=OGcSW6)X26@%^os@o# z9*Lzs&S!kW#a{I3@ZiZVSreAZXPgg{=WMay3V!U!xwPE;jN!(+*uJ~eg-X2+O(!#) zFg&`q@t}cU8OrMFqTx6i;$h=OHGJ9H7^-`iWfx@t_KDo2^0^XrVY|B0b)wD+d$@{1 zWB zc(l+TnUS~j?2iY1!%_1gs&N|lEM4}Ap6`JOkhMgm?N3~&MojiqcCAnJ{0*& z3nD?Px({hPx`c6fG;I$+_ zV7JjWd?lX=L)ZOvd8&xE$HjhgxZ|6@)3_iBzI$Rgc)l~wzD}-_(gBpxMwOI)Na?cm zAuTkj9&XBIw*W#b6Js2#{}QvsWiHD}$=!FIWM!b5E%&;d7{(QFkFHwy)$~V-S?d7H z%S8dhipAE6JC;utmuo2MhU<<12GxZ@o)ARMC7mBMW2jp|+_Y@o)4XQ`qL7|n2wyzO z>InY;PEXv<8_5NAf<9ndHw2RK4p9?G+!KQOJyFLWFO+?`5katy>crT;bl>(c0V;+9 z-K;6=x`=le0PVTndU5>LZbB4dVzM^XKzCb_=X;8$%qQ1|na?x;_cnx7-}2n+kmJHFuFRu~7Wm5q~|n$7lnfa z1aRweRDyfHv{9sKWJYYP$Kk@wCUOc_j%CrJc0U3sd$R;|{%tjxe7N?jE?r}oH3jH9O-?KTL;$a z@vfFte2`s>A?97T)2a8DxoVT{t@%~xyc7vW-JnEDSs*?+{6<$!_&|Z@g6{*LZrD@BCxF|CxS&yo2ir zg6(olQ|`96ZPhB+i}IgW_dA#V3lM(0b1&Gj>A1r6ygA?YFi_T=*1`4mKt&U(D&NIf z%;M#aH!;_iSQn?R(Cf?6fVaKaQd((Za$xrjD8Yw}){DvDwhx_-ZhIu8NwaV4I}Pfd zs&ZG%Vm-)Chi|777&Y+vu)K=F!rb(k3xxHxS0XDdjd2(BO;18VnhP*Fuh|1;xBu_-#D7@C zZ?G5d0Dq6NM8_L^Li-b0OWB`-L9%V)8bbvs<5c9~Owo}>z=2lAr5t8;y&R@B=?$bUy6FBDu&Gk!a} z(0c`i`30c&)^PvI(Hf-pvSOe=C{x^U$h%_^IlC*guvC?ci@x+qcs#tT7R+m|LEJ|T zMc%ou%cJvGW(8#>iGO7}UG{V-b4B~Mo!$W!EQC9jE8+f2$rxgL1&vZ^X|7GP_Uq~G zC~7m8rJMqaa9_Hcaru-B)h-L1B9TiaKc)9_-;) z`F5uMLl>lcT$368)gBjITeuhQ}0W!@=$PMrWl$cC{PTTA&)$_EJ%)670sA zQ+vO&DE}CGaevOI^(F)-sJWn9RU|J07j48c{OSy@{Fp?B@gY`XBmOq(iB zgl3$rI&rfGt8-HHm2!`zQW$jR(X1k%=ZZ-)rx8Y>!p^Vbfvo?T znb#!YZ7x z^d!JR33(+hTjx?pN})M+GO?WqEGp{dz4(Y20Z!ruLJ?j^xew_BB&%$n z>9ihr6CU3EGE9%LJmi}Yr!y8;n?R=zV^7gOtyQ(cNzpeXN*K3pi;ohxNx~VMZ7K#4 z*LnqB*Uo`b7(^4YP|E_joZRQ5p)EOgRynndB<~RrAQ)ZIgd146eNqZ$8V)e}U_hg$ zV7?N@^%F!4FZqgLpmOt+j(70Ku1@5vY#>DFXozhbUIuT>B1p#qnCL1lOTOAAq1I^- zYfoAwYGx}6ygS-OYK<9RnvX@0xb6rb=ghm@d=OKGzKDyr8ekGZz;M$he7-{k{#vpi z_ERHpcYuFddBms^mO$qKyf~mTh;W;FjR8F>u*H!n99Y@B~8Ip0O zCsSGR+#d{`D|XfjPj1lY3VN9_qRMuPta5n`Dt4gg{ThJ9D8YQr_kI6406)KG?VRI} zY}5pz^@3x;R#@Ch4VY&t>Gc0-!Teh4228c)COtuWx?g6dy}&Ut$rJ2@Rv|b-J3s?L z2?mo20H7xviN_hC(tww}C?!^~I@Tz)!*S>!Rg2GscA&}mrh?gaf+`PG_w^>9Es)E5 zud%c`bD&%AvgWp)SG6-u^RO6lo{8F$&{S1CaB+zb3OzeSvvD1TFe)ESOWc-vK1aTGjW z`sBAG@sY7i(KAgDugbz6464zZNIIG|A4S`gTbu(xnWp^nMbe7evZt+nTHna-3V64G{~=9XxQ z#`^sw9&76yG(3Q|{YH&#V=0n@Vm0Q4VK1E|vywxdnI3DKdiTr8cOr)!7a4@%EU%C4 zEE!)2hwc`S0b=u;7y5Va^%jJe-%u({e(W9ea1AaUUH65B_%Qm`9A+K!i4>6<;C8cij++1UyW7=o2K=0f{Ss z?+gEQ3;V$7Q&@>u2NI%yg9U}r^412DBMIAF-l}P-4&`(<{j*MhaZt5#NjkJTL;Rw1 zdbhrYko1!lfGAFU3vOot2=!g2OZ!vsn{XuN8`3K$3ud7e!NjL|i1`&Jta7QCu-Db= z8|-?!?93|d)!-dOf{*Dsacw>j^Eut7wB5AN?_H%6i2HKy)6=tf8|btZLvGGF?ddrTAmJlFf<;oY9z+XcNze$j z+kF=rS_%LvGP{K|FNDrCvrG+E>)b9=uko7(Cz>bAXK&(qpKQl+0=5$Gmjw;n?1=3b z1OIgsgXuMauvE<9GhJ8E*alCu*ldp#crdK6eeo2K@kZD^**?CL@LHE+^}|sKB5$1r z&h1enSZsfl@C8t0VYB+YGfXrz|Hjefk=qul4GpuM=t(U=*q*8p#myO}=SO>dhH?$X z!f>|)ooG+~jm>FZ#V@a9w7khpV{7us@0^5O#S!Y9Cs7_VwtjQ5Md0Ca-co3Ocv9o~ zPMPK-DBu-ta^H(CZ&A-P0Jsd5fNt(zxd^o*nABoV;cVAG^YK>$%JdmRfng9_V|mla z?&=^)0KW9Msjqs>?F-mcb$FL0$xJ|dk%k_@wUzd7Mi*vnoZc5dispA1YGX z9enAFB&0EwZmSO`ND%+L=|b=Wd`7BxsezjtFJ#y9BDy(ZRVUYn^?) zuv@z&?&h+_9E#rSl8)bMs<(Ps2KIv1D5%^ALD&(_Yq0{4{G7_U{QRKOJ8t3>)htg& zs<6ZTt6_kq{CzD%EO!Ps!-L#mha*AWOBX|ZDZy@8_0=v2wdNUc;R>lO4vp=Pq!x$k zYC}!rIA*k7lH?`nKh3tI)Si0`Z8cbaoW>|Xs&VpP(SVkl)b{wbWpO9|`f~GOc@Pv7 zlL&u$b3lo7;Ix}~V~146qt5X#z<%j=U<@7An0%ygpUi|z(lfQqNWr{Ce212m$f3BV zU^f@4uJ2zQVQxc+6T%ukm-5RWA%-S9-hPU8>UI;Hlpm0j(|ZZKMQZC?5M2}{>vS_s zDTr2V@v$|^vGZ+26+cqAAm9Cna_dAn!IJ^JAYGX77-TCrx9I$oD!Rk?ZGDz)O(31j zBYf|NBlzZbQPv?tu$l8s#Wp2mq-#d@&oE%~RNb|ATo9z_C_liXXJY*I#$&%o|Iqkp z9zE9~NP+~L9Zohj9K`n|qm+1SUSh(0RxPRbW3O=I#xXtyQ?tm=!@-Db(ch=OgK9dA zz(7ivYW#Q*pcs;J!_0`i7}o2s54!J(alA++;J)w?wX;{1Rf$?&{?Prk+*Z}3Pol!M z&S#niizzTwLWH|BP>n6U>ufS6EI{4;lY<)mPV_gUnUuU5gyBB1eu`akS73Y>!mKh1 z;T$%-PJi(ZDG*n4FoZ~!XCmFSolaYJ zVgiR}8Fo5@kaPs>rZvZUh+aj^9rjv)Hkm6+qF~zGsh{5%h&-rR7(AXmiNn;=)tT`7 zK9)UqC=ps=cJuw?SD;_MgIvvm$}ivH9r&k(x^I^Z+v5YHGZwm&V5ok_Y(j&AR2#$A z^}w2;)2yG0qp-1V%W9@?YZxR~)^@c{@_2s)+UMw4cRP^zslYu){ZU8I2XosvZ;nC^ zkO>{(^!6OAX9lP4JZ9%x9i_0K8PzYVSH#g+ai%4+gCJ|Xc%CYxc0IfMMIEJZ=}b>u z5SYRXbfJJ%X!i+^WM-Ggt-c{y+f(&!FdNe6>S`S<`8AkWVvIg2Y{eFO^n`3#mv|Up z7M%@us14U|Z}G_X;@kRDsD7^}hK|r7Dfkj^f8f)#EWD`)z8Q&QG4_m~S4Q6N-4btk zn53Pl&o^%>F z){y;ldD<&MG6h+*EI~4AoP8{F2Fn#vQ_TGOk!XjLv=?vB`8ty&&9e2mlTmfiF_}`I zBzo&Lt6sqWgy!ca(pxAab)~Se0ZN;_r)wn}$JdOc=GYJMBdS{_YVMykea zgF}%u>}#!UnMAaPe7de0_>sml8Igs=Y$7RazT~%!EVez(IpL?#kbb!3Cu4Hu$bcKf zxv#qV+6M(Z39-?6%san?@}C1jD{$Z}G`=@vg*sznm9Av_V)kelr&IlLbo6?4qu64P!nxrx<(m@595E}8M%+~B zi`ObK8nxH-bVifFa4AF*KI61~w+&lbS69k4&tBXr$nKk)Tz)81eZ&% z(CV>e*B9&3YCe3)8;N)d_WGi7O!d>f>v#i^#4ds3z_Pb;sJD5a#=BOvQNQ2V5n7tU zl0eTcV3ApZ)O(8w!ocW`EfcJiWUy10_5+koP*Q+r48uxiyXxsJ2-6<0RbUO{hEGvA zzWmkeKL>}LfhhCMZswE^ztc~&rbs2-j%|f74pei8t4pGcTYtz1=)2!Ls}m{nSMw1& z87HQ@>n`|?g2bf|HrgDv`8jk#UEkmRF2d|A63k5ION-3&ct~$UT{YtrST4T9U_qYx zxpeIbX^U*XlrZ)VbeErfaZPIgOuhp_O+POk0{-hbK=ErpED*`q##Vgtjs2ptS;Qe$ zORpi`AqJG=cI#uX-se7QuAwzFwsNK5sHy9pXLmi-9#PfjUG_N^ivlH4&{Ulh)(Pr| z1QH-N(XRjT;%i)JEOPNT6vDxSA65pf@XZwdBssuE{Vw4YF*^59x6dvN_R{I?YR!m_ zR;{|v>GK->iP%jY$zxIITgg&uxqrMtLfXt)LoFax16MCohQrye_AL?(Wc1H|LOA>D#hg1C5_5 zpAq{R0aouteQh5c$(+3FT#oeu*OW?Q!Jh)j2SNv6?&%eE7g&FC95(?(_s01fu}P-J zH)|A9JHUJMDtr1$36-|A*eyraV3{2u7dGef%})~|GGUbCX|-8=qZ=ZZT$yzYTrOnkND-X?9y zZf;XZVSP@Ta$b~6V{6d`{w07?AS1#jMxSwpDvg$@5~vl+8H~s0O0&+o&G&KolE__w z2EtO|4cSAoK>ZfB&c{Mp@X<@#$SX67`TG3py>6)G#alsbe5(!ihY~K4I1}umyZP&{ z<=@tYmA=b271cW5P{x@)JT_!a#uf7FOAV@3&e2=>9M$yZ*^ex$2>NkXM!`1gsLl zVS{RMcYuwZQJ%(e$RCY*4LlH_1WN^?1g~QkTvp>Fxp6o)yzJN8?$1^gmV}I(MPrBK_pP z3V(l$p6kXHd!*cWPQMHW6x)xT;t;Aa7=UBpyTFGpS>K)W;wYsoA3T-oCm+O*CvzO0wl51F3#`6P7gBaItpdj)q&5e;2 zqU2bN5OtQ6km`yUR2xHX*P+nUQKaC4+88MAi)E zpx2;I@w8PMTZQ-Cg=WSapO04?h;@nx%`Jb4Z)wbMzMg%-?qbOr3@iGn%vRjIlf$98 zqPOpFz5U}{iUj1^+APS3?B#?;6RtPq1N{+`PW%gy!8Ue_kT*$JZ0u<_y(=39t=^>v zs$`N(1gV)fLif8b7em*6_v4egA*pw$KhVHw+iZvjVTliz6(0+8N(VHe0<29Re)&zf zaQp7Fb_686Nqk1yhPPz#I)x+=Q@-UIN>rR8MakQRkQb7hck#vBh(k7j+MtAnTrbu!ps|aLIZM}Szs&%F zA-II*>ez^5wbYy7c4?W;hKxjJM;=7*E@@Vu8j0h1Wnj8<^qq7C`p7c=G=NkfhqtC3 z`%j=0#we^e!hYWirR%h!OxuZk&@m2F z9+5d*{=QrChv?CR=|^J!uJ^2)ecAa=uP12FL&*q)GiM$ZHf%K!=~QWWTkgp(+Ya|re<~us|n#+ z3LWBsXva$V)v=V;rm zgF+OY7rlZm7eC2hdMc+n%_7=jA^TQ<%}GeC=t1q0oYnE&j+Yz3HWxK3wkuJgP`sUDnmIt>Smj!kYWGrE)^08u&Mtn*{+?hfDKj-Zva^ zwyrjxqIrQ4iDsMsfbMndOizK17xEKn^FQZ*O+IZ_;Xt)1u<4qLdFM_!%>>k(ZY2qT;yR3l& zCl!YPU1}G*a6hE2N~^qZjHi0HH3$Kmqk{7`kNpe^Mt1z9^w=1_Z8wCGklbPD{?Lnj z7lcCVs{;bij(?Qt4#M{y$L&@JPB!W!O-C(&5-1jRn!Cg8d>1cN?|P;1ito)r(AQzt z%T-O5XX)B3Vf=^KZbQlYNv$39$PBg+9+U4cg%Z-IF{9UC=+Y30BI=Q}9p89=gbWIp zT_}rSViV%&h<-NTr-W#5#KO+q_LzG{C%YP0-||CKe1`7}$z0Yp+KU+=dOwl7s-pQT zBu=#9)iE&$s+imY-6$&}>^Q8fw1J+Wc+QUfdDO-1lD0P^IN_YR#w7Y3^CTt2fOi?- zaEBYtmcpGl9t38;ITEyGsU)LVOtbVH6awn`Om6B2xrQE;d$<==4DqN%AO1qERmqVK z^2fN?%sZ*R3h#B9bnB0T8dd&}-i} z$aULO4j&>g@oK$Rwg_n7<8g0q9Mm)@uzS^T25+BgV~pn?_Fc)Uzv8?^Zn9e5cGaIU zjWa0kl%mxd@MpT*)!DtT9Rou!wrPau)x_LE@~n49GgTm4cba6xhCkeIA+xZ>cwYIK zOwA%YD}FR1W5RG9XJ#UyA=ZT?Tj*T4OWnUsZTfA^J$;cW&2p`lxpx?$5l8FFxDt11~gMk@gL5 z$J{Ui;?jBKZtyrGO-psh1_Ui*^X#5fb0c0zx`pHw2AwOpHn^m)hI~?-Ie%Ti-355^`C9!$lDut+AySK3t3U{O9)62cuaWr`f zP9VACG6QnyKJ87SdevI?#Sm>#9yOd6{uYYs zX5)gcE1p~%MIc-rJ!%NlE@rq5Gb@%*59V(frTg)!9UMakI;y+L;bQd9#@}Bt&_y<(ioJAr=)omYwtoL5g?VQg{VC~Y%q4N z%99e}_Vl=AQ|Kpn{;`{$5QN}Q$rlrTYH_;a2l2S?F7E~rKrAAY3^eYR$RR4{+v86@ zPoaV^HI2F65fDh!;v!acYe5LDals1Y?H^0kv}aYM!EnYe3ZLJYF(O+cw&U4r*uW8c z825M;6X(B=LC$vSx=r+?0Th5tHu{;0fblaGA#GHJjLl>f;uA44Batj()|TwVJXjgeLmp2Sz89GUdF8@-;gLopTY^1P*bHvL^pE(=vUoxPYP zn0Ud;Q#Ogio$T!G36@!|ZKplCRys4XtwcUWY;G z)*jLnA3IyO&!!w+N4}0nIPE)uY>_I4l!o3gtFOMJ8>M4lOq^JhGEwu=L=xxniRJ1P zrwjOXw)opo2b?lU4CjGPILEGC&oiFcxcL9c1v2KzR+4!e-TnBz*X=%3|apv9Tlw@)vdCMm^>xnY|afK;ju@2-LEx zdy>WK%Q;gBg^Y88Ea8z{&z7fj$+Rl#!|wp#$U4gnbo(`(8uItiEGUJACGc%}g*JwN z)b1I2;J!T<4HVIGFn+&J;kQ-%0So`_1-B)HZ)INVhWF0IFPl^v$vs~N+-r!#Ql<86 zcSGJHcN~b)eVkrwOJ*d$7QhxjJuEK?1K3DviE|gmpBx!^a0Jk>N@D!nTK@C4|KDG5 z!+}@iwc>K`;r<%z-#_#Z6#v_q14yVT;bKO&&NBO}eLe_MDF2X{)Ob*<<E!b{tH9W z09@pd^1c5r=fQtvom}vMbRI4*-M=0G{gsygr0N{;0EtCu;1@3c^xwZ>$bVVEABJ=P z*@-*Q;M$*U>GhW0>f#li)pFPWCSq^X4ou@8?1=_KLxON26SI*EDPJTYRXk^G^Ee%p zkNsNec$j-Bl=*_SlVW}NK71~^2$@5EQXW>3Dz;!IXeukqGbhkG0Jo92IeB&TjIBO3 zu~mf!-*YX&%@sq|7V)~Miwsu2x>zrk%$rhzf@ z`h0(3yuZ_QU(}PwMGeA=^RE*MR|LQxF$99@f5pq6n1wAN02XDw%;|rfP!R+G{t!Y$ zE&h(rKQZ3lpKbaH3kBX3sXyQ3H#iDM0r(RIGHCkSAHPK7&-J4QfJHgk=dt{M5v)Ja znm`1=9}#a|#{UHx|9=!)OUwsyrK%pZ9gSadfQAPYj7o)#r!q8}^XG~8!skhGaufd2 zA@i3p(g6ntURz^edNkwVw*31na)5B&h97)vhZ~gsJRdSVpek}(Ut2vCS^u(3qwxb| z3zMHMI+^!RfaH&>d{F^F=Edc$!uHS4_3s*tIuM{qm;&vInEd{-U_RCZ1myAVRqNm{ zONb^MKtL?uPQ=Y2|FRmows`OGMu{yZ^d1r#(ktw|RG4RO1<6i(BN*3JMUS3>{Ug_3 zGFzJG5CqsYsPMKroy>$EtKXu6T@zytVYbTS>9y|!ho4_rmm<9jYyA?a5x|3bUIQ>L zI#WAOW9N3Gy{8?JwV#CvNk~20)fb?!2gzOVR7{sSQHk1O{Q0Bf6x0-##L719E;BpmK4*;j9*<2qgTb#JIZ>U6@p7ZrW*r z`k0loMer98>-CDO8Jc|H?$noy$0iG;@B?RsR9H6a?n6(6_pz38qU`4#$$911L!-$6 zj?`kXV^WrzIH9wc)?za%cY3gRXDI5ODDn`r`EQ3hEN=_N^(b3g0*lNKTRND7 z3D?dEzT50>MlkxmpvR=O1WNtt*BGeHl(zYt27`>Mw`VLQ9G|~=$LzG^`*IGir$8Je zyjp(FTQg?~Puc1l=>rlccM#)NgIsA8_0)ib%-7r8>SKq7B0I0hzsfn){VX*_>eLkb zY_posQ1Ff}Jq%grq%UN-){}#<($z_)a_i2d%0nN0Lh|=&ivjMqq>SmhaUDto=!sK}jz4hc;g%TM6-NK!*a$ zdzXwgAcOJH)oWJ)Em-E`~-2%hQ>eK zL^8~e)Tt^ZX4ip05a(TDk&mZ#X&is^RH9b1G-pDIV|e0gAvPgiTt*fMTV}-H>1ftV zadb0*t%B{Gsh^jnnFaDmgp*?Y!)Vq1g_Ci799I-dl{V-;zp2WCZ=l`Ma9Ii60p9{^ zb=!%I1aN7U#@+YX55t@?hDoei6n<52v!B`VhAN^G+puJ~821mfUnlF3=KyJ_E;Dh^ zWEPLAs8wO}`h_@&Dt8*3i{1znzV+Q zJ7FsiZCZK_^0XRkEli+YVkv8Lp?;L?G&WodIloiSg944pfH4nA$1Vc3el_Y|f`lb9 zjhegvp;`c~G)+fP3L8qRe(odAfh(Vg*wepm(lM4Iu6kPm53=8NkJC`$;^WV7(5fn) zhQbON>&N^@Q}Ev#0O)t``*~bI+lM+wlIeIUMBb+|Hxa&bS(c5XNKf`Z7_QE}p>eSk zsaOk_ieu8*v*w^ajZ|x;xHVdmE^oF|HEikixFcvsIMfr7W=JDKzgvm(5rf zwB6!dsM*3~M3_;$qWrxsF%0{3$esCh04Bb%q-8b zTNyIWieV@oodKrKL2zwVar#kf6JuTY$=Lf#{ivD+@upVsFd z1P<=~6~;c2zJ9}mketP6_XZ@ga%pmp$_RLzEA1PvubF52I zRA!0nSg{`Tf-J`a&Yw5zXof)gF~UHERF3Shs0%>1F@7@CCG51!lB4EjAQLSGwy4Ii zTQa-3qV&TvdIHQ((r#-FNV;SfB<2uX(Q!pYiomxsBRRw4$xxOTcrG1jw8fTrAPAi1 zWjb9#4+&_5no=UkoU~YU=}gLHqCEO-Al(hIo(V`fxh7sTqNr&f#K)8tZvomLQAEXA zf-`TeC|7}4ggTrNq|zi5p=UPrGpMWy6b%Ktrqtk2YS85T53SqjOp=TOGU}nS7%m1X z0i$OM`iAQhS)v?dLp>@-R;i{RApjywHR$0;D8+?Ki^iu^X3^%BtD0e%kC>7Nf-lHk z4w8;{^=j-CmWGq2t2vwailSE&CA@bTVn4(7P;I_W*W1Q~-Qx#2CLPw75bkH8+SN(8 z{M@%OBZNeKHMl|ocoH(_bbLf`C%xutDg8wlV~#%Hi|u_v5MQSKPv`ma`yZ%Ts8pY` zD?B-J;jBMNsmgiU#Om0%ezC?cDzi_?-4%ffk7B>rmOr$J(=K|U3scdHJp>eQlb+B0 zv_2F-;zQ9iM$09aeq-a)1j7;+qGP}Zb{M)Zt90D+mK2TuL>453X@F&9P(Hd4j!;KT0ffQ#cwh7NZbtB2^9cpw zeA(QUR_2oG&m$#++XI<8o4<)X6Ek1K>@2=YD;umiH@0=(r^f#q1%_mITqc8XOTq3xsxh5NPC+f#brlgMM7-O!3stN_i&QTe!4&&7q zU0AYKeO)78xpSk0x>0aoPU=oknZ^R^(k38J@Gsm525gM-&_PHjxKEw)K+m{l{Q2zi z1j0a0L%*EyC)<*mQhZ0?-ZO#7lJG8j>=S~?2H3NJ{>GXJC96=UPb8(fr|z>pAH+0O z1{q|S><25`HJU`tX;wF2v%VdEJ3>V$U?FqrFSSco?k{y|D}F4=tDV`KnQ$cLRMe~k zIf%pEFya~#G%E1x#bs2eHTKG{D_Z#u!_r^q1A?IJV-)JiZKA}6A(1b4q=mnISfv4C zsW=vnLuJ;3L$-7tJDHN#2Ko9|Vha21Ky#K64Y_hY@#KZD9Fu_#7E@i~w=b3N_D1Ub zGlk`+a`y-D=!WM)OC3*KOQBS9Yin5U&^i?gnRoUM7Tn?!<H?<>D~Uk44Z3>aQ(bw6O|(uw4(uYDJbJ5ZAm zO=H>C-I9NZ@37p+6l`&5MhTmhSMmKWsF}LNn;zFWiK%j47CB#mWS80}w5(yIP3C4^ zL}4hDO7)(Dc&r|TtM>l<@f@X8Y_W*kRkmqb4v*Xi0Xi1DoRnpGsOdFAUrBGtk_l4i z>Ea<=Wx`epMKBjkx_&jh&jI+nm3(!h>|%Rrw9#2igU8iY3Ak+`qXZv0Kh^y7&0it8 z!aWVe@6I6?nUSM_?960JRJ$NrwbfK6|1iy{#~%}zq+!-CPGj&^4SQ4wXk%=Q|NB~K zS=F_dkZ$`*-fX8xl?D;nQ!iDg^g&Tfn|Ioam6D2a^c)P7r0w=TvAStYlg8^beU00` z(2K$Q1H2~0CrP6Y1q)PB)E;H>5~;)UrD{TJvWTE+%%~sLGIa*lkX3yu-fTA#*QHQ+ zo6cxHy$4olH{G^$)bnTk{wYK=g-$NBaj3Q2l1xO;I&f#97QSLlf@#ER_+&m7V#%Mj zopn_@Vz0O5)kH$xT<_|83&)3ed0a0&axu*y`k)3H^AGOyne`vhstLyij%6b}%8or2 zQR^}V4g;M_24?JJ407^U5VlkF#R}XEt3XG1JKmyikR zB(wf}%mHa%0Vs~T<$i~@iT<1H1KW8ec16A`=RMg0jD20Mi*RDxO)Y)3i7>=2pbY3S z=|QY#qS?Hz<`4+Q`9Lfdy64Pw+oD%atq6xkDHHp#*x(c`NX1nV<8ca^7w>xPsf-EK zS8AE2b?9Xoikaam#S-boIWy;uw5Xg1G6)!Y?^|sOM2n^x5XXyPdoqT`eBglRJigW8 zlU;lH`t4eID%Wx2(xu+vwMBL#EJ>pmF<4(4)Zj&i#!IO@5i)0?`UlH%mV-l^RBSkw zG4k}Pw6EOD8f+)3ZWTex{m|SQuh)(B#N_5(qP!e;Gpr!`E2M8&MBzl6{i|Zl20VC2;u&k zLXbh^JVvMR%Bv#v;?qsJjH2;Bw&w;I*VYwplr-{o+|!+)zf8xxQ!rAhphbEnOV4E( z!76{G*+M8pJ+zAPh6ui#yfz1hAst5ZQNNr0aG~E6X<q|5qq*e*nvj(r9!YXBI7MjcxQM91>^x*ih94^c?cUo@TO-z? zeiS!7e2ze~O74uPXS!ZsE!Kc-X3h$Np)Ps! zf#9I3X848D0~y7(-yY)YqVQ?eY2f)w)n`N43R;xI&Gn8;M$@!-eaHGLK)UCu0g`C&e^Ysaz9hKX1QXQ*=t~#jDLY5IBKZH zVUiBgmK3ewOcuj^Q5GBama%!WR!4*>qOgCYE@r^#maCQvX_?W3U5a+&uxjT??Ct%e*P~L@-Di$&M{tB89k+La_(|-yZ^Vg_Xsh8Y3P6zN7F( z0rJvcWj>?)Q3+PvWiD7c+(7HO#1XO47GqIyGmQqrOke9Lc;?fF(PSKod=K-J8GtHX1Eh1(MNO8DU_>+?zJLK z*sFMIkDO*oLM4;@uDXj$Huz+8Bb6jrvI==ytIHuT$aP%=PYDu= z{-t0rKyzKF44f*Vh|FWM-M1R6`WgjVBittW@;=it6VFWyTT_aO4@@tpt6b(jb0?gm ze$aF(LOOMc(BafrG96%C1o3#6q*R4Im4Q2hZ!gsCXJ>1)12q)V3+-Hf?ZsaU0+E} z*yjERs0kYt1#+oMVhRS9gH3r)@m$ z>%?|;BG7u$^YOJODfLp7*$n!O`ZOO5c8*j9Kh|dEScbDU+6_%xw>p*8XP{IAWTSYU zep)7Dn*D7=YAR0yz- zQ~>0?`RMc|Khm%fnwdJ`YQJ-*UTom+?vH9lM(eP7Lx-m+38-Ita<$65de6&01{R9; zsj+XH*57~b0bXQOPmkZey)3)Mr!evDd-k+e@|8(z*S?)aS)>}4sX1aJDZtHLbi%9^ zq7L#us1alZ!+YE$hSbb}`?&ZfZkjDcnpfL&^GBAm7@|Vh^XSya3VoftYnlAC#TD#6 z#S`%vsdpiOZ`p$K8Hdi(x$o6pVdv7@7=6i1qv-EYVaVeU?^elg2&F>)CxDYLqb>?- za0f*@RG=_+qgCH4c!~&>#iJOH@zxAq;Ny4Wp9sp8Q=zbHBR$9R1e>)HP0e~0^s`U;3^(R5Ih#vd&gYP9mNcvC1BCKM^UYbEodcDvCZ#dsQ0a@OniH{_^Ty(j9+%em z4>uD0e(Cfuj`sc!(wdK(qLx|KbNjbr*G zrBUgj@v7H7vBy6Vw4HO!ztG0@#t2I^uV--gO^X~qQ_#^+Kp)XN<06>aS@7J-8ZR@( zJp*>0yX{8h?QLW29Hxyj9ct*PZt-3Ne{}mDZ~E<*C)>ehG>U#d-$fmiYL0lfknpWx zklUo}=Nqmn;}NhVYG;I3kd25P5 z@H1x_F(NPe)^9gj-b=siiMz|6Sxjekk3OsW8YcL$NQ7xe`in?qJ_)px9+Sw$dBu52 zJ&(SXAM?ZdRafd!pQ%)q*#aG0R#ZF=)R>?VpD@GR$yO9vpWWrs?*=O{KdwCONHUAn zN_sG-5NijCd;2@467i-w=g1?RRw^M-xVY%&kTR6ST_~W8jnc41mTkyXthF)Oo*MCB zj?G1iRq>ycd~#i#DlVd0Z|ktD^rLJ@`u0zFhdW;%0)Z(bJ%pRnj`$5;2b#2mY&g0c z8`p_dHu<5gWQJT&btWIlV?Q)ks%M==qcfwH@cIWLyHylo{S!hjU+eoKs8=9iZ9Zel z_o-nA77>K2uugU>ce;>@3C`6eeU;K#s+jklYGAgxvs1^p@uP-1{dw=P0^&N zOHWgDtVGf?-l@0y?h2wbe?WGzOQ{v4e(K?#mC7UDm%F^dNW{SCvfIphwx>DL$n&!I zv+=7H-^Vjk=vk%Py68Theb8SN1zP9EB)xE=4`%x zNfqK&o-w2_`@OHxks3BBHr#APmN*vYBQSinyF3*jt6~n(g`!54PI zWz`g2HKM)WRiADQlZa=^D$`{y7Mel^Q49_>P}Rx%;hlkQC&wICB?Fjq7CP8yK1ey z!imcOM_pFFKQtgOx4O?DH?Bac7efPcWIbyfqpk)Stjz9<@dJbCLe}CX(O-4e8kR#x zxAf7(7O%Ayeq7JrOpqpMOV49jGA)9xgI~2?vPa8D-8$u#QyNTC5s_F6H`GPl9ze@P z1wcwe?e^$}`X@#xiRMZKFoByP&y+-xtI2D}ATU%1vm4x0RXV+WztQ@>74m~iQrMW9 zkh6Zf?i&MgauwEYMco)|Y0&N^aa%n>rgc4OvSfgL82@xYTQ-xo5p>poSxhTPDHCGE zufP>fK?7aFcg|c7y3SMLzc6LnbIh^!dikKLmobPyzg%_PZ)J*6f_;iObFm&_ZlJbn zLtURaVe4@^o^pPt&Ev9ZvE3BtlvIL4rsbCTwNG4Aa%1i5TJd~rLX9%Q@}&fk3>02P zsg`v{`y7HRom}QutF0s_%XY(?BY|c8G2)4WL+1&XmB+EC%{Gwe-muo$BY=ASdS$Fz z+a9IcwQHqyMJ*cWkb8cLew6~br{MD#x*)iTV(Ebel~}!^YK*(TtSan$UIC9X;x(mK zEhJ#hWL%_dTrwHR2;4EzlJT_PII_4k7Z z6nAd*hGH){*4HM!!p4^W=l|f&w1F|cg{k8C!4vB*g1(a+?`H4hG0U2w|UJG&QbUZmkRuMGx`6!`YRhQZg77w(EGm_?qF27 zC`_IQmvX=D5^-ni-`?ea_|;bwX(jl*;os7e&Zt)}Twj|)4pI-q%)L1k z0(t@*D4~*B)8Nl0FR=`W-n{ zY6ms?!Rl!E?^m06`CG1sq-ZShXCR-+lDq1@Jjf{Sdzj_W^REyd6j&EOiwC_8uTs+E zy#FQI34NgFzQ44w&CQ>yPioN@6k~3D`q?C`N?PcOtkv{>R%r9W<%x?GrI^4qcR;)m z8C0LD4pG{)Xfb%?BEM^No3E&P#M1T!TI&VwtCfrAf-U#*SqP_*Fv0wLON@oq`;EfL z6;#xB@3S;J?iI^*)T7j1c?H+TR1IX>o2K&*7P?odkRNK-ulO33dX@l*LKaMZ$MeR}7WI;w}yOPCD%Ls3r zUeEWF?Z%TaiJ!214xNZUOSe<^jqb#n?rO~n(;vJ3B1i5sm*^c#a$Oz~?TwAWsPv$; z)q|Iqt?L;HSPZQql_70KQ0UL-z~f6O4{Z*9khVDeJWEKddHUl0Dm##@?hs$NAC&qe zm{pOKAMyUGA5)&o_=)tTl^YVYqT?k5Yw^R`s!sQ)h{uy70o=N4q~`~2Qike;8}c%m z_VCV)lZ#MQ&1Z0fs6dPZClW&Lda;3;q&y$lE42JXFcotwVa(AQIFx#Lv2Ki$NE5oL zs|qH!-c>ef%Jb%O9r=VFEIJ3X=?WEhTj5dK?b_G%Iac4w9w>Kc5Fvd(k2J<*VUpxB z0^u|!AD>mkzLn#q(DM!Z+PIstT%qmoOnUihP|DlrZcDqi?OR!d$K|a!r)3un%F+kz z{4?ui8W676Z3z9wLFhN*((rAV2m^u4pZ&sD6#=4g+Ix61=&)-l!WdE7EcMXlgHrK` zcODwKo2XfPZb$fML%Za9x-D4L5g)qyH$`z7>;sC|5k*;`k8SV`4-6M$H9x(Le+=7m ze^Ag1IZh?~Kze@ZHhlD9pOA5TPcZ`SVEr`7&yck^3bh_kzptKc)QP&xJilT?>9}fr zURR@%J>Kr?e&Md=ty635Jk+`7Cjz{jBT4AQb;x}0)ANoMF+ya_ZO@_RF0!4yL$(b% z+4|nHs*Y$A9YXxe-voH8V6Zo0zuffRa&YZ&02;W)wMY1S@uSiCJ|QG~iMdcBJZTiK zw{DVJ>9DAN0kxAOur>JO9mtarFl=^Xo6Pq<|K&fq`w*;_tFC3Bu|3iB-ho>~Y>=b% zvHEAGs&35fs%0I73}yEBuqt$JugOwUT()7g4;oIEm zmp}vc$?xLrm8Bh6*z?Td5o_mSSy$+mZSL}gC@>Mg8m&!1qQ=y4p&!g_rUb`LT7ZgD z{K_kh>o&}j%Bq;}8SUcvc>SIw+T*MSjp;0cl2-S*l|wjYSN(g$Hc7(40ve&qy;fGC zIUQhW(PH)2R{Or8Cr=R%QnM~UY{XP+1j;2}T1la&VyX~M89oIfWvDXR5g3Wu5EslL zIHV!KzGn4<7#A~sL}oILe|9FOTvr7(%~abM9NdN0B^YSl+8c+kA-2A#t+{4OQw>y>;M4qKWeK9r2${a@=M&W8?Ob$(!Bil)v@;Tz*ml0YgPr zEu(Aj%BftBzdSsgv=P(WPJthTHJkX472ChH-(jIYItg8s$0umdW)lXkHzEL;39;o7 z4TSib+SwC=?;j*K+*fZh;$s}WYznn>Tbfs%vqnnWSpc(eZrsB{P_$ob z>Zo-lapQrJ8%w^&$#mU|t9oeXR%L@leLe*xK$^tLJE}^KB*aO`Mykv+NGmWbS4=B} zKT=XacIN?p8jQAJUvtHUYHofg{~(29a;RFyB@@jZ`p0J;GJl=&`}2*4 zB^Qz3<{o!6fss@V_Fi&~1==q;$n<$K=TS5S@6Uhih?5;#U!Qg9@osIMmpe+1-Ij&2| z`E{SN($;XKZ|MrIVdR)nfyEHUgLVZaZn#k>im$1Aa6 z*ke@vaao4BB^S;qwA=h9iLR?Qy=AR8`%#r*y38ns-6$U40YrjtCr10dE@pf-bTmxf zj|4Ond7H@CWj4H5w4AYEXFm8Hh=Xs4uqtFTXdtZE`TAeI0D6A3gWe5LY&t}LPrY$t zYBUjW)@16M7;-u7$ut9OB){Dm(no_4LdcMTMz$V+>ftSqo!Rg=8P=;v4&#L2J9N+L>ffKWx z9&27xQo(z$hDaVlC-<*qTfq>~6DoG+$9k31UJe(v@ZV$E%{L6CRWrp_*F5{6FpgwbTMPEA`13Ubft($}nyplAIyYmOUk`%_V+k_79 z+O6MvVjSCSTM>XD!z$4*- zf%j#@TYHe(o5cNJa9n5d~qN;xI!OBMWq%G8P+*qvK7150Bo=~Uw zDe;BfdCN7qhL`7ng-_udf9&7k?4`$3Ooa2s6@13g5BG-!A_Y~NB2^H zc7nmzx>6p4c7{xU0#=qMNC@JLTG0uNJ+@Mv8BII{G>vEPbw}TNDWET1b4W~NXs9br z0HULLp62aFOuUL>p(2=BgvJF3%w%ud$Zo)^S(cE&kEzNAp!gD2_3zI^A#B{EM1|ei zWCM|+?m&vU!$^Kx`iRz!7rXT3b@w->(p_h3IJS6+>ecJ-RvE@)gr#p;4c8V2Qasu0 zZJ<;?m*!$GgHL21p^xEKTmqJAfjkFlP!{uf(uB@QkIPQ^!RvdQO`qf*wt5ug<9kxx zQ-r<|0_N#wrzf$fc`6PTBsKsu+WK`1O!rB)sdMhiP3t)37Rd^mm`Jg@Zj)XaOikggPNpx8vJ5V+NFWgI8l;R~swG%1Z{?VA&tUv$g?^Jl{Y>MZ@wvPH*t(i6o;2y+z5gv(LC-Hw8W2T~X3C`&c54 za4;V4b4LRmUVG7TG6WQ+KSzdtjT4AlXlkbL*n#j5W zw79EBuv&FufZ86p>HKPpK00m3YQB2{>mMiag9l==YmN;L%jEmwl8pvvB)$oXVyreM zY&22E*lozr_670b>37(5G?;zhw^AwcB?txx@xmU0o!T;P+J0;uJuXi#C>V9L2h_3w zHhgKg5}&ysqj$y3|Ew{<)AY9|tr0AjqgweQ^zpkSBrzYM>`q$eh!74kI;+gj+p0|lvX?kNubER-~< zhJmKbyHf|+O2Fo5g4ad-+}p;ln;bs={=nac`#etg>6EyHFQkHKyjbJxte7pVuo=Pa zP3*4Q<6_c%HveNJKCvdS|F(QMQ;;MatF~aRWsXVY!k1oNwOD6n77)}XwvAo=kQN&m z`xslJz1_^LU&HPDXMb{M&}}_k+zv~6`WIB9tBX>b4eL&ki@qr5r#BYX7_tM-B1dQU z+sk({Mm*swXk!AHc(-&+yhpV$*B) zwMOS!kiSs>jYsi_kp*c_SIVz!b>p`YkN{!x#d+3zHU^ONxj|5?mzD>F-_!Aq-Sey@ zal>p1$VX+i1+76J1+YB33c(c&l3;H4soo7`fViP!v^PmH3fs=>jQ9u?`E8c~0td&H z3@$y*c*MveR&Me6-Q{y;yEn>Jej;4a$yM7@gwTf(@0=_dEkI))c0LIPfPl5;9NxF9 zSKH=zYL?ALmYyiWELNYk(=8)Ej=_X9rV-X$J|jvt6A#^X=Nf$2P!4r zNCVZn1I)g*ME+W4Ttk2!SoZ23?(p6zb?SK0*QfEl)G*c|e{7GCt!O0({ABkfp8Hm( zc8>d-q=|0WuKHeyORWWU9?M!st8qfnIa!oY{yT|zEPj`yN-ocO%0MkY@&{Y|L}&V5u8#!P;CwEcw9q z+=7t-CPk}4+yZ37O^RdbdvXC&_#lPthn~S2;tx8%QwyA|&MLeCv93{e6PY|hrO?G|nAVod!5OZ%2}lgsFr zCHHu+AjThOKf#AEw+{0#K_7|#D-fHIMH)e#*20-c*F*6+cxx55*hK>1(72{8`56cD z{^2O0p?Hg{DU_6lyrKjwx*}u!c?OHT-?%PbJWv}9al%(CsypU{^muxfBEOu2Jv>V; zcX}iMCZ(;=Bi!PjRe*hvk9#vXvUP&3EQ%`<%B@P~ftE1)9b7k(!|~8pQESfO*lsTfqM8Q6b{wynf2E9d5?p{ueLfBkOd2*0>07s zrxYV+l;NtgvT4&Tm3feyXR?u+XBtX9$s!#)Go6!fVD*ecVdhNrU;8z6Qiw7 z3x2Im;I<25jpD5r3w@mcEUaf}j~)Np7(-D7_A2eFc&(%?AZe z+?NO8_*^z|$*KfW8?Ce3Y)kV4vN&17nJ;EKT?)0!4dR@?pGQBgf^+SvW)D(Qbv%hk z=4Cz+rcB(&X!h@moW0TUv-oNjQ%?1+zH87aP5#9L2@8UxZb0lwe-N}2u^sT1HEosX zlUtE^d0DvLrOo;$yC!=(!0YPB;hwqWI)k4{;~joq$a5e9M$5TuiVQ!CBDF%zr+U#vmSklYnY2zm)b+JL9Zj#fC-489Z@ ztkgi^?^~&}8s|<^gzK1-x7oLvxxX&$4wUn$U4_U#V&!4XYG6Bo`OQa#$xhy8ZZMgp zD;mh*;lq~uM;!JC5G=J!9eJ$wMlGI9e>eE!@Kby&&Xtnh7F?w(4Ku)zihBKN-ot1B zx{Km=u0_EhTHySzgwnX-4^W5-=l%TJcL9#2(ECx$R*Z_37UO|oZuAb06D_IYoi(L7 zjgt)*O0YTQyhZ9T=AV@jCHlK~@pY`)U&~GyE9$sPoncq$?Js@wDTvh!Xt8QaJoy;r zSi@6SPZ&r>#w3K`OwO?y3f#MpL`;c)?`;`bHM*-G?&bo_>q~R{RW$o$Stl&3G~1zo zN-{f7JO6V#qb1W|l{WSOO>a;pjIi6g@#QGs_GJoZd4F_W0=#2#4h{sIFojn)SPN?D+~50v?O#M0a3IcJxC!AMBk0KeY5Kl#)QT_F9bSNy})a( zh1RW;q>{uIqTXK!vt6i;63*%1X2gTKtJ7c3XbQXdwor2`!dfm5`g~vD9Bm?VG&8(^ z$Za%{K|4_Ok#=LJwN9TjjaY~_PiE}%*qFoQ)o0sv&I=0jMMj!QIC4^3WWeu4#W2* zTYy~4;-C~<2h0iIRr}-TKR-0PHBPhOd7o1n5_{JnoUyXK`{v7aquyt9ceW_Lr6so9 zWZfMhfJnx>fhhSO?7O}1&G;GFMOOkLrw=S|JD!Y(QXKlS#(1CO)MRmh&*FD)MJJJB zccENFi+GBXa-QKS(bhSXOkP zskh*!%e2tlyPPF7W5pUcponAqN6N^Og*Y|dDB0ZAMx4>n7PyCgvN6DCvB7G?F>7V! zorNks?uOYdcC~Sx8g#--Zgtm-?;8k79FGqmxBg_84>3w>qDf)8E?+ys@HTk?U8Nav zN1nmZ*ZZABppnB38$Bz%H|GxzBFu(#9?c`7eZ86sL!EEk#73-0$wkI?n|yc>|B}v5 zLU4<9=y)mB_PJbio~$65O(2wapO9#;TBHz_V;P{_rV=>QCTsm*NG?S>qA)+rT%NnsHna_+x`y{>-+%4qGH275 zSW|+)DCHby}*(JR1DqNXCyI zG4ET@`X#skqwb*}b*#$c6S3Dgbk>@VQ4I;^CncH^Xijc#%Rr(d6v+s4Pq+tFP~uQK z#=;(9++E5kRO-GZ&R5UIrMC7%zOA@FCl&QE!~v4~l;k z9dpCkFu~Hm>%iByXDP)cu<7-^g0QrAAD>NRiqC9BO>q~T*U^Jw@6YAU?P~JVX#o~9 zXLm~UGaUy2Ba^q^I?9|Kg>3eoh|W-ezw!i9ZQuGC?$eMhIxjbM9LcG6W+;fwIYBeF z{uG@(`y#APd37lGLl`E11M8*XcEM#EPx9YdJ{HZg7E;#+AY=;Z3VPTqyQzM^`dL3d z$l+dl!aMJZrlA^4NpC(A7I}YLG4NG~Og-46OG*8NOA`8OF1r9+x7Ta;7a*P@oYr!h zTjodbH&Jz$=VOtyD}I+^WLnrp+&#qhY6bGXe%&!qeQ(QyRJ5;NoN>28c<)To<>OU@ z%ePV?6ZYB9{88Y`#p%dalVJXwi=x~7y&Y4`OJ3e6&d~>v@R+QURth`W`h+LU8bMVi zyX1w2nHd-=>M3E4Ss$#>s;xSPXqp^;Y5BT)a)2}zDk|QcWUOz#g9UEKg?SprjsN_G zeB2jJpSVwZ)y1sTyZFG+pM<$CQq?*;e^W4T>k8q8p2u8d06fsUNn0e*6$dk1FlB^J zc-h-7I1hBxL+_;ZI>ySh!hq{-FM4BvgEy}$YipzE%H)=wyO~Z}C%;VL&!Ge33{nZXP`fv;xEsGCQv)x)u zpbsrZ2M9|DZxBz#dkfT5cuG1fA>u8{9^pn=_PC*Ez<#j)V@v~M2%nT4-zhKc3p~aa z@5H?L?As;ORy^IPC!)jXuTQbg z9Wvor)3@8~xw8*XH1aAF=j^hCX5=*L5yFCHESIk1|})7^V+WR!+V*1X5X zuTa!?rAR!U$1U&!{k;rgS3v3Ibq7JDE92tu=AEcJHVtFIgx&B-ri&{0*%Sv&A8?q# zF^K_Gl-s*vl(KV>C`9q!=r~ODB}K)a>XVtmTOXKsZOS(;m;HNQ0`hEwKLL zbcF4Hw{##D-}|fl`hl$2%b~`=MzOPeoD)+?xX5Ifp!D=0~boe;dWZuruW z(x82(s6%tU)$t8s0rST$SE|!*0m1@=82}|Bl}Dl1$8V9L(;iR6Gu=qb>N0_P{WNKp zLc?;^rUc)yxR=g5B6UH}_B6bPA_v<$+P+f)?&y~Uq4)$B4{x?fTb^#WTK!spQDA6$ zmE_e4?gl&54)+inevJ>p6PP&Zl$OoX-9EbWLt##N0L^D_OYo2?h{zT(rY;ZxePO0$ ztRF~kM*um7)%FZer8jMGQU*U6(5fqN^T~*WI{ZNJ`$kSw06by z@al{$g^E=i*~dps)}y3SqFrdmt*XIUk$ErGx3p#t!D1es&p%q`5+%+qQj>Qme7`b3 zln78Uo!^Epf>wcfgA3Zc%8BYNb7I$atnG5TL^C*?zJr$lMiC$YH}{7A1#=__ham1H zi=bZ#oZ0`ex{I=EVZL5xu1-$5iDn`OXKfo zkt!qxSU9>|BacJ*vnb@Un2~Z|gv*IwQ1kCa0;k&hWyk@9eQWZ;NR6-oo5I$nQ9rxft)*mNo=)x{u?DLy^VbSnsH1BED_o z1!*sk8cX;Z&`qRYBslVbcRa=>n|?(BFEkaKx29Q(r#z$7TtE#Qi;Dg$Sj-2?W%aE` z{CHG}H0wuJ@-@K%Re2zzuyh^}d|CuvotEB^auOx#v7Xara89um;C#Lw>7{V1>gP3N zYDFZNt9UTflp^kT2%L4*4WHNTzn$K~aT5&bC`STvw_t|F8vU#xUXdDT(>qA2VO40q zgM;R09_1ZR+C97@Mu0@8yL3DlaTT90u!J@hSo8p|el=&vEV;^Jo-C%Vld0=?hbeBk zu}qQ(a3{PN@3rC^vXSM`_QDHax%TzRIr)H75M$a9<&6~>! zg*EX&G;G|ldVco0(VakT9NTyRGIsHtenDQSatD8D{Rvs#69;+cmikDnwajzy*;z+Y zf3`qra`+ydGkpP)9q8Sl@_L%oV9EZn_2akDkixmQ2AP_(+?7Q|l z$eT2%oylcZ5-rox<|DJkoRO{6LC zI0|8;3ZTqssu4FLL2%OpCBF3q<#7uUv_cJc>1klX+8aU@c$zru-ZaiWXEwI8(20A4 z`Vl5jW(uL1tM{MxJ6<7Ezo_5sx1ggR^&yZY;bF~zF1OOo`M8p|dFES2a+sd*8{!gC zOPX9?Y_inzzWcB}3o->wp5BLzkDU+powle9g@!exlpPF)*3B{5i7*1_>2P5Qt}ZYy5~8f7e`k50)V8`;?K~=H2S0RNCDLDvnCZN3p&4@f6?I z(ooT{EE~XgCX*DH4b#ReKGhy^?XqPeRDH@q!BrRGtpS0}FGdURU(3Lb#%FP_OQmQ= z@#0plU*xa6voYBBn{M4C^oxGHAna38B2@eF&`e)qJ2f0hDRe2!&K^qX^_JXbYV`{3 zeMK|TXR*2$8aDE?t4WtvY&*F7m}PLpZG^`~{~Rcl1JZIMk?}m14x<^Kn@eIP1+IHP zH7MC(ybj7n>vbs=z3mvC_T$sCm9X&Xi`wE`hYU11^SuAON}9){vR(T|Ws!H@NzHw= zdXJLCytvC@iMKjhpTCAn*&Lc`?M*irY|6@ZGWXcI04_x?dB>a@X-9votTCpDr`_^8 zgOk7^yx)#~M_P0_t89JH&RipnjE1EU@pm|TBztpX=g5Qrx6PFVY?<=iR(W=G?6n7y zK2e@W+VxnYihF`ycNd?ba!2L878#fd})t|d0Yp$z|B?`k# zSa$0jD9dTd%N!EanQ4b`-;V#%9#~j;r}dj&^uh4ym^(=f+M;tZXOcY82&N?hgnvJ~ zt82($+B^}LI5~t;_d%NdJW%q2$_G`l?~{=!kH1L)s;~^=pz4ro@h0WDhCskwmGFu? zp|aU;dXr6AAr$n$1n}}auOf?9b1}RXdr;AfjgY68O0JbTzl4ngx_s1{s9ac4)<|ZV6 zmA4{cXO<-RpsFJ9)w-2}Q=UEkR>-itLQ;@4*;I7vBRwLBZ5DXrz-m1El6zl0uf);I z)-jy)v1M#P!L3i=9^--su=m|YtUhOj2#o&5hw?Av3(Yr9h9E91fk7>2?>dmaJvq!X z8Tk+Kfky6Y^YDOaZgxvTZnMKV+r54uZXQ9Du6_H|KsB1s5q^-N)Ph=_s+Uy#&g49c zeZa0AumOccI%XmLsf)Zff{?N6W(6Oe&0{lF8DDKgp;DjSS|6ut7NU5b;Z8T>>Ji3l zw$R*>uonpe=@INWZiKk;{6fSxeex<+nHqaJQ*y~iWt2L=$|;FG8&(caB?`pi1_A^El2>4gfz=?rqS~l|Hfl| z{I>fyn`j)vLd7IiKuxOp2~N&@k*aRPm$`AQG7o8=;XVapjcdf{B*)Ze4m)v{o&y2PT0_A z_1j(=hWSjQdaJ&AhmuR>YdTiAlItz?p}e$dOI~4{9RN#KH)}55b-Bmw_oUq^3O7-+ zXH*TJu$HHJHdb#xk^zAn$EnCSk*8(ASqjSJQx5yOlc|A+UotapKcg zIF9jKVu@dhP=wVOYkfi?wK{Y8VNdTo5HH(n*m{4i8>_NlB)+q;wo(FKuijX(u=dq$ zC3Mn?$PZhoN%IqCh>_5Nu{;nmnLWK*B-WYw_J+ z4^JL$84nwWqz_1U{yx(e37)L)9|S!}gzoW(*Odji5`PAs;_@T^)&`VS0#&jt#2G*&;F0GUVQmRC=`Ipc zCr<`d8}4!Bx(q#8x_R0%zN=Cnks@sm%ewy-0Ksp&huW*ul7&Yy)Hud^yq(J74;sjoN9#E*o61ih2@9+N9dkH~JHugvcM9&q@iSt6KU|A^;4 z7Y-M6!@gQ0b(W`a2vN~AI$D{B7h3+F75I<-4$MWqk#F{?2E4~?Ncs26|M3f5+m}1! zFheNro{~V&V8dvo?9Tt~6ZGmcIdsX0rsV{8UzdMKPC0lIu50`?gy^vv6oW+lH`Ma? zjr_im9|-hptSDN)`PaSw<0gMS(N82z8r4hE{5AN=hEJkW(j(!5RM)rO`Ty~(tccao zd?)YY#-ZYf%j&N$hmicyXbBc=jZAt}17iR45M?t(P0{*&uY?QyXMc-~9ST!zp?_dX zo%Tm}-%tAAU4@I>;9clsX!x8XQ~$Se=gwn059B)QM2ue0w!6GHOz2RpDiG zeTaSJzQ+}nxYthB=n)Am2YYd;hCLs4?pZSM^MZyQk#biMz)kDRbCDO~hi5$L0t_TSpoe=@@VI^t1}BjpLN zqnr{*ud6vo! ziSE?521@#F1}EL&iv@Ss$ws*xc2`HyYhZ>49Pz9k6@6wj((*F!*aNM7gaMJ?!1lh4 zi+sAeY%}%s|0AZ3nb)k?KPeA4Uiy)cl0Lw1L%8Avuh$5e0|bgEGPf1Cnw1YNbx@kp;F^=OEgH&0@N~ zRXla*K#zB*uW?6rRJwLoNNpZ#u^C%5=Bd5VXtg>YZnC+7ALqRD`D_fi%Kq2jRgx3O z-L*!k)0JiO_FXmjjVF9Z-B4~P{7bcsb8#=kvXTzv;c#ewULbXuyT+rXaz3335p92z z+)2@2>}<{d(*70G0c{3ocI3c)KSgA^X4vxO^PW+I?hIzR*u%|^*0;+eCrL~z%TRjI zTINujR;ag)NotYz7kVCyMS}uzo2y=X5>B~eHOdB&miL5o%Y;@?18E{jr%r1Y*Nn-9 z26~>%0PD72OBEGXcjHbnzXj=~7J2;O5IfoE)X>sR>bC`33^>{>&&&qBad8&>U|f%g zI)gIjff*unP!SKY#l-7ZTJy2(aPi0E8)EM}E^Y;bjyXG(k^WyuTv@-ueqB2bHd{+5 z3wi-(G3T~UZ|!yBxYq<7uLR5laD6A4ACN=(`XABBX|Cro2irKX8XJyZzBC}oP)RV* zfItuG?0Otv6K-GJn23+KqB&>QAIBGfv=M0ITc32yYc`+;zO_ zb+R~zQ|Qo%=$3D&?GPmwg`1V)>*?L&b;h@MU-z0=18~z_h4JzX4ClzYpTIb9TU8rfE z)1^s5_hMQ%Nng>jwlAdZw^5o@Spw8+kF|Xmq=dDhI@E%7I(a*Smc^@qizpOd~Ax$@-?*!*BRoKh&+Zii#A0aN;+Brzn?NE!x7Pk+{ zwrVNnRge9TJ|YQTmP{I<&2K_?bdY2`Z-wMo@*7?nls7FKkb=FZ?Rn1kIKyJ#Z6YR& z+AQbQi{{yuOGgFU{JX3xZj%`C`OmXwjtJ|lO=Ox#`}-Ls4x8~@szgotc&(Q(vL1>a zd+!da10e}wwGP)u%l6gEpiL)2*o=JJ))#JSo`AWJ2FQ-F$$dANdZf#PoN^OY~ z?lAdHa2o_IJB<&_wLQG3WG_wZbz16$&$p{H`7@q-v(I4^-1=*9TlNSHj=Ra*@6+5} z(ZF`LF&P(Y@;2?`C02)!OE#Z%0|gokZp~<{O0YrM`)rLFliZ8z!TBU9khiSq?KS%@ z^E`}B2_{;Tb^L`NVJYT^j3qy5WnnJ}2GLe7>T{$K&gx89fv?bLjO4BYufnRCPbhM1 zymVyHvmmz~RR;Ubh4c^`bwB3fxs8qQ;w=051je%Xf7!phlu!^guUGp@Dvs8n2xvX_ z%3$(@H8Z7e1+A$;IWv>m-iG8u?9RCDn}OhxdS&RU!Sr+2lO$~OLh{B(Y&VQ@oCQ}V z%}{5s&D%9EQ_ULM%aMh6c;7jnIjG3HtCpXv(DuT|7cD9lLTIMr#1`_8^CfooVHq97 zv(V+Zf$Q7%Zr1J314#=-7w>f5Ht?l7m4e+|G938I{Q4bT_LSY zl>Ari2u&8MV=EDrZbj)Q_^AN;J8E^Wpz8v<$x{Gvpce(FXXUcO25)UEDM!q3jPa9j z$xx@c0T@$@my?FKtyOIpwMDGPNZ9wj4^j1u3q{WlX)o-d8++SNFfzypDihW8tm0*8 zhu?d8-vWSmIB_E$fI>A)+0-UNz)|3m2$4e!@0?Nk3#oO0U`kh(_msoJcmb3Pegt!!zm%P9BM3s8k_45GK~Mq_XIH)66z(7aJBNr{_&0S z{oR{}t(tzWKxwC|Dw~RLY4MDcGcsdsfw&KR>WuS!!R5e?Tq4QkgC5VqCbT{ic9|=e z9J4_6wl6cECFZF)VQ0M5_o3{h>Q)?rIyC~?sGtjdfmHA)2xm-cky^@;bm7%&_&D(3 zx#^AX@RwMNd6nJvrk8>f40orgRbggSRR^c6GzP6+3#%s*)Sy1o)D@kKvix}Ih83<- zKTsu?qUJg^06m*%UMT6xmh>l@WjhjJ%%L13B}WPZ>}LMJ}>PyLs$ zuQy{Qav-j%#zIMF`n{N|F|aOHw8%rG25oUa3n8(joNEXn7XB6xI+?=Z@YL8Ljut zK=3TeP8!cSVhfuHmXE)e(5%}};|f<6DS26N+dmo^+*GW6`UpP`nN_p`4%)E_t zZRA&2bNi0;r35%?P9tpu>>Yv9Fe=~j^GIsKLnSa0Y;^}kh_%T2}&E#D|pA~A%@(fH!N z?$QYd(1{g2FB2|$8T6e{kp?x0<_mdgHG!Hw7DAKlvqzILUXXPY{4s^rQ((SM70z?kv;#bR3eYZao(5?3h6BtpprE|6Y(6Wi=1dxL?^B+Lf zc^G<=ijM{mj9($U{0CziFRBN$&2F>s+caIILuu)_eO^3U?1RZr%D(!tazi4ADmEPJNa=|*cQZ0 zEfI~r`)QV;Tfk!X(gm8bVk`Lm;7K^1G9mL!d%&y&pOGSPdmVYeqEU33o6sUM7_-{+ zP$EsI?Nx#6`)auSYHOcD=}3etu4Z%|6;PUy?V8EQEmWoNby?BFIIW34(6#U)2vMLDWjYrGU2rbHO&Y3{bmK-Mp(3t;VnJpO$$_ z)#vIROrX`#5FwQX&7^%!N|HCWFn3fAp>JMQ0<0f*9d0l6A~F~pVUAC?VvIZwhB!B% z9@J(5?7OKLSw>>aw5@y>xYEz!D#>rjYzSWbPSz^q`|15a*7&{i?!ZDYt_KgNzQ_O`!Pv$k8B=q{R)*ajAz+q2mM zu{lZ|0;gxrY#*PU;>7Cw7!kjV8c-E)W{vSb7|XGzwfaBIy=8P<&9WtE*|NZ5W@fgS znb~4ymMmtrn36u^iGuO(>6DLkYROHSbk)#ZY z@Wqs_UT1krY>Tzjq2;aez>VrNLbX@8i-jYTLisi!ihE6$>OsF((X~`J>Tb-kbqdh+ z4>OYC-j(C;%;ZzU#3a?{ves6nOBt|brA8#~OBBO6d(Vet=yW;j{^$)pL-*MREXGS6 z55AOIh4<`O&nGuS0mcH=Cnf?`KT(JB1RKhlEb5!iWcAj2+8Q@>FIMU01Ie(%J zNr&Z1i>>aV#PN%q{!z&D3y25*G2LG}l56W^%sWIYG%XnrV0d$TJ1bv%MXwmk_})Gx z8sonURl^;K^7x^N{**A3t7M~=IhH}CBTaoL3z20zC2ly;%)!N(9>MOC)1A zEj;9V62m)ENLP7cb*-Uu!FM+O<4$HEloQ@)mmsNebY{@j_kwfOLTv$rEDHOFK|@}f zC0eHp5guwjgW9TDhdI~<)R;bt+layhvi_-Y>4Fpl??0QXN{6wJh-}j*HVgXcNtB)0#S5N&3RbA{UsZFi^(EQ>EU z=wPXr@Me9PcZosc1ax(k1n5>OVG!C_{g@ebBSU65*OR%_HjeCuX0?5XgT$~O%dFIP z%6NmE;mW#SY0l^^eT4x)0*8Kae}J8*A{5rF1fn*|Clu;4;*#~Mrt@&%s9#zVp(+I8O~6VNun8gsFyo3~s%qv1m{MO^anlGy43_Pj8QWngmb_vL#aDPHB4Lw5VD7$g%FnLeme7P(GW4w` zlT5bIqK9$r$I6I#f}Hz7=grl>N#tZEhb??OW^7@(l=xFuX8%f2?P_&;5j{*F4<-4u zo;>MWWlk`}4iz4p+7|Sivn@Y-7$vJAeod0lwD`~E;zRi4Cu?Hd=na4 zAdALwsB!~;#Q{}i79$X3InbN+XLzr*zdgOnc^Z=AwMRK%suFnczR`|7SDWpRb=JO) zQ9;F8yxnHGHc`%dtpC;ukt@r09yWsM|In(zaZ9(Uis%bpd$A%W;T&Cnu+juVb2Wbd zq(fn~&L&RPl%=51AlMQ-j)_hU@@kAZ>qu}No@xfiDFNaxIDkvUxcg2X(1irQKc4JF zP7}z5NmswL{6L%j^zaCpBCYV$sfK<}tr2s%>b)r+K z`qw{Wtnf^9&&cTGlx;%I@X2_37GQ=pm(`<0OT0LCzlTWsH~voMlZCE<>grB6M_(zj z*=7=HhvQz&INu!6@qRD{zz0bL18({VtiED!7zlU22f;h4%ikvDA1}S7Rlpy! zcleQ@U|4`CPd$gA8gg#C&E17P0&Y;zYBsb7V9ieu%%cw4FXy5jNQHpDYSdCU^Zv&Y z$;y$x6Xi$U>t{f{3g;bQA@VzCt;ZfV-*$lug2L*DWOh7j$JLYLsXqmMYc-5!-w4pX z9#hMusl*n$hGXrX2v(Sj)O8)x-^oXFZ`U%`iVJ~Lg!d97C6f@W*34HPhr+O(`spt3*qF z+A*Tg;<`7Z9tk3)>kb{bT>2lQP~4dS3gW$*xq2I%5Q(`_i8idw6boN@3I+L5b`kM!mD&~Il`U{g7q6%l5JkJGirc5L8u_-UeI zp3G$XWcqYt=-c6Z(;MO8-EqD6Ffs|EG&d`!y?*KWvF_|AN-y=aZcOh9i8|`&FjLZQ zf75Ik;js@nZ`Z|sjIQIx>+l_di4=PKv-BlHH!tm@YV@jBFClCjmbN?eotoeEwzqIyQj6SBlxtWyk@X|(!#C1kFqpGwP#0>-q1!XrR(@AT z<*V>Eiw?B0#yzlt+FOMg#*xl-nf&beCuDO_=FLG;R0_GHMgDSN(+Pi-8uUOS=&U7I zVffxJ(dG;pUqv7R;YmwZoEkpijRY-d@OPNfLHW^~@yx6Rx=n8ipng^W<`aE7kMjHv&l9#f@Fv-;}j4L64b#}cJbY70PB2GbG3ZVIs3t` zNTYpoF&bO9a~SLAcphZAr^bxB&aD&|3twb^1oTiPeoWV7QNEkJGYGPs)zoa2xwD_? z*-SZk4E4jW?X#WviW6GCHF?M}nlDhM62Fg)Zmxx4(i&!T?5+u6<@{0yXxYQj4d5xG z?^$*~!8@}o8?`wxGpP$4Q}KLzcj8u8Xn9&RtJ^S1xGJhiF;J_vrjYP!LfaQ`VO&G= z_E(nlF+iG5`1(Pvx$sxgSPKc>-XRW1e8GaSVC1hh1KaGi!C>!Sph9+c@5LnWBE^o_ z=mY%mU>+f4N%`EIJTzU$%0D~Qe%f#o{rcp>7c$2PL4lMJMFkmmb9xtQmak8U&%fIz zVJ2q5y$Gj`%W&8bO2<1#Jd5=)@>j}d>%@HJVCLc(X*`BO&-I_wA1(Xq?S6@eL3t-) zfhK{D*qe+^Egs;IMXj8cLCDobQ7#uBWu;})c8Ph~NG}$Rj5o+0IN@(VfwV^}{}Lw97F?3;OYy3j6r|n}dPj zR`yj5Sy1m>YQ-#oaoC%M`AhXXB=btbI+vEM1i4Xq5B52uCJEyFhnV8_IF}^g*r|O6 zB>?fgpWa(;!xlW=A}e2r*qM(0_2;)q=5o7t8l9LlFCEBa%*t(roQGKJ9Ei*j0tWAt z56-lY#F?;VGVPzE8vgwVc;W{owj|ZrH+^iyS6hHzZ}01x4soO?PGe8FFU2DXZ)eX0 z`43&Xp^T9z#>Q`b!W@*=q09MW(RGa+@zQEY9|5rb$X`3A@?sr!2dKC%_@i7DnA``X zK$pTCIzN>tSm@-mJlnhyVH2;$mX|s1e>M0B;<*0s6`8ih-7ZG^>v0*Cov{xIT<8Ud zWRmJ}J2-XPjqbS7Z8&|5Cm&b2Sgz0eqm zl`Jy9v6Sy;oK1FqAFuBhvm_wR!{I_PQyUe1(o=1&X=ley|IbK^6rm4`ixF#?YFAG5 z=ii;9fww)$@^i{OqUhBG?mW|?1SNujF4G6O9TF@Jd8i9 zr(KwF;dP|K30s!5(ITlA`z08c$}fa;srK~v#Q7gi+dKA>CL%)R+BS3Ny2SKkzUW*s z_(tO92P0lisVOm-m*cCpSWUM^YV3PTaU1e!$ik{Prem(p{d8W=w)&=$(amiIJvZl<}-Z z&IQJVXnAI zQmrzL;`PHS#umc?>_gp*O>e{r%gbXhU$_p&oI0S?5X~DL68i^rSk#+EwQ3{mBQOg zb$-_Oc~!jUn2HO_$5~}ME%lK~c}f3ja;iUN z43h3ioab-(iCk{y!pHzXkwbLnEK~3atzAOXLJp^Hr34ep zbgZUE&v4UEWOMbuOE<-E!bhOm&bJg+B=;Y8F}6gllh)9+0csm5&pIVW!V0`xqs4l1 zO>U{)uHD+H1|98ni2$a1ugr$y=+kCy`uykYH5CO!t6eFL7=4@$UhBYjeXs|Nv!Pb# zgouXun%@C0FTCaJKo4}2!gkWl;lZp7mhJ-u1R`hkIk;{g)~N+=PZ9GV zthnt&mH3j{f*)~`DnhU35dYqj03?4rbTe{^I-V3+R~|BSwr*VP!7pA;41ku`M=z>Y4l+Tm9D%(SjJ^u zF_c-AIl20^ny&uA_VT3p5y}t%i1_I2;z2{SxtG8B2tONL9kk78s&Ibtd+ z^fLN^MEGQCx;JH3y{aKyxkjQAdjBWiqd%TaV0)XA`@L@FdGHtsKw&gFKKy8x;H(Hw zf!#F8Su_toP;OkG3Bx-d5CoL{5E-Gc_spv72iG(q_JbXgvo(6M6`u4wzsOd^xdV z_w>GHEyqrv3<2t3)EEx>4~~1t&2rexrCmkP5fNLPE^FE~dM0%i^PWQY@(0d}yVTqL z6E!%Uj2-uBRGDH<^B&{rK;*b*6QHEe#f;YS8_g3IHHjbFwEb~9Jp;pXM zI7+Ii_)0CtvRrR|Fgnw!&Ry;I}=V~|kj zxsnQC*O{!Dc6^3ZOg^Yn$y~A~1C=SU<&X^XRGgmCUpe|tm}5x>i*KaPTmqx&@WF%; zJyl?Jl2m^uBF=i~Ps*mZM z<1WWutLIxMlCt8khizrMGjZ=Os%xlpBG>DbSqwAJ*2+(eYmuXPYL=M9ri5a?K6@;o zch(HGY7NtX-tSr}*W$}LYl$3Hz#ahebOQ&orn8xRmZ5(>f^`<*WG=j<#@un}ad`r- z@yM|8ZaSG}I-2-9$9>SaAM<5{9q!;^lI)q&=MyiP@TjcM=TSE)0rP95yPj=|bzR~) z2(W_n5#qm!Pj)Mh&k#>o!ID(Rj}TZp>9a8X(3j(9(Cm5FOlPZ@$Q()&#`%X;*vr-K zmv%RNw^R!PqIQ>OF{e~fHMfL349f}5NlQm)=c*c9SRu|N!IxD0BM#9Db0mk76142= zM5cM)CZi-yr%>4o%pD!E9_kaonaw)~EFJYE{{UJU{sG)`qrqFhP81N?)-xb26H>8% zD=A7KXwLJH|Bej8@2AQj0xm2Go2MiNIySJC@s%|@DP@5ewvVf91J3edXaGFN-6e3N|pRZ`Wnrc;nZyv9^}1So}PziSE{~JScjepxJCq`aQKzRQ{hx zy&*sdudr!eT#lJqOVU&`GqKp}7mg#@hE+J3;7S}G6eTByswMugj}o_pb)(_rYyoW| zRAQRt9TRzx_^JppLBpSvJVI*yaXwgt~R3Zcm9N2%&8G-jB*ekrp79prWYiG zwtJm|-@!MQQ+4H}l4Ot1X~ADUNgM)91!lZRf33LOFP7$N(?ReYR0YfTy@-dh5 zBKcrxKT+KjzcOR4x!xCpI?mJ+s41G#iUFr^YpkflDO`dF?5M24Tsxu0RRtP0r-KjN(sHm>|yh_4*)tSJTI zSyy5&BiPUI;1&dPla2 zdb~~;SEp(ZQw|-U1J`r@CD+?cF^AH5md;oMd_mqBEx;j}E^4u|LzF!e3 ze>`@Vf^L=K=Ycn>#aVJzS&a~#7^4xhQ%7{BCO&4Tsr@t}W23l~!84{~gO#+0EwH=< z6mWs_(rY8{An2tAlq$~WG7kvZY$cM<`-Og9Ra{}`4y*Nlzh&Q}9+=NlDoLa|o4)Es zFrV*2YG@_?y6fCon4nwK7;f8~%Ou;KM($wlY7yD?`>e-*tg zyEQY1V`qJatjd|2quu&G-f~pB5WMV(xSe!rI%Or8S^(s2KJivHEMH(R#fEI>n?_^~ z)m3cYKlAwoFZYalEFWDA{ELk9b^Mo-3$ntCvr$(fH}2!|rT(59ZQbbHXIxWq=m2P8 z{HFX7uZCispdb?>wo+AYc}E8jn76q^`IDEzO0=8p~rTqARDvMeGQ2V`Rw(x9od zRdn@-4H6I^J#Zs#*VCkvwW;gG?{WOPuCSo$-i-sptuxdldKjWb9+9GaM9!owB#56* zJ@nC@W`kPKgQ+~I;Av>LJcLawE150bs`iomBc12&)JRUKq@ND^a&@MsHsJPBMUbvn zMc*+*{BT73oxiR=^6u~X@A4r2AiuxK7MVK=!7$<#u_ryDm#^&~J*)Lszs}K zOC0B%3o(1FfPn$P6-)BEjXzSsPC_8YGzhAyK9uE^+yz<8>zW(h5!_M0Aj(?j3yGP+{L2b%n zTh0Anui(bmXTZ^@iGUp^dwLwf z#&;1WW*z3n?bKg^7$HW@nbzk^k7#_h;eOjTX#jhM!SE^%LvCFmSO~4L*8|TUiTv(y zk&?XxFIZ|Hu4hCfcD8+2?-JTo8M}R&op9rUbn3MZG{)Qz)~>tW?*zraU7GUveS`DI z`gHser8tlz7RutFalv*hm6jG;NS{vv{yeo0*$b&rx_JiI;N4N!#Psmv#OD3@D6{sx z`ezVope_2`w|8Jtv?{08Ygood#k4N>`|1Ud9v1RBGJ2zNVNbrqEpAcA%UWa=w54(* z*&a-e@f7oF8qYQ(kJU$3m#D7eb{6Yy$Gxxf+;7gL>&NWPpH2ZEJlU+*V?*Ge zyWBdx%wgp0It;bITt4Le0t!zL)`8peqs^U#wjDN9#C|zY@mpcA53uro+zs6+WO8Jh ziEBma-deqGXzeNYba~hhk(>ybrZadP0 zK{yubg)CDFi$8mT0H_4x0R$XDrd3Vi#z=TGcZ4P6w6hV~HB=2^TgB?vYhG=>e6L)1 zrZPPo&e%y`J%hb^V6n%+69h8Z?_bO}mQB`Nnp%yge`TTw;3CaBjS8SCMSuMVI!K5J z1qR-?zrx3KNTn5cR`5u|xk;I*{d7W?Sd$+%Q*QWU1wn_sJCXy3lPnTrx?Ij8L4?)f zMbElB=8b~jMZ0OP7Svf?qAY!%NY-l&ZWy*qqj$Dd4!F%@g$U?OjcaZ3Fh)02nl{|- zdJPcimMx=RMvJKUM(5NDe1o z-?gGxHSL$<1Y@EhBs0CT1fZ`_H7Ye2P?_FloY;8s!kJ<8!EVuGk@`RyXFBd2Gcb(N z&^Gbv9E<)i(?t?|CfiPyga}n__c)X6gg7}1>z1wC-ARJY$-X}Df~L70D<$;)0X#S#!}ELvvNl}1lm~GB6gIO>E3yOaO~QqX`Kw1 zf1GcWjV+#b{XJV5Io|eA3vgiXF|Lkf_@}MOs~+d`cOS~7;hGVg_LU%m-HCC1TGf4@ z{B%)CSP^G}=;|7^o%Il9qYftA)<@gWRc%6@7w^M8_iqK(#nZ&Fhx>8;`+LT%JIL1l zr}L|b(Vf~W@>x2;U9lFco$K^6gghIENf!%@_ah{Cvpxd2^vo_9timFw{pZNlB1JT6 z`Hw;%cCAldEu*#FN<>4Y*}br5^T!M8bisP(MP=OcEhHVWaSE4=y|)cr z`UGl=`ocApo?^X&uWQ_=_oA*ZhU;x6IqH9(m3Jnt_4WgWRQv9mp{k_5%O7;scG>6c% z+*K!gHud6PS<4eG4@MZRdP`L$fAj@cz6 z?USgRK^3XPsS=*!nl0*d?8IW_j7ALL&g{Eg5UwOG{7xSy` zt+S{uRf|?@wI=O0T`$Jj)p=+X3^v=(=w^@iAR9sb54cSh&F^GgOqxQr>=m!eiVf@X z6b2Wg+Xv(2dHzUjE9fY^!w}J!S|U~fe|&0Eko9|qkiV4MF@TgvsYn4-w0S3^Z_hHL z$MshpA)OI=^T+$21Hn$j$?fpskX)37al2PAmDK^?FLDP{VzNoI)7!R@X1=_-G_0t7 zeH;Y>IrG@FNL2HZRC#vD94ZZ_9rqlhLIt?}o9>3hpQ*I%QPS4anRo9X&?54Sm=cKB4)RSD#cCTzkBEB=T*y5?^I) zBBh$JZLwLi^`uyVsaxlt^|TYkEW@N4=d zWHWEXydI!n^EpQKES!dEgx$68H*S87k*2^i=kp4a_RL!9cNN=CZQC{KOG|9Orz!+y zX6K!~WrCf7dPh6`xq0Kccm9*V_ty8%4k#YgZgJJJ2LwJ$c`xkXA0C%|XsA#kW`txp z8SM%o+ji-5+9)Z;>*ExAy1jn zj24FSV+N3cck;F>e!`Qc8rU){Cs!=%Q_{^<6jZwT6m3wSU8ZxTNxjAC6$}%Xm)%FF z)bNAfcGeH>BNdA#YUaTz6ZuArF>`WX zXM=?TGt{gu;wUTy%oI|_-$N~Mc5*%(L*VS=_WR9#PT;(?=Qm+1AtM{&emj2;`aI9= z%7mrl^s+D0W7DA59aYrE^5Bx=GM-61#?+Q}ERN*SA?J&G39cg>P!{5Y!QCp3aQcyA z?SJD`9}5zB#Lwkw-6k@x#;nDxp`-5`O)9T$4zEtl8XkSpc4El#!DBGL^a);uHrWhV zU^I&KeKfYmMV-%#0q&o3{qFod9m#j*v@Tle|7+Joe~w3pa(F38`SP=p5<;iHa*!?ziCol2M(r$$`9diRpfli zvOj)KJ#33C$5JJCSPJmJEh+kNA8|`&R+F6T{2Uk+_w@vfrJwnt?VUucvvm6sD|(cD zZMwQei1M(mn~n};ygxMC_C9oJuU$zV7E`kcz-+Vd)Qf{jdGE;qTnPwBl-rVK40b(jF+$uq*$Yic=y^cmXeZdKQ?|pw+UN=P~T5qzodndwa|E_U4T9 z#5pUT7M)mSuq*tPbv26_=gJP-Y_uS4555=t0YORqO9;KdsmH$X(0TaqihV~pteR0e z`zD93R?2Z|z(9i(Fm|7E{Dz!T zqSR@j6h?4oYUvC|k|TPBf1%J=DzRU7A2WNx@QnHeI&K_0b zb$izl96t^KV}2HM()giZV4c&*YL zXi%B(Q+Btx;|<%RLGE^4J5pXcr3e|DAMhfPxF#^)O&K~Gi3o~n={3b7|IC4|L4 z)fi1_`oa({MM8G*0Ac4P)3erb9P!*HvSX&OhU+IxHczaZjQ~~^YGOxEORoLDO98Rj z->v&y1~WqxYUyDZlgLV|ni-UoDv#O!INTIn0ATm6U()8|%uBL)gGN-yILczZ9iER{ z&W02uTJF>YZtCl5n1Y+T7@?1sFzh{|v`K4~fS(wf=CVS2=>48$^BMT<0-rVFQXAZy z1)dCVYt*ELm7{NrX_Ds&8=W^C5wJI-knQ)+4H!G#N9E~@K<~$nZxaP{3cw_zu@1)2 z*^Hn0Oum8+7t(Dq;?Ttp0D%vqs1n+`$f4Ic#1wbd;9*?yGKrC?zR_ngE-Nf{lHha` z5eC!34%J5rwTSmD*sCKw+!#^&BNPY3{!i}7feIuQi#tQfeh7&>Ny{}V#Rcp08WrY$ zW@%mGgr*CyPNUOIq}tNM#p~GTqDQk-#G8v**hryx69|#(c7{XBArPKt@ZJGm^hYKi z8HPGip^{{VdmV=nV&x4Vo8*{nmfCB`JD37W3)~U(5wOn7AoLvu8cW}HdFX9)g99=R zByP_sW*j8oxn!s2=1Yv4Vr*yRzs~W4Z>b!gKl%29kLa}BOVSYZhj!nRaiu?0p2`Eu zRSHXA66Mfm-@MN96J)BNcSQRo<11+|U>{D#j;DM_ms~ai8@tfUaETfVjd^(DU>}^) zl3y&ZlmYE;7#~j}PPLq(@p7@GrK{FmnYRK{>oT2Uo05t0sRKyf2wXgJ6p1|Vkzn&A zkLdZT+!kRl_~vHMqNmDeAOqM~JNN0096&tRkV4QTMmJdH za&+zeca#SU%sZxavt}_NdM&KDoVC5L=bZp{j)oD+Lba=vfbpmX-HHw@AK4`(&gw9D z_8IRcKTZ$(Q9WACDO#p`(ZvdL@s?hF1oKCb$R_0^G=B>72Ei3p*if6Lw9~h4nzvjM z>apV)+(~`A*r4dgZF7PJvP-8Q3Pa{jhsW&3JBo743fB;xBk92|1&M5@;I7?KZx6Vw z4@49BG850~+l*6o`MS@M!?hGI%PagQaePOnlS28H`e^5`?PA0C%L3DWGANj|NQTU_ zr@ZT&)L1{N@>**royX%ICxmJw12_*W-LYtM?@;*-&g-tCB*Zh=ORyaH9%h)-N3N;7 z6{;`0t1uxQ9pNB6nR9jbKrLuGQARS zAUusp7dSsu1Lw7G&)jhf$V54=du;z6``vKVieb3TT@e_ohaSiFp7x1;@`d6{+9Gj! z7M1uKY&U#{MTwcw>9RAe2%i>S%lueZlpG#q#P@q&M|!IEwV;0cW9s6xZ9jT;UJH8m zvbBx<=EFL-4@O{oR*F41NT>hSqub#Y7&YpXK!u-q>ZW~k^1!q}rxakzbSToeaDpwGJGhmN#;S;4^{%>MN@LwYivnB zhArDvrtn76pzo&R`{np6{X7+g5jqN2HzFm)clsEgv&t%&qzDa-F?*d@XY1^!gz7{0 z%`fAlZfU=Q!FS;vh@VHYTXZ5ZvPqA5F4;(cthQ|19%_Os%1-rX&Xuo z;FRY_D|R=1gQAUC^fblPMLmj%l^u=T&hlsHCS|LK1n9rlWV!5zM4*y5>WR=;T+n~A ztEh%Tq@Kcw+lu^Q(SQ^fWz7vsdMawU08U_icI9^IfyX_LIewUg>o_C1E7IB;HW-DA z*mxXI$s6K!kjlCFz}Cd5^TE8wwX^oROO|b?H-1hL!N!*U+$%;iHf1=Ldsag-j*Bcq zz!W^zbEs|A#gEv%ma%1|e+qq2xLlI!QM@ZYnb4otrMM{^w{{f{(aU1&hZNMolJhLK zRx#WlSwMgCI6m2N0tw=OTm<-Tc|5w_X%#`7`b`8_0MyP1-559~8Bo3rZbaniP#BRw z$pqRP%Egs}8my*;T1}ASIs&m_2l^u}V!5a2wY%t;JL}cB{L!W0ot62^ch43p$%f4z zqXDrREaQ}a6=zV_EVlX7Aj=$vU0xF(;}awAm zPkM+DwDegt>!&a*=Nold^CqFlk7h(zeL{93{Rg~g7Jw8&(W<^5c+jnmvsLElVo5%S z`9mDB=w{|5n)-U2PY%*bx3cH@=FMx+@kmH*yA~Zne=>k=mdEzmub=QMXOT$zGJ&|G zs%mYgoh*@F^x)HtFnw-~o67v|j!Gx)J2Jk3vBzoYWzaBZc?nxtDJG+>nYG`ro;5q| zleN)|<`sHjU`7Tny-xVyR(H>WmF2cJJJPwgOW%IPtWTsYk@3t*qj~yPP1Q5GaQK$zDPar#36b<0S5x7_jBvQV~23nLr-H>dM3V= zK;`!aX|vbD7WS=rpE$f4KxxeG=xxNhGXDtW3<;^cu82`CoY|gx}6g)-FgM3CLO*hn)SJF`rpcG(6aJR%b>O9{G@%h^t8Z^t-I1 zAdXB?;^{)>C`10TV?l%HFodo6XYZyXlu@v;m=v}*$nEVwR;%j6qUki2npoGsl6hR` z7x9Ksv&b!7H)STVTwa}umGR!N1yhs10@F%vY+n1toYpj*{r1W^eeE&i-d^Y_I!IGq z?#QzmzJyq=)x9HH@B^RCUl=vXrkpL{CSUKmV>zUN}+V?97 zi3hV`d`xXjypU;Mt#I27?kU{PTaF^U5m=k+9L0s33yWgxvcbH~w|AIKPophk zu^Q@UgihP9wuLcRei9~YcxgaSCmjV+te)#&T1Anrn#22o^9*6mnL#%L!h!XaJBn+^ zuxyDnTc5n~qqgRQ@YX{c9X*LGm&Hqz19=<{a%;im)mFB=QsH4~~}8?fC`j%;qi z@G+Wij85O}K>6&m`_SGO^d0+%_wTu(2ben1iu)8ZvRunH@89wptD6k~t>PXw_0F;L z2{!u#Tgf|;eN70<+bC6#qpI`OR zg++tn;j%;{D;JfKnRm8SQnznsKWT>%L4R;#Vpo=lKgk-QeWdO&$|X+EZUOl268A>$ zz^eK|J&+v5p`%;6sO!Fe&xw~{I~w-JYk{qiCF`Mah^sKk1aTM3B?y0Lq*9}v)Mx8? zaT8mwkLX{dxIDj}%@22R!-aXq9fv_G^{O})LDD!{F8H`Gc+>};#AIsi^8A9P6RFja z0}-6wScH?-b**fk>D8}*GU`+Cn(Kk&(3v8v>4ZuO)BYPx1cOg#WidrUWajyPH~o*g zFNCgv@GwZ5ODo`Si-dOG8TM@5dk3JDPP^C?5ntSdz{-CHq%kYxHkgnct|7hyU@iD- z;Kmveb-cR>LPpP$cq6y9IpC75u|oo`eYrd=|FNJ^It#MrVKSCr&`rv*Htc=c`sLk( zMSC{YFZITa64-y1(ado;{PAWeyp%+9T%lhTcWkM^qIS1f(8#G6(2A>;wRee6=GQxA zjJI=!=gj4^%m34>T@6B(NO@*YL)`i5Ttl1a&@?p(R5wB@LbgoeTl;l(Ey}%+`sXv-l^Dl__Yk|5 ztD2;Q$NCxTySq^5f;3RL_&gW3s^JEQ%GIj`{Vv4Y6;0&?lR>fW!m{p)j=R~NZC@O6 zTKGMh*2ot&WM{O6kDr}FhSaBt6N;1}2t0h`AX#gk{0BqD;eFVsp1W}+Lx0!xu0G*L zg*WGjlIg~FgIY&9oQ#l#MMdFUipZhm&;gMYkbv18IvfrAKQx1%Tno0BXRi$%SO~z; zEZTrC{BRl%*lae*F#KYR6w+mf2bOaCvm97ExAsXo!pV5FA=mr!jtc8d{CjIdB<0J$ z8cvEf1RZ@KA7B-0zD^0LrGE^xGJe>PRbcFk_Au@pDbPDdlDq%lAW+8Cd?9V_UT*F< zn*PSyJP$e=M$o-G6<>SJaw@&OoKioad#J3^wAx76pFCPT8J{%P8^69`&87}!Hk)#S z@v#Yfk9e8=UHfFgKe@orao(2|CI){LI@OHuo8w2suv{=h;h5Y_6t6%pG3q*k~V70 zcxzYkNq6bSM3;7CWhY|#*12vyMRjTFaObL?;M_w2oZ99IyT|$d)wmZatvqw(3yN4H z`MOHLgpz;p7q>_NML5uUvL(O8q4u4l7qu_hN3EfCGp1f-N+*%M{@FstUbKXvE3G#u zYsYBj+h@w%2}bZ~1!5Ix_T^yiRwtE{3ouF-L744^CO~rRsQM4qS=)5n80@wQ!1Olw zRg<^qTNpR2L6Z%q4Cx5(WYagZ-AThkT0Dr93ft%%mW<{Cmh|!*i{CPl3YUoq;`_!* zbg=baaVot<@zxEDes8tMN)1}qlt3GbU#*2-8Z|w#I+m$eif{Thxz?cEm^S$unQXs= zKzdw4L4iAg(*xvok^gH6fol>0HH4=0 z%%J%@X;jVT-IW1r7pD?>Z0pNd$ad7gkbkc!1PNyUSL7tIaYy-IPx+r7`rn4vBwu&w zxD?K?oNDQHcF?z3rgvR_5a_Ja-ed`OG?&n1WuZkI#=&x9`0(8m zsj4(u0vHI2#I*17XZVR`@ekGbpBME-3Z0}97&_mUcI0c%WU8B~zTP%ZDawYm!4Pu$ zuSuVx>IeLZ1-Jf!RP(>n&c6ihK>^PXc?I+CdJdpa!12z!j_#iYWb8&|Bh+B!Dw(fU zuK2O(Qjj8>uR#qEKc_XvZ8W5zZu~oG7A8c$DZm%5QRp5#ykl+ooktRU3SCekMPgWx zf!pj)^lztm!twh{bN^SybTa+$szU_ye<0d_eaHW%(tpT&|7~a<4>gg=IZxL$0Bo`8 zCH?v@vHoup_-}Lnr=R|>hdrL)=hegB2XW#Z@+yV@Wm!=_{FZe+W%HYG3M z{9L<37C_SZe{{nCla()t2_ygith%^M2-MyGwc_nCz-U}5X$L$Zt!9bIhcL^U(9J!W zhhne-&G#{G$p4j={oCOL4*jCkZy(vON3#o%u}?Q``@Q#lBZ$(fz`6cC=YP=Jj0hs4 zm?s~D1Pt2*L4pLS)c-}=TLx7YEZM@{XakKj?(XjH?%ue&ySux) zySuwJ&cWT?;ouGpJZA2liIErI%!}{WiHg0Ws&-~&*2;8c9G2v(xAF+1j3ESRcL$A(Q`++Li1<`^E zZfuiZ;f3~YF0UfC{R^ewJ*{lHr&3Ucu2|Bm*@+-RGTS~SsQzoo5wI`aR+DSpQcS*E zMd465(a1pCeq6Zyv{+Z0#Hn8CGD37Xy0}p^-9hLe-8HA#4%>!Dk1Uiun7`GwtUpNh zHMp~o5af@{54^V9vSlcc)Aalhvao8{>pS|~NmflYc=5nbRHoR3G~VB0cruhfCib+# zeonLYyi5(IH*oppD((?nfsLFHd;u|Ez{?I zb+trS&_de9J|fbJJGj{?pd6jmp2GpnHwwem^^uY$mQYiY;o(vrAG+=`8)MX}${i|cyIYUAU;#Hav#xVsmu(33O1x=QT2KXAYP(Nqy+Q$Eb~=vA+}t6Ul3 z&%fD+Bbtn_%*rXN=WYVPkQA!XrUW%2*R6!oVhe0;0sPb1dLrx3e+f@A&W9{OH@Km%OB-tmsIdTO9)E`WJPcv0J zp%pnzc_8<)CGGJ#S763LF($t`dx}81#|SFA&6X6oFBQ;b=_aOUeq&Im?XF^#;psRe zZjrc4lRiNCi@UkU5kfaR!Ps27LOmeR&t)m=6&)-R+?DzV$8mW@)UwId+1%&81Na=(4E zMzl^BBGa?qHK5k1`lL~-LvPid7mw|&+3@Gvlqx;Cb}<&vgoYJbsZsq5a-WM~139LTC;9FZKU3(H(1qbT5e6J<~KAB0YhX`T0AGr8&rh7APktovkH6N zL0km2OqmJS1dja~ndXgnl$Z^ZJ&U@b^OoiXs>j5())QMdl07Zc%N2YYO{e?fuW?rz)cRyW`IRy=(xa_&S3zRi@Llr+8ra* zAKT*!lci8L*soNYy(qyJq1es9e(KR+$%sZy?Bz1ngr9CkaShRXZ_5Hh5#*Cg@^ROiQilyVdU^SBM|;oZhTAcSW}$_ zRqZU3nGI^H+qgfV8vt0fhTjV96tf$SF?weAxZgZktzf7zS=2ksZ8R2|@?a}hm+(<^ zMiVt((sT;s@PVaq+Tf5UMK7=BXi{Pb!ihMQi%p3}-&@+ zEG|u3gZEDlHO1E+A0DXqS$>r@P@I7A+kKr;Q@!InQ_mLBf*I%Ghu4To=8MsprF87- z8#RTNJ4*_{jL+uan3l^9RIHu|{(_UCdEjuNyiQ7giu;p+F-32Q4|ZgzjBQC%K|XiEq9)w^Y^96oP8p1@>O|l z%@7wI_t5ML>_7?XSMhY98i(fH)p{o+=if{4{sHm&hY4Eh^jEI_LM)Fb6SsR(U`PHy zUFr1{t+!Zi#&%u#6@N~}!k6pDA>XjF+Nl4o#O0J&XLMuC%@&uA`?P>PDAi~Lnmv&P zl~w?RI2LT%RlduLXvR|AD~Ho&&ggVT5810etjk&TUmY0DrCLW@!c>)>xaD>}6kXk^ z3)Rx~V4{Y~g=N+2$gBaJ_`}xj`9wojglb=oBqR^HS)@gsf48|PIT|&Y_|S&Dj{}6A z;A3|DB*K<_V06f`v4ws@Mn$bn;#;SVD6_ZnHP$J+bX8iDxY{~dDh4f~t*20pIKj16 zc9CP~DJghfe39fLS>rqqzi1t(L$ljxdEfo1Lz1KZ=Hncyk!exa?$)#5qrSelMmK9L z+^_D_Sd3H?W8)^lk-&?DWFm_IslTG)$KRz%%deoYr`qbmda%MvrK#gasU5A@8MBJe zWZxBpSQ}j^{8!SrjAC2F_iXtLnL9jGjgR!q`C7SKuk@f4)EF_mIM|Tk)OEn@aszg= z!&xBo%{-^4Qo@SoTIpBTu#Wj5H4>Xs`~=R}$}9+?KGuVy(S>8$=raI2!e3Tyty^F3H9V zWx%XkT%%CmlHzH;1KIYI(0>rlJ|X>=)O0Yx50%FW1rF-$nSri~y2fiWNB= zW?p%kq4ko^4n)CL-Qd!BXM#p?-5tI$t85#J{r8A!LnLnu>q4DwY<_$3j`Zu229g^j zA(V#o(A#4MQ5=OFbu%y zs}m4VZP(kk9#?}c2X%!w4kz;6{2)@pbh0ZiY)up?a+cn?z~d%upShC0Il`z(T(YmD z%H@Yai!dYl1~;@gMJFlf%*JjEo zaQvw~tK6q;tkV+--Wi*4WHo z8ep(RO4J+bL?%0ICB}QPm_qTwIzXFh*3b?v;=rJvpOP&6XUITZk6MGeE`n1U?W~3x z@iDQn>Q%1;vgCtc3{xDyx-5x6dy3u``_fatF(m+FjW*lRi)N%r{p>!4Brzm&Rqj;K z%g7MSoZ}Y!KnNY8V4Xz*xeSpi_sl=bT+&P9CsqYQOUB8W=O^MoxmCzgTOx4qf;>nc{p{-J#e5EijyC1yr3evQgQB*_klRzNZ~q+EG)6#PuvM*g7X zJ|%{)JeCK*bvF045EooGBY!D2+ylqUnVy}Z;?L6=kCgDv*~6`1l<+xfM(H-kw>vg$ zDA_rzj0yk}d7>HlP#d3I!B7RQ5pV@o+sxnc3=uN(6f;AI?Y0$n+waD`;jea^5ge`y zpJ%MSAK*dVyg^ei4Ayxk@2hV;$SYv2w z9B;&f+St(-Q`lM;of>)2MhxX@IcX%c#*Hd?3%1WzV=Whj{!T6mZ!xY@Jfh+LZK$z? zoU;OJroIP>Ql^QA$@ov$FVP8y^q|YnnZ!WB)~d~2t%cB6dBte=wUXvK7|i(uYqLDE z{Pe_in}g}-Q{S>LJVl*39JC3~)tjhB=fQ|IFgzt#-f|?crHM-T8I8lqyhP-tuV$;^ zsIXLP_D&!+d)N37x}~&yW~t^u9e6&mL~_DqMO~df>X6n#VI)Oxi9-)_<}ed}b*OY% z=Yufi@)1?=_-T4g5GAoZ2BL5(xgMoipQtWpvHvb>o@ z(m%bV);n$*tk#7qcRGHMV8tfaX!!*LGyp-xc>C8Kt=G4jq4nl3(j@7E6YkpZRGARQ z1$qyuYEMh!YOSAA)b#Fx*yHn?P7^aDcRcj92)VX9s70!7C3d9%4r`N^Qn#O?ugK z6lK%ijb7PdZ|!{2mHM9rMOuJ*t36?k-h8Dxn`56Yk;B2|A$JZbBZ39RGx|+BRXX`K z5E4Vr1plmt-QL5#V>R-ZgPr&6P>SB;n_JEbA(#e|?YUs5fZ`%;MuAKM$O<|t-r*AG zafP(@FpvU7}n~p1e{@&A^AVvaYU<*kS&4t1&KW zWq%z332*OPkv9EBD7XJHS*)ljDcQ0XeGn?$cTxOqSm5uZ8?$6wJx-8#ZSot;AKZb= zvGPhiksr+BR0X}G(D7g4k)`kNJkpe!Lxe5O+G}Pf5Ea|C@b|Z;{c-2AU^U9dxE=|VG-Yx;+N z*(W*A=N9UGjXBm;MOet3PKH=?s#5-UDcu1A5j26QT*AYo(Yg+5a9u>%6LZB+7ile8 zHB3y)GE6%4qhj(KC2Kk54G5wWa651ubj|Bz3iFVhVdjh7=r9QLGqw3V{{>x~oTH}GK>Jv%JUyxhr(SHDIGc)Pq zHy=@*`dr<=?+|)H{6mECm7NW%+bkVa`8d#VAfgf@^>?JD@v5oW*^K^#b^Vhn%~K#m z5ES)Xb@M$kvku5ia%a=g!k6n>?rlwsNv!?z2yZNM)kjfzGl=lcIouM6JJbQPif{hj z%bP#nK@ODU0L}z}qtBDc(Ec4|j>>8O;UMmM7MT7omVTTa8309uZ(9~K+(+O_jt6P* za?fCfHKIc_>v&)bJc%)PL~=M)?(gdN`YAatbDhBX38G$ck!!^%9i} z_*r-ZKW4*s`NPOPW;AIi{iFrdK2*^5DCp2ldDR^^eF4c3moX*9l>2yRsJw7i#h<)- z+KgKMMq!3Nk#?PcY`F%Tx_~pYL`{HnHr#NUW|C<~Q>jDXmmZ|SP{GZpv?2&o)-kOD zs%ylxx;lvkoVDs)F%oR)=_#X!u5yr}(4MT$f)a{<;7a@3H%c1dC|`w8VX2|}u40wi zJ6RDDs)^ZtBS7ccH0p92!d8qa(kvA3fLyQ5UNOjHnDTn%D?l%AwLB`r) zpt6Btw-1Vn-bcKoWj*QUDT2_5&estpn3;F@FR(Uy29Z$|$<{{{pOI2$O8*_IO-S9dR!t4m7fJe-@zMW&iC#h6GR*D*=f#AO6fY<(r>2pghmh z$k-iA9;!%B#~ZgpjUVU1k0nf{>RHyESK4Nyx~rf1$ZWx!)`v&o8Y}VORQPsxe}8yj zJzC3oGybkWScN7$+99X~2fgaxI^0$UKNT1MUd=vjlaNowG8e>%Fz@0LUUx5{SLCqlF&!4hUIw0jG!MlYSD8mkCP@ zZ6onEVUBLM9c1L`#pM8mRn*>hLw{s7rt8X8|6@`B756HP&S2UHU=6*P+3o;GamAml zDQUq-%oX6co=?PSxV^U8_Ac?_^ruUZb98rBKrs2JDQPG)Pq1&u4n^7NH3;2B=JJj8 z{1mj_>P9Hz-CwTZSNK2k3;f~)2#iAJCYRzyl!=bbLF*4d0ZdnHwt zu*Y?g2F1c~ms3}E1xL7JTQQobgRsl6e5bW73u88>JFZ4Ung~9=2s2vW$YKl| zEXUBdQ?-d1NHgr8-KeUL#Z_2Z+L2AfG54H@HNU4SFY;#71bmZ6;Pvi?VcG40`sp-F3o>*$HxM3{HN-@e(pUBozAeN6e~^yudU-l+4HO`W)3MNgVI_EPRiN$aYhQ?tKJ zC@8>NgU2C(QiEUb;Y)NW_o0HuOJ#S>x6CM-(WIvdb-7IR)N`S|#N@;lf-ohZku8zb zH7n0$-AX9bmD?9o)|%EOgl`NxUwk%u%Z=k8hfOZdC11un)Dy>3xn?nHBjeLV6^$8-uueF8BEY%AOAAN#B2m?QBw$73G!Tv+p zw9*G%X8wxi+~2xW*1*eGHcnw6=-PIK>odxEnK$I@XgSZfgs@knRR6lrvA*jS-vQO; zPs*b1?!+^%KrM3Z#_uYvWG3YC$Pu952A(Tl5_n)A)f$Le!I&lCzuP14$du+`h@h)Iq868s0!+Fm6%hl?*~5 z6A#{+WSeMEzO1s0{jRgi2DyL52pu$=Ew+EXJL}-KR?*;!RXsM-=(EdC0@r;&J#g?V zJ)!rR?2D8FBLK1IVelS8u2PZRp3r<{JB#_5ErWn~kx?|BM&8;w*=iAd(DwuPm&Iw-<}0Z%sdNvO-!KQ8Z0A!H3iq$t$9#u)q}eS zUJkQ;exwTj?W1@CsQ!&pIA7M_p%IC1z>&UmA=0MAbwCQn8KZz>=@v+K_c*etn;fWf@X@ zh#?z?`uhp&hSySF&VKq_b(u(mY>bB&=D^>P5o4Q0-`hj+E%*(F@gJ zKl*~K?GbdD9KweyZ1&t2%27vU#-$tkR7M9!)RXYWJ(Pg~h19v^(U1Db>s^6=uKI%P zMu5MQ7vBsSo74UoTtXJgtHFXsvbj;)ms`!#sO#QuW$8``x;__)(s(x{xj z<)doHnQqS8C=)LeX!Q*zwKrE60g(qgt<7pIRSAuTvnJR*9#G?z>jc$BJNTtMfi=A{ z5X7T_r%`J1T_})Z(|X{O%G)o)Q0X#du6760TZZg~`P7J~u%ogQ_3u@(!S_{=GT=kc z;eqEuZ4JzXhqSBW{yE!PeHq{%wii!w_oHl?coBuH%{9HNyJv!6%LYN4BZ2)4@Zhn8 z>$Y8UWNybe`m>|sJp*(D;8bXk7Eqx^C0{Z%A6oYpJ^QFLH1@P=!D* z?mWQ5kIOsH1>St8AT8avg9OsQ*XTOHvf66+C`oKxgsCP~zAo~Z$-6_8@Bqk?#>?2* zZQO&3YAm7k+u%*2QDfdAu(ZBtvbev&Ubh%!rRAc~b-+pimFtchLW=}^Tp(=*5Bg5PKWjndo5rF%s;2FU!0;K3?PyVvUeH*?knP1gtHWtha89FM6!;QL$97#6Q+BkqR0e2oe&FJjb=O(%!n=p`xq8nA5%%anp&CY1#HT zlN%=$51c@hBozbqt#|QHEy|+s~O*vs^njnRPkHhj#ak+ZLw<=DxY_K`3s)HYV=ydh-RtDwwH$ ze%sc-%3+G5PfdB?w8LE_8eqTc+}=^O#Stx(8QfI24Ys~9|C8@+Aafn|G%qH)^_bA|LL%2lt2Ssg!I&zdvJbfR@HW6fNkw77V5(1bq$PyUek zGm;ZzSW~HJnF!czn^M5Z>V>Oc?L zCo6udjtK|fEzm@Rz>Y#M?_yIg?X|Fn$!tJvdU;TwQm2L5cIUvAIM#gsBEZqA3sjO_ z4AJ>$AEgIp)Z7k(rSUKPx@O>)?5qetc6JaYl?SeC6ONOG8h}XShd>RscM}MYvxdM> z2c2|BYoqJrLhAZH!FU6j7A3~g47*_9Hi(*mQb;Vq600`J$bj6;i zk<*u|lHvmD$nfZUcFGS(hAA)h zp;)m*2*n3e=;p4}%`N32;0}x1e#}I_Nn;#x-)gNXFEhTk_NpVGgZiL5qkEI_fX>+f zDh0?C+1fu@c3VfAm0w2qW`pdSE&Hm%>;w`X!wl<%7C7dI! z88=oPe$KEwbc$&?bjE5hc|kGkpRNb3lK7r0uBviifwc*yY+vY ziBrKtAzcMor^`%!tu{}S0j~*+j&0-Qn6YsNg?#I%sK}D{PVy(k$%=R7P)t@vdsf#WKjFv9 zDJfPentZipa{fk!fBY-bYeT&g`^|$E@HRI+TF&Zan)t)A#_I<&1jNyyirmV3sc2H2 ztkac;zw((l8_kVbJl)*=K=2){n)6XqlM7&NWLq?2N0kRuBl?~QJ^0?WEuyA^M>K9_r@>PrWeHLc= z>C^TIWFyb1^zMbSGf+}idESHhn3bnaXHqWVQ=LUOH}Z+}rx_=0;? z8=iNz)lM~jH6#R2u~X7LErx4*cI*_I>l+;;B*=XI2kHHh`qv+xC4rOE-W)4wDF=<@ zYEThUs{O1Sw4tDJm0H{Oj{RYGv6%VyEX1%3MpFf*RCK=Ivuq@URogqWthY?<-gFhK zSI$b52YmN!1DQMFyimw9fSgMshsWADpOZlfNiaz%3HT+qo%R@$m6Z6A@*B&h2gz5H|mEyPTh zsyuhdKU1gvs%$YGZXT;cKOJcvjUIcPXU=-ha!CYcL)5D!k1 zKF#^##zyOvRVuIp6@bpu)WPuwil*~eM5?}iT2QTX%D8i)Qu{qlVIezVlN=j?S2QWu zk7k7~=iJ6cQ#k2Q)SUn(q{0Pbt?g6rvq@-L2np zIH?qRu4MY!C-{D$KRu8yv!VY^+6T55?-!HxSeCxkhI$91s=wdydsMwgBFcc_w}FsC zhgM68El)MO)c5A4A#yD-=$j5f>IQQCF`kM`2?hg>ZZlBI%e&5@cOVIzYe!gQcp^ z8x3~1pn*ABl9HPtyYqQKhe9#s+fA;UK(qW#8A0Y)|65(KBgy2a1aS_g(O05B4V+YR zgyP7USx06Qy3U~f3K-kBVQgH6%&TKYOIHL}{Z+J3Q9Q31D8Tp_X5DqakVK5{GaYga zawW~?zmZ^S2CQpZts^80JUbUPIuMrv6VJrpQ_Cj18XAcGf{5^kEaluM)pwg{sJ(LD z=7XC(Hrs|;wfi-+a>gNZkuCdXmeF%L^@W>DH79wf%&zx_GBwl4uuAE6Q6tH}|3($q zuGE{i&oq;Cd)5D@W{V2IjD^F?RCjj2kL2sDeK8^c7da0uG_}3cFe;UZ$@yKhZrra? z`IFsi&r(0D#;YLWMtj*=M6+}6c3uumEmsMmn!;Jg!65G=8%RZ6l4ca_gqDl97M7IbHVUp~)qVm&6&pg(=6!Q_?NJ}RfmG&MTvpd-jytA(}uoW&x6l8Ios>OXSJ zz0thSUU`t>s&%s?oi9}na9YeGS2W6MVqSaI8>GQNGl744Co;D6Zbckcs?5Fzt2L;k zH+OiksbnDL$xtG_v%k*wrssI&1V{Cp zN|-hc!w{RG>;rFHlF-~xH;|lnjt+gn{|`|T72ilL*XMv=MWrRi5*?3S#isa**K9qVB(UMb15jpp4+7Jfh z+AYp)9+817CHRQ8joEY0sB>p6mE-aWWseU`Y<9k|p8b*8*L$Z|gVP=z3 z#?_UWNC+(JE4~QKSlYA2aT=7)*-rpwEgW7V$HO98_iK1ka{IW&%$Bl%Mr+fT9kGhE zG8HzogRZBBe-FJqWdC58P~<`aVKkQ>S^8tcT=q_%fD|~_YuK_RpF1i0sjhl|Z(ShZ zs3|Aqt!#l8v;1d3aW-f_blI@R*BQo7D)ehBiZ=cuJ7H*leiLQ$fX{>K0_J}7N{>rS zgo0ek>z+`t|S~ zvqlS%4Ao@ovRlwP%Gx`WrzLbsRzsppwZlqOWL_X^OP%`?KV_l#H+$7eEwJfT*0eVJ z60IfS&!m*`Tc}FA;vHLIVY{MvsT8Xyg=~hNTmwUn&yeq3zhZio!1z#;fuzQ!1btv7P7!|DwHtWckRb$4E!b)5!dKT zM1#|`f8BfV0Z@O3d}m3=)HJC?tNC6I5*270SZ}f(-S&p9wyg` zUD)Ooofm(nt!Tj5pBKhHWd_f#?H)s}^YjDm!W~w19&cFE90R{qkB-4H$KsEdxw*U} zjw~KI#=OLS-YX4Cva)Sr!a#B|Ow8o=T%^ zfjQNQjowbp(<^M?f|!7W0xt%ua6)rZS=&Qt%E8(D-6kp~3UGzbws?_rFdDTnn_Vi; zA}!|*JEUhZE#}B$$0b%?P4L^gV%1TxtooPwRw0L6{2~6Wya&MYK-ur)sQ;?|2Ef=c zZp(dI`o!6Ib3D#%#%d#hmb^eCw;L!!-Kfxf%NwB zcx2qgv+aWXbN|%q^_5j(dXh}icPVRHaeEnz)4qw@a}5x1_Bd*M#!tV4$+Hyyn15Pq zb54@m;~esU|Jj+x!3d9g-_!vC#{GtJ~&BE4l*>8+R z6TL}@l_x>HdVxu{<(&pA$o|HcnMg;FtS=)YBOEU>cZH=OtbiAm@;hG34KYz^n4J?m zaWgJ14EZ(EXdaPO%S&T#_Uk7jUW+NAP)E{;oG_f0v|m~^hbUtC;g^rfwm1_d^WHRM z?0kbd1tZhb$v2$};!qy*&MQZOWaV-nF%@Oe7xggeDsje@T7>~e#CEGcigRvL;?uQT zl+~ib^(Y&D2goVqB@*;K=ri8$-RO9GHf6j`i)5Da7&|wy&xSmjr2bN zzJKrHU(nzG(3ln3QJN*0auP3;&hVbbc5%pn>%u-}Mi3X3uw=so&=6j3$;J-kl;>f$ zTrS7w_J6d@f6FN}`gd^ouiYyN;?vN;&gFIlb3Vcrh8%wW8}$6wr>`Uk6`N&<3bl@0 ziX{D?{|TG^>kIW#_?JAsh|)vszCG+NQeh;^WZ(C-kZrfZ_h9Mt?kk$=nDo-J!v=%e ze2pH|x3Vn%=g<026ZqH0CW0P3{w6YuDKjY)I#QgpZx_RV5f1_v*i{-=KazZ!$!{=Lt1YeiWL zOH0EdA?!&1{Tu%G-))k?|1WkXaVzclOMma5(B^+!4FA^=N#*(w(1#M`crn}}V*jB$ z{3)oe*Rx6oxcyTsgjjdz->?r!%kn=3vtz|rZ_>zkl6A|vEI9CDo$+z+kSnDB`OuX0 z!zeA351vNM!?<+W%d#9xqn}ZZrm2LjN(UfF_mX91JqwD*b6{U^i06ja2u-}_TgArf zF<$FyE_WHWDifniy*+7+*I7Jb%;vK?^u@`HiIpKnKK%^_u;rS9%*y)ZPL8F<0qEz_ z+)BE|CG6S%_wxFOn*XDE5{dg{Jrlu3TbcgLGWmazVp!5Qt_u;#h=KpQ*8X2QNhAL5 zvX)8A-;@21c<#T?_5Z53|NI(%5!i}3ouVRk%>R#0`iPBrvFH5U8_Dsrl^z}Zh|y#FFc_K=R* z+d|)d&ClZi>HQi{ykS5?Y28B7gwuZ-@gEI**&koCPeLn#(dJSyw7xTDcIt{TQ0fO$ zN~>@qP3sz7*AfbqZ!!W+kj$Lza9%xtl`G^?OEnzI>VN}MGnglZ1X_~pVDbhmz8tn$ zS@)l>N7Bz~RI+TIz+Mf@Pt(X=kv!yyA3Y#PW`96rT84 z!|QyUYUU;6IkdsR^Sz?yXr#v;xV;5@ahtQM^Cnt^-7Y=3X6m6R4~}-v_p(MeBPPS= zn@wb8^H1o@=jOHQ3|W4ly<3}M-H-C>;)510YcS9wJPsg}G`&SFx_*sB!9AkU{k7b` zL%J4L>|LF>Lay(Ycvid*aN>36XsEY!{C|tMi~p45-zM86+I)T!skA2u^5JB(1EcZu z3CM9#7vfhGz|A4|g&s#OiUBXJv`s7;`N+Co*{pL;z?AX;cDtiik*f!H-H%#iESb{X zHt3NZ)h8?En%-AHo;Eg(OtX9ukx`-^8g-)@S7?1Ut%Z%<7|D>ah16_AR~lN_Cuf;F zLnOmMQt*W>mEe9E)+AP(p{0U*i6NH!Ol=os6qEpB{*nA@`k|N%rhMS7%#8$o&f9PZ zob^BgERPYXbpP7Gmj{N<*K7~1Kd=8Pxh$Q{50hzc_J+Iy*l+BxtNEv$g-)8d0*Livx&Du}AJHIFGqKSkYf5yMrboRN2!+c_W%eh_8eLp!%vtLVu z5J#E)M$t=3gR_`uqu|>H;aywxeL8htv#W5WQNMWu&kz6gbS9sYF>S@m_8aHL$;AeY zr4VEbT4HI2>HK_a`2C7?(a5VCRH7{;@X@2`s#;9O6K{zcy|={N{jn(-oQuJ;qBzF& zU)0$)I$)B0E`-j#z_wMy2~@Y3dGA7L-52WWFoz;e?(h*ox`k{VNjHqm=~>kW_z0$^ zZ3fl3+c*?l$rM|s)2}@dwdAF#62xl zO1CD}!t(P1+SLiClF(S@ujdw4o9NtJVTIMnf>B!*9v*=6CV{BrFwn0& z-6_tzFLE+CA*kYU=OzkOdbqMInk=wZ@MBGM8n%(!vkQq1?EdLXp*VETuAFzfuW=?q z@knZ-OlK(hWuwQnZE0VNbjEX?3;9#ZR2Gmb4US@FVI!!N_*9#p4+X!CLp9&rVb>($ zywX_kV1_r9!y{w8hUv~|>$J32EHsc$e{%b`ujhC)e>T&(*L3jfAO%V&kE1zgN3qMo zsi7Gq8Z5NTBr;jK>B-;>i!o<-v?Tb5XG{!SJ=7XqC2bxDYq0Y1h;%0*() zthQ~|-f-1FKhu)ZJAygQv8A+qqg$awuKHr{bC3)XI)T>AYTD!tM}RE^kG_Du0 ze*;hauCkuArJ#j{g?8*Cr`1keyLonsYRBxuj;*>`qS6Zp)fBj}w-P9uSgpP$3wMQZ zrq9QJE&>J!ZtM`aKG}<~qRZV^x}2$&(2hgsx!a9W=H3NtO3|@Twp4>BhE$4`2)o$h zwVgeU9#hk%#1vch4>;r@Qk53aMu^p4M=Fo}&vw~=t8`3Lr2ZNcU<`t<89t92z%Z4W z5LviT%s~nTt+|G>h=WC#E{~hc6$=u(_Y$ABA)c;9F0}PH_IWQ6A6$z<8&4r5@#h>y zOo&%)K&dY7-4YK5 zHw??B`U((%Nq{DaiqWolEnK2{_VDKG3?3Hyfj_Fpu-;b6ugcVI4gZeK@? z#X%|AfS!0B>jmb5YKF1h)lc1@ATu(riEFOAwUZD~*leX)Mki`Im1;7@TAO>hEXQx} z5kywsTevl;v(OP06`Ec+hQj~E{DDvEfz9^L-10m(1xaa#TeQ55w_?U8SPf#C{`gi< zl06X03m5%iv{*Qr%OhhfFx!aK2vF<&d?l3_9iD5LoK-qwP=?ek&(6M^8mob`{Mc{?wzaHm%; zin=2SRyg6ZncJIW8~aGuZaout<4xO*&L`I`4LH2f?Sz)$&P})(NnUGDeNq=AGfX40 z$GK|Wf+YS8u8a1NwYZd&kZ2m~m0m*}#$!_= zGBU{3JEpn{3p`+qUr3K9vNp|c@$s}+D`>Ld>|BYbE6#v;zdXB@`dSMVMz6h%w(4J@ z-mhsyl&&;4!sNLbR~6o3c4o_>co<9yYR$Q~dtac(I_HjK^)9}}2JPlLb%lr1FQvkH zkO*mY6*1PzAT_+HKboC+2WPglL{-iCM?v}CV15NNbSvy;X@`7r3PoemB^r0MbAN_@gG!zv2J9sgVSqNvBqjE#&A@Qe&)A7{^WYU<1F*|kr4<$NV{d@V@~=?yPd5={dMuMQABQOw$m+58%BwZk#c#et~bX6Ea$(ZK=daj_S}Uug>K z^ErB5+OGoOMx*4!a4xnO;IV9({noy{3yOhCzB$Q6{5Apzr)R8v zcMm}`H7U~jXUJCKVGCRB@Y52C;yMRXdOC)?69Fdk@+I!|H#r8YT@&|^Bp>t@870Llibm)G7==^_c- z7V!H!Q?O>fAVGGEE7DAf{1H;}^*)u0zVKhQ`iF^65JMN>jyGR*6mbhjlZ75pps~nP z56YShrL)#&9B02ONaHpfI$v#v2*z5=G~5*Fi>*^Qz$&*YWS#zcEbjtxEhM(BYzjsN zT^RlzUs3Po{_Y+MwSf=vdXB|ppzh2?lStJf6YOcHu*%nb<<3m?Ho0jm{`>I-@1I+P zzc|!j_k7MZyKknClGi+&e>8sG8g>hQER@^N$dWy zVok2wEfE!WTVyWw5Fs);TCZ1I1nNEo8hsf2f7rXn;7qhAZ}i=<-LY-kwr$%sJGRY^ zZFKCUW81dv_c`ax+?o5%yi>R8-Y;`&s#2B8$E32Kz4zMx-&*X2o;@=7Q_V6Jb(${h z$_?V)iwV63{3@(015_qYhD&hN8d{OMC;pZx+nbTEG7ILZ`z?Asc*&k;=;r`9_ric%|$~qa^q}jS9{GftZ znY}-%v9QZVrFDI8+sj7jmcRGzVNG}IhLX7J_w|z)rc(sB4PG&#Lk92r5vU0&>>T5t zX3ecaq^lvJ{Di&rtA)^UHh$}iGm4rz6~;M&DnPcwPGTyssN=iX8&|FG1RFzL&6{%) z=rBhz$o=DAhv(!bvYb7~-Ke%nB$I|tC3M4m>?cJopBKXO<6RSlz{PLFV(xNX`opV7 z-C{%KFpjR{!eAJ`%Qb((6hmW5G-gQ)t9~+8YA6LM$0jCX_4B;dN~9}XZjMiRm{#DO z*bdy@dj0}e^a~m^M8){U+0blHLi2%1^O9l7v3_j(4!4gBb!N-V^K_91NjyHL8bzFK zo^bk;8K+YU)BPoOS9dmu=?WnuW3x*JvP?r=wF7L>qYT&8b{OGtNtMz^8JfYa1A~%< z+e>p)5GgD>jb|`_4(H=UfRpAJqG95}fLp$tNzPJoHZ8A#j|kR0?;<6W`vTm7(ADqZ zaUy3lA%ese@BT}6@^upy6{Whm=+{m8M4;>w>+Xpk=Yhx`?of4~b$u)OBUL@0da|$Y z$Y-36%Yaq??dCxXn`b^KM{vc6Dd9c+R9@VsDFA6eFJRx#U@(l~aGuq*X)ha%m%~Q& z@m>+7qL@7omVHo;xZt*vfsY)eb8?zdFzLKKH(NQEdj0F z2sn4#Lfstmf6Z)2ih0n1@yeUA-xZgx5_UjWXUt6dM!CCgb2F4TK)}~bvdP9_CDmgb z8h%i5l?7_BOHy>DoGw7VvlX-!$jLfgt5=xa2cjrM%lvU{{-RG>kac+03)V2t_h3zC z-CoA8J7ph}UdbP|_ujo`N%vlj{%MSzAQC6W0?qeP*a ze0a=f13Ul0z82b96^$UI%SzyoS4XFS2P%0+Pdbe7xNmp$FF&CI@1cnPk0XlCZ1{?V$nVNfQOM zUk)tOj&g{1Sz*p~C@26HHd}-PN?}FIB&zL*S_s&8wS5^Oa47c}P6Wz7~lPiA^zQ$2JD_{y_JK_HjT;NWY`sz+^VK zeRIu5!~I#8-s^1GC$<>nj<_6?Dw~uC6~EVnBJmhe<-9&bSg57a^MM_TRy=cBfppzm`m}wTwVkowvh2tEuqG8KqZS8e$O4-LLWD86 zhk;0^cds!C7W#?}T!-Vgl6E($#~)a2lj0F+#Hnd%{7}|aT;h%j>h%FQHb`gskL8XW zvO64dSqU_yz+5eM!vPJ$_|mqW%OXt2Wtp89BJ*Pki67VaD+O@aJpp$mGnxeFPR*%b zDw8#Leg2I4M+EOI581Fdt0CTltjh96{X1163=gkT|Jn-x!g{mZ;@XeYo(S@|$&Px( zEPeeE_C;K)Q57Yy6-+B4_~K181-fM7pT~(h34SqIFI4PmibrO9WEqiXhd^_8x2_qv zxe>0Jr{d#d7PV%h55+5Ltuha*X$3Ou1WrA#NYk_YS{p)Z0UcmT)WWb?w!*yMhj;{n zqKX8K0|g?yTa8i*YIQJ%{7s05MX}^)6SX!J0_7covEMbe-rE@*r@EXNBrR0?ZVyZ1 zQ+Hi@rKGOO!GVDKp_J?6asW z*NS3z*ouz}j_iFFvD9&E-RzYc7E*>OqM{TC9287XBh@eq%FfeLea4?!N9=;8i!;S^ zHny+F*34}dk=L4HzYjB}Atm=99bM2wIGE9WKJb8NT|H6U%tnPQY`T+p_*Ie*`^JZ| z%bTfyO#b@1N5H1F7zLodK`ni&&FH|3sU2FwUm~nqg@CA00XrK`Sur>DSUb4h=UVcu ziPzoK4>Nww!WmaPh^o{SMO_dgo&hq;@} zCS--vz;6|`f*3SLlr07(s20;5GF0FDD>(>1Cu0$cBONtO(#rk-N?YlMn4KrbzwdM? zQ5GurJfXu#GV#CbbPxa5m8)Gm-e%o9x|yAU!*0YGxJg|y60jtM0RvW)ep%jJDo2m> zyqRow;Br|Ff_@R)TFddG%2=D>b4N9k!xIBc8{Zn)K%DCG97QHgajn+7#soEKy|CN% zgMXg?Zt>^u+YkKX+A9{kz@RQ2!`tjw6x0sl;vVsHvxPh ziUFE!3`H9HD!{rNlOx2_gC1PoU@dl+dhk*7m*O3sP`05P2Dcvvac4ktYR6-;jWMv& zIV+@7cZA&-vSgaW+r+*1%O8_!ZgIy0&;33Z#Lg3530g;DYXws}RyD&f*BTm#5M}NM z>1^!fz3n6J;MAlQP-<8{QHg3ptb(-hEpK5sE8S;|*35IkZu2!%YN_FNF@R?VV`$g+ zV3OIdC5>&L6-;5&3!?-w;8jx~IZA>cqDk~(@5!+hwVK<*{R*Uz2q7}~rxve+I6YA^ zuly>LmNe9=*_S((LFej4w~K6b)QJ;$0s<`488k?o&pfUrAJDz?7 zNpyvN^vmCL%}Py_N$N;c4yX6iul-wr+$-&pjQj8X_&<9N7V+@c`+3{E>z7KSx}gwL`0-ls7s?tHl1G`0=YecZX6 zMA_0Gr;Hh1J?$V3E#_V~A2e%mZWS5%_g_ENQ_>!*-$$Y=mXc{-e}dD?E&6vF934uo z(k|E)^^;K_!Dtpsz->fYQbn(q_%sxu#G-lwOWolKOa6=-wKB)hW7U(30k40DRe;pc z6XUR?Sh{#l?K;vQl?v$pOsRj5{~?7Y4thmA%v1Nk>WjBlebBo!ch;wCGGe8jEca&0 zN6$lod5{auLo?M|KED3V3R!n;e6*gg;Dp1mqQMG<*y!<|7?Gv!l%ggv82535dDBrZ zQN~;Ha!K##ElGb*JU{1K_i%n!n5i!R6GY6pN3>>vCA17uZ*bo!J>J?;g(UPhZ=k6K+IHg~xV7DEl=laBIm< zwXBc%EgwthG+GEo*}IU>i?M*#!E#s*Hv1@=IT`_no4VGTvYeAAYH`VN{-9$LbNMTv zUQf1tNF%ypQu`^iUMO47uZrAhxk#Z?F~ByNs(+ZMP~(0%^PG6jnr0PsaM?1v9(X*9 zT$#1nwO%-o*qEXbqEXQlCBw*GAIPCCO#E9GR!9n+9u8iyb|)b_;# zqeypWAM}*xPTHM62srC^r{?h~55QTPjlSfIT&sEK5Qo2aXAnDb;qhxA(;}?vn|!F# zNtalQV)1L}AzP#E_=-`_d8}5RS!{Bf>G7cmDnlYMDwdb+@$o1WcX8dtc@z8A6gzAy zd*3Y+fohWyLZ*L?}?b+_mvQE6ghgRx%8Mydw8l&epI&X{*yRt0ir_jtptOC(-D?_@xnfE^#< z=rLsZGs3(~k7C3(yK}Z`NQ;kb%AQ{*@VVbFWgD&x$C_;?XU#iWiV#s~lqaG9iAL(x zkFQUnDEjk=;ZsA@n6?A$+F`Z*{wy~y9tPT|c^&4RcGjKOM{57}h17Yl;P5S{NX6_u z`{kXEQmmTb{Phxv848EYJWgEpbA<;?k+=mE7CGTS?JFhgv15TlIpcHETOc7#JV%J( zFJ6 z*IA1*EAn5;#bPl~IvdMq1S@)rHEqqlh?%@3`(Y)Vly?aI^z1kMwU ztTubSx=BYrcG$k|STx+74xs7=;5nMQ_R30J&s9YCMgqM3#LIY(sDaM&Z6{zk_)6COPme}Zs3J~m=1o?ezv>^|{&fu8dhN8sTn!=9(tCSe+aU&^N}Z}(pz_{!{1#@?R?jR&@JrUb1!>OMf0e77+N zd+fj&JU{3RCdZ&wt5b6 z0vQA@Q|XHj4ew%Y+{uO|rzFT^JFWl(<}h3L9!T@co=J6yGBD=6bD8RcWi$g-Gxh`G zPGV+{6Xz=5=F5M7Gjb-8?+2nwSGd}fPH)zfHzHN!C+62C?Qd~7 zU#&=$eo785RGXoliqzVUHk?^WC^*MersLN&WB#AqX_H}qVP_8Z_b{r{(hu2oS#c{t zp5S7i=R6@yn4^U8oBrnml@}{v!_$@EgGObt+;V`LRUOlRS8&&>1^{=8qO=0s%Q}Qp zG@*b<3_j^!7D#{%^(S$veb8-H(B#*%JbHpT#w1s?n!SXr`AmFy2MW4in=Io+kPgDt(JP!>};^OWX1)DU$p7z+bIw8DCw;Gr&6GIj&k35 z)S3CAs9M#MK}}vG(%9Xl*z1uZgWV?f`+GYFK3+M-mSBB{l4H@W^Gi??RbgO?_ptMU zhEYevl=;;Nll|;DqOYvs<$o&f0+kQMJ=n%&f4<#+I3wgwSATx*YGUO!!#AL%Idj?O zE*A@NICSgo1w=5TL^H$M8#bZVcITeCVCzAh#%>Q2a1)ED4DzAO?EJ90*PPil*Qpx! z?a(&Keqvnpcw5x-BlWI2qfe#T4YD-N{gjoGiVvUgC+5o=J?ZfS=p4#XwG`Z9_OsQx zD&t=C;jNyvXZKSGbUT@N-PqcvE}57V67_8@+j)iCX6=pfKW+UrVkV~`X}xIx*Dzv; zbG5;|@?7%ei{siQ$EPey00nXFMGXZ-uqsUdpF2GBL@%fVhpwmYT5pjmY$K~JkNr2y zlB}y;nIesxRGiH?=3v4>Y2^YTVDxu;6hDcEag!qQ(HZoUBmKoRCk6LDmY>L9i;)IL z+3tv1&%ko3Y4bvbz&YmVR-J5I8H78 zck}8YQaT8en#BK;#~V47Xc(4S%>|T@h#rxTanOgd^?XrUzsF?jsY^Dr#?9q$zD15w zUw5bcRSx#gP6JjryH0z+$3gW>qjvIv01leA|j}^}r zP9oAj`~yqv?^=>#IIOE}79T~{9@K69$68%MKeJJeaM0?9TR86~6*Gd+Dy=q#)l^^s zo*lX8e3pi#UpMyp+g{txr~Lw;h`AP_bXmnr;~Ylei4UY`q094vY-*#vI>i@vb9p7&{=&zo7En8^O;0h^wS zmJ-69TMq~}?KXW%vA0;fzFrb4^M}GtK(mrD?gsfel}*4Ce;6kaFPON6no3;0?BpMi z2%mN-l1p#qq`+{Ba@+at_-aui z>NgesR7XlRMpwL}N3r^iH?~_cEzo$RSrBS)t=kiigmkO6{}?Jwg^naOC0zwP;B8Qn zD3CB(EYaIHT>9MK6vONP702uS*23k2`^fZMKoA_I*~Nj~`&mxo{`S-Yh}K(rzc57< z#qIjA^+2BPmeKC?NH%0hXt7Tckv$rXJykfEq{D(Ar;o~{N3Iu%sOvEW;fP4#leQ*m zmc@@X&y&U0fBViCl0>)X;K^HIQAR#yXh>vph{?^aW&=4_Aszp*evKEez5gaXi?l#| z=S0^Tuy(tcZ~CF=;P+<>v9a=z^IT;$<`*>`i?MkZJ=Lzk>#m|@FyQ6;c;t8I$){NTLCq15xgajQbN-x~%{}ft zhs^;O3a!F!o3)>ony5>QPk{gE(yyPN5lfZB)A2a!f=qtHYo;9+k>;gV*KGFV8NpP; zdZJYwo&lgNu#pZ@3C7|@E6qQ$uRh$$g!;tz;(lv?UT5_bP@|>g5&8Tq_7hmTqmq~L z^(c;LJyyTk`tEY5w{>qs1PA2T#O-B!Z$%F6Y6xdsHEgsPc&~o?>u&skfEmky#AQa& zl4Cl8^3jvK`%CH>+ylQ3tK{(uq3OOl6+3J3E83#N-0+a|ED@Qmd|96js-4kl16VI9 z%3yQirZ>+_F}gk3^>%`71_EX^A<9T5m8Ys**nPIKepm%8X()n>gDqzMP@EEs=ErrO${5XDqe9cFo zj2r+E^m(K(C$IjJ;uKZp;byd{%hUX7>AGmR#vKbyPLb8hEmkFPdmu;LxH`W11AIJ!B>fdQ4oq27(+VB^_2wqli zLJUrSi6et6;If3{CBiZt+N?Zcg8Y*rdM2)Q$Jz*6+Qw>MXNE@&#r>;4Qh(+>lm2SV zxGRyBtNG&^A&HbGnvLjaf~Ne=^n2IyQlBKpf5P=r91-rmU?`yFwH2dw4RgEm3AsP# zHRD@;4%SL8rOZ0E6n<D(8yj={|OxYOM(p-^7xNMY{p5n6+^fE57ug};H-np0diC&5>}L= zF%zrT#kq(E6nQJ1s6LlMgc_;=!bqf)^@7tJ6rz;5NLIkZ|Fo)&263p48arME)&j;0 zVFfK9QBRm_cUq2>dey4`c3NL?i|?1wWD|cKAn5hrII8!<)w)8^04!S6GGzgQBf<&q z%RyS%(vwZF2qM|E^v<_WP9SID`F(Z{+m3P+Zq6Eb*3_;z{h&TRo&hh|D!3rwH~$#3 z6j#W9Y4%S4Fl1au$w02@%7h4cqOw-^d1FEzmNhZQ$EJYBoSPwq3Ow^or(t2*LfbCo)@2Yv~rZ7oJJvk-THMO;y;P=RoPl0&bO+gl? zbqf=XlfsInx^FD+vk?}BZ4v&>f@u%D;%>jw{dq=7ckfpRX)<`HOV?XpK75p(R{s;` z)iC$>jV?ghbJ%+$dlIG)&1QIJ5wnQLNUXg|XGVa={PK7>LpzrOvsft(J(8>98AD{N z)@aW0mz9*~i(!SjEjlO*E)MT$gDB_n+=O6V^BOC@*le%zkqiTEoN2QkF0?US0}4tN zT1-yogjIYtDTk+OylpC^RTYG_&)>c;KSy)`sI8i6b@fMTu5VkLxKqoQ!oVD{>I|}D8I|{>3^(+p1OQHtN*aJ=rheJhT5854 z&K6ur0EiL{1t)7g5;d}sXWw;Rn5(tzNu60QD>dt{zZuc^1M+@KI;=C3lO6{A>`ev) zhp%OPvM^aFHtWGFcia~(_hQ{=>F)XuyV8|fEpTV0GD@YBoHiCccY_fIwB+L`?@uzj zDGPvL(tXcrzpM4V{Mhlu3d*JjS4s(=f1L>F+)!EEq! zr>IUgaZgr>3Rv0~*hBX!N2EdOUQ$HR=XFN zc7wIU{a&yJJuIv}MOn)HNt|H` zOyUv~{7ffWHL%zBdC*35ubE+gKd}D#v#~D*oMwV@GTtgOdcS{1ZG0~y$63Ili_cJj zZWh~8(9u)jpFs^s7s{JvGR-hfK;0&-Gv2$Lhp0fVu54D5_mL*CVGo>Tc8W&scQ87y zoud3@W*uW=A(LTU_QA+(hQ0~jN_^p!hq4fB1&a`qFZg86IvF=B=l36R%n)XloZG5y zoW3UpGeA=%r7uYrQ91CvfWF$@Q{x=>UrHJr3)+>4(JBdWSZ~x2@eN|RUEiAulzDtA zfa7kwrQdLXZuSqaLJ}YC=EmR&m$}0#_^4249x75!@dQvATEy~YTuv+L7KFdSCWzQ& zU*V32WOmxXPDyNrV*z>k0Ma=znUvf*7!p%@fxt)LGj8wfudi?R=k9ouuIB@cb7@28 z&fy4NHwgu6kDmplH^=$9o$HEqSRXZ&H>cMOaqX>R&97(rypx$S7sGWP7Ug7UPN@rd zon=28Z7mY9d2AwV%|uO{z?M=zJgRFVp4vla)`ekk(UIFaFQZK(H* z3jOhxoUOWba^5ue(WlDz+oNo~rbeZD{(a)uyjpqnBq3TMBC!}X&N`|@?oJa~cd0rca62NQi6DHM0L{qo zr}`LyK~$;C=oGXEsdN4VEy5gu{BHCbB^VbMBNWc$cwdZ@A}eLMOMLhys?}twj=*5L z6k(o@A*vIWzRo#D#HArNj80rR={+AXNlbgr3x$a>hoQZaMg6BlB_(fzvOR<|;s*h5 zc{T$>A+8*v4lkGjL`Us~1I9rD43CHCJWT&N`1|Gj-xd6^HC>!&VU%qmWuI%$7k>yZ zF2RIwh|5aA%(?L!{09J-{qi`9`$UI24=a#kHG$R$hz$Pb^FGf^N_@u2-U@orcDc*|edg*{`0f3w zBlXNRa{E=dd7hh*$#y+fu3razM5=cXWu5z!Ua9w;2Pwh@$n#2;H)I)%m8B(N1a*S% zRv2N#D}Q5U#C~65S{CEb3`N>`NRV*hv>tJ0O6|@r;4T38HXF5{0YG2z8l(uRAjTgQwvMKStW|ERo!2Y*wS{Xps2Tpvu1Q^8GP6#JoDwE_fCXPg zU3kP_wgv!z&KKKMGqQgHz{mAk??e&+0IUbuyYT%R0N(HZ4*>jE?(_Ga{67HjulLvg z0f7Gl0RL{-^3N6U-*DXj{{SF>R>mjn+W+4^1pf60Qt6)PoRnyBDJtR24UQf4XVHJp zI0K%^w$V?AVxrIF)z!(rs@8r3>padpPq81x{rmg#-+#P3A|NoF1DBBcQ^BQ}1@$#= z3|O3+g5@OoUgr|Z97xid8&*Ju3x#IIzi`&H(N+hnVCZca1n;7^4V&t4-bLRm++y5> zm!SznyI-u9vl~0LO2*yC3z1aQszkLM3nLRth=*@D$EV%k3`ANP^lG9(Uu1-Gzldy8 zy!Sq}*$ZW(=Aw6u&j&iSXq{{$I26wZMy_gH%(26#_dZGOV_~0xBZYHI%lBT+{p?FS zu4cpKs$X5>5hW!fVK^wd1er{8(Lm$_8=JQiS>wGfyd?jJ$IQuJ9WUNh4o*gAj>CCs z7hA?=^c>ntsM3E&E%D4ock4A`q8TdcIqWHGF>SKD!Adlky`@eyS@EB$$*U%?dEl(_ z9ins17W+lRU<-`pTxKGBBYMPY_+QXAg(aYol@Gq( z&t17V*D@76uGvQ&mENld2LoguMkpw_OxY-+9-ApqZ0mnDo{%@A7RgaoLvEbc7nGMG z5P#={vk4B*6hZF$!?vLgbhFOJ>THgcVtq{TKnZCD-;&2p6%;+q6QvWIyz|F|tF{&+ zv{^2o>U{{6s$}w?OCggm{-!prDTLN<<8^e88Z|h24xw@7L1fv*vi8HzIXwOeggi44 zW7n|XOzfo~iv7g{P`Al~y}2uCKayrpa-H>F;Qur|uB*^HIHql=1~qvdVA3QO^D-=7 zJeY53#b!M6R%#$@$+9SEcakD*`~wGD$jNEB&O5WseG9;b;VT8+ofZDdd6XhKI9B`x z2haZv4zg27O@`0`2Fe1e4R4FzH{qpv9H6GE@cS$Fh4+@4A-&M_k|3iDr$QF$eZ$h` z)v%u*E=8Y9(Y_yKge|nWuFe(v!Qzw?28K`M8kFF4W_uaf9QQmLo)AEH4Yur6GG8a& zFPr9~MRiz-+1PPe(zqONY5YVuB!Q==@y^EU3>@m5$ngSpGv#Yr-D^C;ycAUUTxF=? zYwt|CKi8aZdMYsPb)N+-?isEV<*)ISuS}kk9vOQefU!8mvQxx)q-Htx-HcefA<*4u z1WAz8>CT6I30&J7uM^9CHBQsa+d*oW`$i=$PxiX0yqW^p5U8E|sZ{;+rmvJ?53qx; zb9*RhJ<7=c&JOw@Wu~wZJe{kK!|v)h7)~{qpZ|b&(&^tFSs5v5i(}{gmkWs{PzU!V zGs=`QY*2c+doe+zQ9ko}M)>WlqH^U##yqtRpr2al=pV5HIljBJf#-pFX1RN-(ucJZNp zkX6*na7wIkLwTeQY_a)h>Ii`Cb;^F%bYjZrp!E-ZkM^d^=Aalhqb%iDYLM$J=G>f~ z4hp?eEm6S@zR*}OnEJVaGiW4zz=D#pn3MV28zG}gokfQ<7g@W^g0%y@xYd|*YS(8-F@3l}nPU*J)oHTp}KnII% ziW*J)J{t_pLj3-UZ7DVy?>(j|xtgM*lA)RWJ309Hj~pEBHL)w`J`3{~;S^CHktc7V8C$}UTEq-n=hNnEy7+*0=qXDh zNX3kQkTLQi_Zu*TS09Hr`X~a*FL!KeAfQWv#-vDENvVE?%8^Xyg5I$EcSdKnaN5zw z4IT7w{PI%x{E`k)9|P%be*kt$P2no193O}{^~Z21#$zd=PXuxPTTmn>Q}NDX^4=n` zN>uD-Kpns8pIOdbW)7=PP9eoe61<%EmZhOq8kxltYH(@dd4lZ6B+quBV$IvCa(T?gv>VBK7? zdtENsMbtO4?hV!#iA&&v#|9biCtHhHmAT*CcQ~67Wi+U&&I32;)NX1F}|x4`?l=K4zzOw?S8Ma14{RS+AMPF}ex=K(kYtg%kLWSs+I*DaCm&r;{OIQ&uzoV)Si z=hoq3taW3;nmeM{nAb7KN^?||oFFsG4!iOyn89lymr6RNl4D64l26b%I{U>(4IEQD z2bu=cY#LPZW#g8I{|`K&$11SgyXKwt!Xgbe+)?Cf0!>KPkf-hJW^GM0L z@+p~L-4~wv#JRnr;%U!0O5MLDL!|7q`Te>5Z^%w}oRyrlZ>*!TKr_AguKP?*zK#5* zDs=WkG2<;ygk64h%>eJt@DUml3mhPGt&{#VN|pgU1GEkum6Ugyyn+%z4HQR}KgTVW z5Wm@9o`J16W6-kvGE#?}YKtmzYyB1|=Qo?GlOymxDbc=^C~TZjf`sqw zCkF?Zk_T&o#S}pB{%{;f!~z!6H0H7AJyFlU`gwn)qGEJECK2(2rDf`^4eVoV$)T4H z{4XyU7?mI8H>^QM%W)aErm-ku6y4eI`SmynR$WfH@zwoybzC>ioDPOX!{oyGoUq4b z5TnV+ZfQiW3(Yi_mDStIXe3!y`l36qJqRdD&;1eZG~9>nw68-PcBE$sRqTc6pHH%K zCn*XanEI&saNYFpU$O6LnEJe!I?8I$Thu-C=(4)-S=5|rSX4wSqIuo#Sk)`&b$nYG ze8S2~p;>3WKO;@GR;aGPZ_BW>R@u@P#ac1);PiBH#G+r89IGYFYCX4pE=PowL(lSG zSrq`AswPFtJ?KT=l|(t97z!2xcdJIc4*0%v+nO&o>F!&y^m{H=F0GR1WG;&yp}ij1 z>cIEiq&>FaxW_aK>5*_nlxRgU2gS7vbK#psLa2%D$NPq56W{z`W5=<7ZF*Gw8i)zj za$-2U;vs#dx01JJD^EVtFrz22LNTLtm$vA+a{hRZ!2UhCyo% z7Wsoyn4YP)5Wlr)+6(HA7yEQ;;aNf#Euvk|s49-31&dNdhtlROG_O7AUYIRr%P88} zu{NWRAXV_*{H0zgtHudsjfnU&6jPIZY#=&2n5djADMfcC{p?Y}_cpI5UnUKcMJSzT z%2Wo99;`kWJi$oJpylxU(zoonBiTXai5+<*h3j+z7-_Obz<5h{5^}UhlHEECsdze^ zu!JJ}so~RB=y-||J!iPB^6Z@-WQ@~J>M*PnD?#+}a`os)p$Z>$pK(UC^_qJ>>Wgg^ z6%!IGo+@C<99!itA(3BD3CPG%&D42-6Blvohx5dmv6e_WXY1_CR6CRNA$y5tPMCllLwb-7T24QlT3<1wSP>;pYSC1bvhkHrLt@ks zLlXQIi0s`_iGH$|rf>=22Xmu1oKCV1+2yTf%Q|M5!2Rg>z3>9ns5Cau13= z{;tLeCp()%+x0_Fh-M}2=9b$)`I2<>A@GR+9%%6JA&3Kwd9 zp9lqycT$g`ywH!F*6K0s+K;2Lm118CO2HGt4q*-OK%;Vl1HDqe(gVAAi^d&8*B8<- zE0&G=n6c6z-s%&z6|;PBGb5BaWU=xM4U+;6UNRAwjLWrjI$_RoFJsG~*(mzS`ugTo zy#41}71Z0>h2vXxsmOV~431rD6wx@Ttc8S%t5dETWaN4|Csz*=TDY&_fr`(upid*! z&c^_s{T5FeGhthyj25a;v8T)|@fgHr{HXRXXNdgri9WfQ_A7O}c*Ew(@mBaw~=nUkNW zO0S7g`q0LcC<0xW#*++x;xvc%tACj^2c#M3AJ`| zvMueFB_JE)lqiH&*IOk$5%1!4u0fk4y&Y*ID~uMYqNmIAHTQV3(hGT^2{WuDl>q~J zCE*A);1B{Ue=5drVp{(}H6f>v<9=-BRsFkz?yF$pP{Bf&BBw_{A4rm>;Nt~)c zuR_v(fM~Pv${iAXGCF1;@hG@dM4`RA8!!dA{)Rk%o^Kt!?d6j*TNOJk-L~nu+9D0q z;L_!W%YkLJTQH+?>Q?!FM6)*;lo{?Xv1vrTG@+aWgrB~biJbIUWW%c}eMZ7uUcK}i z<(uW3aB#g)Sy~3=LN`%dt1>CgXaejT!(lklUEY}) zI=f`{=bTx%D9O=;H=@PLl^D7hlZe5#LYeXVZoP}zxLDw;ip|H=CN$j<xU!pj@ zTDnYEsNT4=9oB34f^^4fmS;tT4FC(!s(B#jgEa2yqmmVV8{^Hb=)c{DECY4hsM7AX zKX_{c_M8dTdYbU{93;LNA^mmrbWUl%8>dNt_{~tYxPZUO!`O;7_~T37Mr3w7JH0%6 zGNh2&zzg{?R3$(6tDx}(s_S0eICo)_xC!BhYupcHwJ|hnhCVRTbJjIB>){b#ER{D& z&d2P8bJhgy(E$*}?X+aBU~R%mbn9z@2u0wjrTudDnACFz$>u!kN{uF{weDw#h!Gk* zt3epgRn?>$DAh3q=(inNgV@zBoHZ#qLYo-FT&b_IEn^?5scd=h^QayhkpC7Z4AkjAQ}O|M#HzqRPKE2e^&bt zfj(hLC%G|Ac(a=Q+Uk&jPJ-giq)#`$G?`ww5r?Fh-SWHiIPM90_Bw+%X1y9EoJ1?v zC#C5bk^xgvEh1GSeXCtq!d?o3Fx`tAIYB1{i0AGMQLfYKT%F^jIXQBm1|ln?VkdgY zebC4;xm$$hwAFa`VqE)zT!?;KS~5NB!xIF&!|&~;i&y#(t1iEYqco?VERRNts)T@x z(?Q38`5}r)H>4Pdq(w|KMd!m}j6s<4LDPi9FtZC7i`1HK8uuVe3udts%py&gsMrq$ z@hx*=3vXpi&)sCvKNLk&U|+PO)?ufhE2`t{kHgV*wBA&ak7cmS^ylwS>9tKNz*1Ri zRcg^(nb$45P|Cg@($Q+;=D(*AYlyEnn;g!OJaE8QDL{deV~n!WTa9wLg8!22n4 zMtm}VsP)xya4|R)PVlZO&tAzi#bxfx`lTgE?j|g2q1b(_00dZgKwc0e~YNv z9Z+>?bOD)dnMAo0lgj*I2@6g`Qw1chg;$~I00PM%x#27O3U&X(1{$7}1CQ)-@3_2c z;4Q<4(GZT2kFp0%&pV)h#llicX1tMMPXs&OI%KwZa=`sY8BubSa$n<|rq|~tAAP!! z!G=s16EloR4Q1ir%5zOGnf>ObM3(LMB`TWF&g>>r``&qY6*Rp<_CqgfW{gB<5Xti1 zAl3F9cDR%EsU4qt>a)S$|D zC5wjx?B0<@81&&4)UZz3JUjf0_Yui@hoF@FwW94^0Y>yOnXyYTjX#-&&jT^V2M^9n zuEr^Io}?!DhqU82C)F(;9u~}(Vav>*M6!axhBqEwFFFxj#Wr&#Y~9nCQIR9yianj05!xhiBe));V{+MhM$p~=!EwD!yj3TBop z{F59P#X8+;o^EFDXn6*keSnnOvlY2-*`dk;624DJQ(hJds_DFb4J}-WokRTLW!;Ug=DC# z8Ya;j^ZXj!<@xN2@y1y1#T{$avK1P@+LSwp`O{!dmEdC)V}Lj^A^U}4)ax(KVnPu{lsQHX|T z`eE=qcWEWZ+Pw=sSZAS#a6y3&X3lqENn(f6`z5@d`=S$PO`>7!UwZ*0x7p6{GT7@u z5tvfto^uA*vttC|?)1gPXGY3JbcK}ivkw+ssOIC#SvZT3@BnA_`x64GGuX;BKa(3& zNHVt)UiT#eH$e>4#7XI}vNcu^tAR#*Qc97e+)fA&nIa1NpcG}6kH$#yJ(^Sic4ftf z=-%;RoWwwJfl8eM1x-wvi-;kWO zi_2iH{brj(PFh%32Trob8J+1@-l~<1`={eao(eqWl)bB4JR!ylwaQlLbh4lFI=~5N zZVR}DvMEBt&?P)#4%JjLYGz%Uv*uxwE)1?Qx?&w{)n7_oOyjP(k?Bg2vYFXJoV-;v+d{DpAzOT@btypq{C4hLBO;XpB@E;p z3u;)+DqB)sbMts=Y?6+bnpTOfXW7iDjEFj!Y&lY`by>VnuV6YkH8g861u0{qfO=T| z@VXu`6<_xdL~qh0{4zr$S-Oth(e#3;+46-!8hxf!5PgM`oO`Z0Q&%D5ZHY+c{w+Uv zfwZ9F97WbkXSj-gHF=G}rHu2qlpHU!P_AA%gfQ-LCYM1vld>>{Jlbzm>8%2q zcU&2OBeNq5So$pyUSlG~79S$f+v$}W7og-BS|dGh%qr;g?>Y$7%a{EOB4q}vpJ!-+ zmn`YXk~yCvp4``BYAvWJe~T=Cp~V6#;;GLlcpof&(sI>f1d^3Y?MC3X&ckmG99PT2 zI=B3GM`s6FhJ(uLxDO_!hR-3_M0BjNF6{Z`m(}0cRqcbLhs!~{h0RLne>!h1=Shi> zQ-cw-Q8X7}giDAkmaA`;+|^5#OAc0l&HKcz$3IX=z=Hi%6!2CuZ{jKjv>}NRGPHs6v1@y zaEF$(SDSMiA>0Ix3iy40@mDpp21)SwCz$l3w1LZ0e1;C#4h@0K4OlS5R|FA- zchTw{(+ip>5W`>AaV{Op1DuFOi4nFR!9r6L5t+n7$1loJ#K@1dDF0V`Zy8m`v+aup zLXhC@?!hIvy9IZb0Kt9X8r&tgySuwX@ZcWY7P4@cx3c%S`|Nx7dFSl=|MI>(Mh`}f zUR~WiXV027LcELMjtj)7X5jm5Da3tkq?(O>;P+S?Bd&n(lAa zx89MOC(OwBSgImZrrZ!+|Db}$AR8$7x)oH@e%`rYK`O69zw+Hum=`-|vXth@R9rIF zcbnel@_Yy9&jy!Sp1Ty{uL0d*hA%962LpN8DNGtx z5Z+99LX|`~yV}h+oUPteF(16Mr2Ax)TvW?%9mg3Iqr1oT7?2ynLiGZE!mRWIvYjZ0 z%^`6($tL{uW=LE}_ZNOqXyA+#gy($A{#eZ4T@9z6xHC{(tmLyJ9~gA!NQ=x^w#7%Tp!=ZX{-g&Bi6>N7RV z+r6&Bw}KEUWC*WqfCi3&0V(>iu|)1`~gjtG#4Q(Jp#(w-I3iuia=%-K;Q~NQ(xJ z&~4#Ww-j&pX?dyG2!|jMqB=KXgyUDW7)FTix1laQEQ!^6%ut;?xD|OXIA{_2#DT{r zE(G=af`;w`L9aHGLSZw(aW+HXb4iG`YFwc^c(ktcfqfGgu=eTq<} zxY8Coa$&)fMTqO=Yq-o?F1dlRcC|jEL*`5S3Z4)crCstlpir16PdM)Gj)`S?AG|2< zZR zny=Tt)i`Boiykv?`HB`*d5$BSWvd}Xde~(T8dwj^4BnhWEJPady(a}McPE2mBJUT83hE?{JZ3lMFtOuOQ(eK*Q(qh@BPOS zF?;3OIDW(dvWs04u!g#QCl(*hHXekJyu|%LECz81Id1fXsQkPFei$?5;h3ERb|PaJwekNF8agj7xNZR1$cL*xBja-;ix#qR_2vjN?bph z)H5geA-x-9g2{QB`5U*t3F<_3%09`eVYB$Ve3Z9?6&2QV)N@WKM+1s|%tKmXgzA-d zc?7~gonP1Bid4~j&D4Lt%$J`kzP1okD$tJa9XvRKM(J&jM_z?7^1jXQ=-$wkuPuCk zOYk6&RBs78ffLm4wB`R{;)NV)Zoel~1{*&Vn^uqzws!ZvUHg087dCwn zR(D}=4T+V{V$eQJN%C<`DaCyQp~)oIF_G~xPRY@O`=Z#p-bdvpZqQ~EYJI^I#1Jgd z&_*UECq~UgOe+2A+dV6odD5q+Mbt$6m+t9Z#rDw>(}S$V(DE+aw^T#RlD64=-Xkg$ z(CrK0VHUAyi65<*s>?_vFEelfk7+!oP8og+E^>7!qb`);_<8FzU69b^H88Fnk@wNY zS&|Z|5*wPB^zUD~)tlWgwsQ$atXlM=Ix zqR;d)ozU*8kZAEB`ZBQ z6pIClWu`H%bInYpeLj9M%SSQU-?)~*RHxInxf2nZg?wdyL$dU}lJWV~nM`4muKpPv zw$;WOOmPy{=qO`ZEsd=0d{!Fj6wt!AQ9MQ}$ZWr_HLtShrau%{R_2z}sBC-g+wq;> zVXYE@IW=!Gd95JhBlolV1j@1lhX;Yj2i}!q35t>t-5(d>e6$_T*6G>N)I<%NearP+ z$JecjRX3p)uH0|HE&M=%nGzlgS-t&u?8#HJ!s;)9rJwr3!({N~wQb4xjo9f;v30-l zdfq~QYCJ>WHum{|2hsE$l0g!`7(fi0(UpOi zo8PnV2*pxXMX``ui!~Gg)4-^E_mC;$i>FnVdzlQjg7+uTa-Zi?a$jfX7pn)|SD-3E z%VtVJt`vfCm=g6^sEkMMnZQ;~`<_}Ih35!yGjZK2tjsMX9g-*6Z6Z@-pxV#Ei6RZr zDoD=OfHO1|cd2wuyt|n8!RMnp%Q_S(9rO{#&NmA zS{kUj!6Ks;zQFZh?3n{{eu=;d-O%1tkSLV6Sd~!R5q!RiPH%lB(sCIvlbVIXx&fy;sfcNI51t0*TesEdN3Wlhw%7OFhM{y1+SxoWV0M%j?YoA&`_Y+CJtOy4&P|)$ zF>5xf+HJ^2$Ql|JWzsbu^NZA7L@PTD%Hvn*%J|3M=HY-r$;!sDbkGKvHsSr7UwnmC zBM(ibrW%^dlTW;aKhFdiu31mdpw0U37EcOgtER)QB4f`MCMDMm@?Bl|;qw&~zsz&) zQ_5k#Fa6Q9VrY$kZqvwtP58o_r=G}el@d}sw(hXNOgZADg)Q%JJnvtm+>)+*iOhXML=Ooq>;>-3j>oo{mEMa4Ibi5=bgHp8-Ln51S)n|OKB z)cfX`hGtCD$vq6ra4D^X7%AKj0=TCY2;gb(u)8;w9S-eeo^ikMA zA6Cp(ZVQlEXEv7AvUqbBOrLV=XAxXbX00-~%6^>qc(*Mb+vyR%5gLniQt2VsrSj|w zi;2O)ELa&ELsp6qBnBqu7jmfS>A+HrXGVWzkrwVML62G|4)%0VTdQ8VH|8{sXMjzl zzWzCnnlNh|_X#=-qi{Ia_mM+PT-T)zUxMT%k$Rv9)ieRTeIulj`GolRW-V+43feB% zhxf(j=%Np*#9(^~qg@)r`rU7IPy_H5TjClZ0px;K)`yw3E1UVQL)PSb>#Nfr$Yt#``9F1bZ2-vcteo$z(6fgDd{ZQ$!-I z!PUe#nRut9W3%4%%dmxSr0tH{%qN2}HLMcM=sIyJl?-YC@U9#l$~=G~n)<9g&>PGT z+;^5m=JWB6IwFh8#R!hdml_cZ&2%sfga;kV<@{eJxPtM9YO&NQGZ$zL!<23g>00ZV&e!{NZuM*VD*n~9oMk5HuUu1vuQ;IZkE6D^XFE0|@_T}x9 zuO4wvlc)28Y^(c|msa4>=TjH0XQ^{tqq@8<=xqdpqplJkpyHSiV9Sa#8OX4XU;sv% zq2y;gv-Qq{+X*xEm5JUVY+vRwFbF2oq9&uitsR{&B|`O7y6E3fN zoM=Mc$bi{GO+N0mGaOX|gy1&Yh1&uq){&`Yy-sJXg&C|_ctOYMJ5SyAl8nPULzrPq zC1NcA{g|!iPs~YM85br5CuP9)ZrULa49T}RXM)yuT4gU~QWahKTqm?m@v|4?dyldD ztvpB(k^rDrV;18wPQ>o+SNxq3#HxCOk53~3CVfrh>Db|csUi+b3Dy|Af-6yRmr(E; zN{s5OiA@4}86!|8CkTVksttEEj5NacUm#z<^!)HP!J_HJurvtrJwPcW|2u@q3G(gI z*Er#;Zkux{J+OXCfbI=?RKs*y{hFXrQBENo_W)4x$Ckqek7@`M0ilh0$vT#S;7kwdg!jjV! z=>GQ~tGC%ia*Ae_%iYSyYMv$tTYldt$;Bsmm#Htwu6>HBmQ=8?_*Ik0)#~jSC7Uc`5)yM^7iwO}=V$O8RB$$6eigVp{Z-8YWHq zC;sjLv2p;cZMg>*FjVB@r@ci^pl8la%m6RBAN-q+T_@%2#nhXh<&WgBAeaH(673(S z-~cgEdfmyH!E0qgd&5d?s`I?F?TUdfBg}l_US1U`rj1gaPDrS0q$~Zi7Bz_dT;NBl z^G(*r%?;gqqH-pt7v48uLYGB#LKL0BrTt#+!B4?uwJ@GjwQ2jhP0wl7i#1o0zVO?$ zISdR$JgA6AyLEmvB-Yo`X3?(4+v|qzjANx5_Kt}IY4F|x*9AJ>i`VCgPuvBz@D|H< zO2szc-U6so$K$chre|BdY6*9>FDI0tGaAj@?TfbBy?2r*o2c>iKSIW2uHTXX`%lHZ zm&R<_U_{!Zgt!)Ab)?^aA&CZ8e-_F!sm&hOv&i&*P7f_qvogy6wZEM}-nM5jrDzP00zY+gnG1x&d;x9a!_*`y32Y-PS#5s*iCZ}?AqSm2i zV6qDQ%7U!V>a*moIo^Rn##YWK=r1^ljpRAAkgrl1rCRFcnXIP%pmokz5&zX@L!A)3 z-c#LtQv9To`o;5O&M%pV`P$l8=kUqRM${S-E-`l`%XU1?gYg0@!#!QmhFYkwH;p%7 zwN)ryM+np!>^c8FbYRGzi6sZEh6mVi)5IdCIo-qt-hAg}&_i`L35nlp&nLPFGo%&< z2ZDc(u1B?)rxC@_z2rl9sX{IP^7#oeAHc&JxzX_|E~@9VZ81;hagcUQ+B}tVITWAt z-{!2pA%G}0K~Ur1G7Av5@Pm{PIf^qe5i)wjV)!0fvOGa#p%H4RrPD;$NsMM7t_53& z+vnK;EK1*hHdg;(wf^^w=kGmlSI8`B${$^3PGGcWM@nf~*ldZ>V25@a-b;l?DQ}NY zHS(S;lNrbP7-9zp{n}Uv>A}T>1BY_|aoqptdAZv_KFYUoA><92?z>F>Pn%h)rvnOD z#%U?ma9km2DRwXWPbB`0&>>2c^DIH(rf)xzB=9*yXj-}9GCo)qemc{|=k{H~!C}v#ru7R?cMH=`lpxslg#CUl983w z3#J^XIR+3YfE;m zHfIYV8ZpX$e)9iCJEs+nUaM}PoPv8cOCtp*^nWMre|awd;|=i?dbWuM?R?Ppe8PwS zPAP=^=FK9TDdPJhK@t>~vT~d>>BTmjITrzxWvD!Pv(fH0=XoaqlrbR^psGgJfBcwb ztveDMLWj&RX=ZA!-g)x_k>6-U1-Y^a(JHBfIU);@fx&Fk1ey>(?d|L?E|C{?(WL%@ zCB3Vvb-jhW&Wb>gd}E@6)Ao!71e|IYGQt2aHS&4gUKS{C_Qq{t{?8v-;!^mw^95ME{+;ohN|_ z=%5_5|KCQ(e`qkV&GR-c=fj$J*=jtQ6;5|l0=msYfiXkTeIOvhApHN<#lG>Lydn)V z1eH8*Z)#4u|2&RB%a5Y#+wwk$ztVhtB|C$Q+*AJlM z3#WnSGYhd`jvL>N$~{`{=13i)z_oa_LqcaOQzPIPp`HQ|3whEpu-srP7dqdBmxq4RS)W@K^2vu2p@l5!d}beSq%<~;brk{?&r0d)04;h z$GqqxZvy46y-=^!pnPsjvK=w#usVMCeGIuUgrL=KIOmtUx%fp1?md3N{BOHD+#|!y zd{IRBW1{Qanzvgzhv=O{ltM;EERKNAF!A;~qlL|Oy{Zob{v#`B0uyY4=8GpTTV-QE zS1z$QZ01+mZCKbJrTGNOAv4$LXb>kt%Z%xXFrzq8qF%HjP_(Vyi`QKYz~d_rWo_IO zB}JW*6{|m=S>Zeu_rrFVBRy#z5aDp_W8-{ofrqSoYR_$P&lW`c=7v8>+<2j1)H9dv z!U~f0!xgRby4R>;6MUu<^rA#O_Ua-VaqFN`^gd-p%Ej3(UpQ5iCu*wj#=KaqV=*7# zZw$VeAE7gk8CBAI_`MNn#;F9*j*EikH`wFYVIlG#)5K;$8-vfJWRlp*<*V=piDPu& ze@8ix4gSrUP!&8$lMQP(I_WAbU+om3V8|A4MqxO4mx4Oh$nIyRs!5>xsxK~0OYeo* z{@V>R%oAxl^))RaGRtcoy$LtMEP+BTsNas-Sl26<|2FBilLt`c6VAO8xTWT3=@4Qt z%;4{(y;1M<;n)`+erxQ*57q79-X7J&{cv;JDxD^5lZRY>&8{c$47mv5nqKEJSzYgk zvO%)Q)Cg)5AU?R@X#L|Lo3dt$L`TAc;b7KF%5-;?FyiBA@HCAsPtl!&l*G3< zJ1fG6B8*F8iqN`DW>N_8W2zM}oZ{u~8)dIs_wOsVEZeTw6fZAf7W-{O_w(4~`GeR3 z?vpQWs1J11n?znwQ4R`*ttBW7#CJSI{B#+CUVC16}D8EvP*#~aOl z;&|3-3clR;rpO5qvRF6nr!cc~bRvUvcv^Qc0i$)U#&bCoyHskzkF%Tu3BszB#P&iAT{i|OHmrT!L&}z z=RS!NUU~Jtd^;Zbj(_geqP(&_Tp?M?lO_H~Em-|<)JxBp`8MQE4&BOiGLJQRv*x-~ zYahPtY~~Td|1vdcj-UJyh7$&idDMG_eSNYKS&44 zV)G-q2`dpaXv8U(5v<`o`JxpUo?tF0JTih%kusx)6who%7=DanQe1A$#XD&khQGfE z-_=!O0u0>&*6M-jhl@~FR3rm7P(H=Q-8YE1EPkXo&{c&yegd(`HMPK(eb0WXEaIXf zR{2`LC{xGhCZ_hBZD$lgK!yjSVr=irOILB)X1I_JG79$+c>m@?QIj%raF>YsU@Mdvka%z1UIN=rO87tZ~XgWUAfu>mZr* z&M?>*U&ZEYdvoOwhDA>D(#ORiqp@1M#P>c4e#EHX%kVV? zp6S|CiC+_m1(Jg0Jq4gg53f<~X&;dg#J_x5Tl6^Q>(lUcg7Q__lgZ`@d< zviHRy8*_+N!d*6IOR7f}a^Apr9l`WA4UQIMG(>*aXIY!~6lxxqjmIEacYO1n=8Bc4 z8_A4H?}djvO_B=?3g;YeEp)#OD7}yM8~JS#%($o_TdU#XBg5Y!MnB@+{QQQu=8cl{ zLMhIG5prSLS`q^s(cm}o*1WbAS3P+3L~r^T?YfTIO{Q+s?3Pr(Zbi5a2xHFeN3Fs$ zo+ZQTLOr58yX&0;CHPNPo*;~jb-!J z6i$H^qNd_gBWIj$0`8zJNwOsGlxS4K$DHr?a^5;kKSyR%lYAlvRi41hAO&VLVHgyn zj>YYvZMeF*X0^BuSS|U|BVX|#DEHkGgjZ!!a$i}pe{^WX!9U;+kxv8TIE@yzU6vS< zcsC^=izpRu_+KT}yLj^?0S>+PZ}}z|@AZwa-%2q|6r7*2kKtA^Olxp= ztY>XdsK)h!zl-ChuqCRTdE;y!7wt$s=Q{qc!Q# z;bK$~=32~{g@o@+j+!VcG;8r<0NwrK_ett5w$UIvF5LA*4$wW)i&(pg%@u&K0K6@b zQxf9(15gCQ&2UcFTFxDn)``fU^&k{)69A7vJn`{)6A-hjT2X9`v+S^-^GC6H5D&Yy zXrhwwGAOvW=tNPjY71$xHlH6ZwlH4GFZ#xK>f;|SdHO&rkA~#M#~>R)3#ozyk@bQ% z!G3FYFv^D`s>pIu5o!|l4_Il21fW zqR$FTzJDsFb@PQ_^&;)1!y1&@MieBk@1{-HjF`c%A2ek(rvCX{BN^K`^K0_aGqZKy z!kkLu=LsE+2*l!{bpbA=atFBqD@0*V7piBDjFn!P9NeI4IQA0611nNnVK4s@&f~Am zIV89=(n-i~qx9YnMjfx>Z+Yv_GrugVA5z)GsUaEkR^_)xUAdkYcom0MkR>-@2!f-g ze(MK@;#z?dWEjG(ViHKrQ`WU~4gz_LKE^Q*Ky%({YGcMG0%e|v!kS9}jVRYkl+eP% z3ZeHXQgOb*b$M(;N)5uu6pf0J?u#{NTks2>T}ia_mi9^zZVKtP&fS)&=gQ)nS z_u`?@qp`yJ-1XB%0Ob(r4YHwtU+Pa$xK8qO;>L+PWJTDQEdqLBM!iVyQ} zd$_PsLU)Fo>#OS#!@H6yE}!h%ts6wyjw~BGE;>8Fu-?VvwQbM+$XagLlv94QT}PW! zBL4>TuahYdhYTwX_s@}Txx@xK$~dRKNV#3a|yxPU7 z^a(nALfstY#ONt(y+E_cH*;+toWVdK^n*iJH5YFso` zWTOQta}}SNrR?0#UlLj8zJNzy-~9ZjViW_M6KZ;kWKsSXRs9yz50B2k5Eh3~KbliM zM`ae{E&bZxaSiMKA;Cz=caCVUkrh*n5xp#&O8O}w^FX1pUy^MjH|RnQ^Bw|Wdu4i5 ze2$SRVftN|I!VK%pB_y1Hj}(Gw$tC#&^T673UvY^!Z>&#UTj*-uxq8!5coQ^sIB4y z(3s=?nB?lfF=X;vrblehelmDEo|wv@U4h#rHJ)?A5!8PYi-=Q2Hqi~JqcHQ5fb_5$ z8YnaFAgX4cqpDbntWRbI@4_4Ka)3<6;L}22vIRD|=m7_@1hpd4lzdoE5dv}j`e^k= ziOT+xoEmXl4~uRSo`0Ty^##xRb|`?6v!zN zTqa3&2N^d?ilgJ=`2FBtZfP*Uk(dIBZbMO8PkIiY)}LaY-J>sz(gGy6v%HtystOXw z^~G_(Mn@Rj+9HJOyrLkiU|aQ&^iDQ&Le>(pV}@6t*S9W z;S-!jB7z5+HXa4n!at&UlkB*P!+j#J*(m8~z4MVmCf{G=&2Io#S#-C^Qm%~OD<^yY z>g0!a3#vk8BD5hQ7$ee;s>@*J%l=GdW@_-9@$KqXG=)Qvo*g0#P+Sa(O};3q*bDq| znn(CRwNUAaexP96i}7I1!f~PkUHE|hk^ezEdrfW7b59GzyPPHTj3vH^X{=>*PzZ68 zh`0lm$FV>1XfWX5wnH{>K*RkqFa^5H*voqLYF=3e?&qR-M$!y0Uu|w7vHH zI2~0#S1^*@raP3)G}5by*@5n3V{#(7OZJ&w`D5`n;&2XQwZs<{pDx9U(QBPTLRw|~ zUpJBhUlQWt{Ho#HW`nO>(q2&vat#ax}qFcdVCA@od+xMM+mzDk0KdM;}zLyZmlN^`_)h(Ul07Z{AuS0Tx#^T92zRMN6CB37 zK?%I!YF_J$G@6N`Fa+LdlKeO>&oKQRWE_h%PSmYnHy%7c~DkVn3|kdKq1cjR7{1q3uIEXAXMqX*is*G*{NXd zZ{t7hvu1h!6Z0vzX>xrqAv?e2M8|zuSN2>~v&HxQOp}d7-+6R&&1=g6twCp;2A8>v zPu1k=f{lBP0&_{dHB`oqb><(t(b|h3yR%alp>I5)p2_(Zdf>WimYYuR?)V<$fcUi_ zf9Q%WzUk4FlK5pD@&PFcW~$W_J#pp}qw$l&Dv!alF;p8G6-w7~(l<9oyePd+7bD*A z;tNYJsDM4mp73)6zA}AWzerQ`Y3VbLF2qjX8!<$D88r)1q_J7`RdS~l?x5|fjgjG% z3Lhcb=@O`YOu!l6&(|aFP3hWC36|6m zazZDt^JcpRA6%t-s$u6AgRkg}ylLcw4#6G?vaHxUiNVV)gVZuM!MQ|zgjKN2ou`ZA zAv1KFNoR+pWC$4#VO`s-isYX6BdU5i5&6n)1U7OHE@28HF8VG-vs3H#C1cd%i`XH4Lu?n*bXSqYQ8#EWBM4FGBN^l=B8UJ1zMBj<8nOUuXM_@v z92dIC&3aIH`$U+X)S)t{u^sp*BrlYot#HcZx{HS#^tB*02)v*48t(ytXN17C7H75J zC*-7b4qr0&pKJXy8_P$lZv>u|$G4kht#VrR_fX-W`1~zxI1+!}e#6{c0Y`)1P029L zu^Ak+V=LlNyaQ)W%+};8c-~m{5$+p#j^D+$`Jzh%(!AHd-Yw6n87eH4yab(R%c_@f z>h`31^D0rSykpSO@7*WZyM|e9+6s9J_;q(-kn1ajuJipm)Mu>UCxXx+9Nl6!ZzAO? z)VQEdTl-Udqs7tOcH3Sb5MvWb&TR+ew#`BJN}GiFC6gb~YVciCOK9PtQhdN~Vb`1u zE|z-9Ng@Id6PDmE(mZV0U4gfIn#hMedp;@zK=eY;u+iIwoxi>%S8zUTU$zaS8 zY``%6VU6|@u7%#_hX_6L%~n0K`h9Lv>^$d>ghgAS^>WJCTKQUr>GNk+tak8BEl)t^ z8=lRj=0%7E;0WZZ#`B)Fbx^XcdPRdXs+D=-xVAX?&5Hh=cRg(MDx1vo7DMT7vR}pM zDvoZxvmH5>@D;FCg2PN<=8nY@sNjaD@O;T47L5}gOX5NXOI@30=lX72#ms2Gz=sh~ zwlAAa8|QZi#n-$=k#A*a>z@lYB>lB`ybCeMLaL=6sfhR=dSAsinC*URW|VbJq%*6K zLl`1JBYDB*GIErKoJ;N6^Ra_)$5D{1xIGnel(|l&<(&(;k5M_)QXO*DJyu{vD^;zA z8=?A@4kIYXRch%(+SlceLTv{Bl;}cJL+@4J|9GVzB~!O z@FqWMI1nqYV}@}kGcl1DhXZuzQ2@Oe&L_4AWlA!-jdzA*8Z0*8S$2raW@&%Y%5<+Z zYJy@-n*hL&5BMCL-=++9P%iE7$D0os%qWiq1Rz$c)}o8iSTR}$$FJ?Avc*af5Qm%c z!*HfY3L#E49zW}+tfElhOI02US`YzzRRSHxV#p3UckVql;RsFh(~tfxR*D4k*tIOS z7-#_6R^m@dbz|C_JSa1*x^qt(I|NIHPl(&*%!K&lj1TO=`kU+Ef=pV4O4y?{Jt&%? z-W_b_{O;q5LUSbIBs(PEaWhc+Ek-(N)gLLdX2tm)uZiw7)xlXY)Ul}ey!w))*!z`(QW8jB=-s;5-gW^B!i6!C~ZljAgW z2a6R7u-g4#>pm4EruVy);gnI6aCqwizs5qHX*1hW;v@AydS2r%j(hq3Mkk$g9z4;0;5(@s?zJ0~#k~_eGE%wAmay#=AmC3sylrOXKuo$Y{VWmtTsYd<_MOMu zK;zX6LjfrgI2K6CS|A-X$Fo7|+SN{t8Jh_@ley#~fdebQ4(mkdelCLByxR6nM2!&s z-1t9n-vze@JhBU$LtbxOG?gUM!J~w_X~OD9eeQ*nB-c@FZxGlhv)g%*vdbra?l2dyn?H`ifX0s%XaPeb{6y}`50 zvfV6X5_B@+paTb~Q4o!6>8-F^{7!}QnGnO#$_M80XeM9^VrV{)AfB86$x{WbnPl;M ze-`H(f3UE==r8E=a#Tr@Y?5rDNxhf0T0S>30KNwj9zD#+vIh>(gB8fIv3>rV&ve71 z-DC1z&{~$A5BVi4^Q_IDpC0JAU+$n|z*oX^vdr{>KXjOCDP}h_=P6{f(G^}%IW}nB zq8wGww1r=2^3uR$%se_V?F74laLc^7itnJ~J|3X<@8mzLYc_sJ(k4955*Kd(TFmH2K&WvZg<~Hh(>a8%B*oo`S zZ4ul>Qng|ru2qd61JK|PkLgY3T-NL1Fkiz3-bnZ1Qm6Nr#N772yigmjT3b-$r))9; zKHMg+tJ*E>KYS7i+Ou^AI>*dPR_I@c&po{HMC;G@l{e2np$8JL7ehU6=GWWiPHlCg z$Fybckiw;e_!Ww3ASBLDh`xr|>0xwhJ8Nd8LVt_mg)}_XWjg}K#})B0gQsBH`sF~G zD}J*IIVQp9Bok)2-3TrZA5`da*=-)Hyg%-Ti#)mgUF9J^fFLWC9VsD0V2AokZ+EG@ zc8vHJltqbr*YiqUz{s0p&*P51l(e%8Ku%gO3RC3=AdOw;>Ej40t4&P_+~b?NI~Dd$ z(WIpA$tMDb#ArwX~_y)Nj( zzvYDk_{f*ujfXv2cu(W(^Q@vFPJQwvq!q+J^junFw9N88sqr#F?=r7zi_9n zpO?i&j5t!^6-Su#&e(qU+qgT!hvdnwkmXxpHXwrAEmQ^|K!llHm-~0dL+xV*j~;e= zVi2)m>+?9Inu{FKp$ANiR1Yjvhh4u#k{bCpweHBjg==mO+EYbqz`pR=@dz>-*caBY z{Vev!>j=aHU7-3;DLp@M|H$kPotcT?VtT>y2`4PyKA&4Lri{|h)Qe&X!lY?NHiE3e z+TU(gD3#sc_lm9-H=(DwMjcNA34|XG9@QXcRP%*2$Ll9n2Tl_0)X{^G=Vj<@2jItY z5n6b{i+!G|_|jNtu8sJr2k(w=Xc~AAyhD%|;X*nY+<%UYpdf|zwJR=CP0-*lY{Vs; zQ~VYHXmvjEo+!xo;kRuY8=}E7Lne8$Q2F>(%w0Ora_Lra4XP*m1|*s+{5%ldq0V{FRhyz%4rk?Dh%k7(vu<^&9GHXMkfr#4AoC5C<%!n4PlI+D$5CK z4Bs<#Lt}Txh2f5J3@IqVM7r}`S-ATt@-;W4E=1wq^jzRo~kPGfIJYC8%_!2d*E_(Hh?<*gum~UaU)KsqCgPD zyRa1XE7y20;7}2Mvi&(ZqyNFt1%Su@#(#@u*g?qu99gWlLrIlA#D-9S0|woWOl%6f znNHht|6)`!IAr&MFpC0FKaPmK)i3p74{pT?;oTc8GzDg)M?GIU0X&ZTq6++TJUxYR z$s1wn(E<5#<(@k_yA<64jgFA(4j?_@nzfby0+#zt#chwTn7q)D8hTxk@W33Xsmq4p zP3qt=VQ?;Ft97@NhVe;VU#qMQsDp8~5k^a{UZTVoWpJ$we9GF{RYPNjgaEDDpGM|H zFS0@xf#+a%y?1iBv3vd9b$SXuDb?Um_GKCA3{Gx#JE_M7kthm{g!OG7nq7fg!p#-K|ei^ao>;Z z?YSiW+d}(S1_?*#HZ7+xGAo)S(*a|R274$z3WfyUchym#04=gE&VqZSLgPRN%sBL+ z&6O&CMHm@UXTT?-k7PwqB?>%(u*5wGBdnes?+1kEToB%^!14Nl|mBO8Mb zxY3x={d-l+N%tIVOPLJw?{-Ohg4ywgt=pfR*hkzVv0X$bz52k45?$$}}p z@?P{b31sm_29)~JK!8iC#-*E`BCaOZ?pz^@ykl@Hbg{deaid2*grgF2)Gg#S(%g}l z-E^UQT&gK7X`_GMmX@V^)=c7sW$285<7jxRWjE2Sg|)jZYMC@IF4K02z>qbTZsP3h zFYFl2DuVrb8tsmsJyeAY%CR5wx?Zc3ZTKRpyHZp?t1sf~PBOZV^CrTMX8v_Emic*k z0f24vw7HGtP%+A3`q+PN)gLnHl{oh?^W2J$xLFcGn@&P^0Jej1Wq`km8z=!AH8^8_ zI-5E7lhWI?|8KBz>FHH;1F-(=j1uMAdePcVZNd!S+j*p4jLz^8H4vMA#nD}O*x14N zUJK~lGpDcgqGouwk2WP47`RllV?@eI4CNQpj$m;Bg}#2f=q~wv;k6CjL1V^&*f8^?My$xkf&B%CKPSVOt`euh-=J5;ae_mFJIKL| zE^tMYK%#zAGHYNr?0gaO9^=8 z^kLK)OMqp43>Wpm>bCrFMIqAfucu%<#q#oJdMOqKrs8Xp+cS<<;gN(VMdzk>V_CRRmF*o}I0!E5kv4D!rm@-evntq0mftP` z3bD!|in#N(8gY!;1m8>M9*tdN)%R4x!%_puX;*(N_;*K)%vJ`VJ-X& zHHbtpFZ>z&vEh7sUh~{utGhoExj7FeKkI)`-)J*uFk*yK~i)cf0*~LZgeH@ z)&+rMA@}e7C7QOLg@d3+P;$UhxsWqsn3n!01zp}bf!u!wj(gvl#qArsDp)Ja$R;x; zqCZdA<)u*?QHekzk}C!|TBJER)DL=42dvI>iXBl^CFv=d&7n^QkoM}kFnD5|T0WSn zN<(gPG{D;;m&3dd7!F-yRM+sKOM0!u7$el0CN1b(KKN~)-``(Eb^dsq3J&H!5ZkNp z-DwPn`lh?xGHi3-@{KB)>}~K^D7U?(No%;Sr|&Oij|sc%VY-jFM(k19T6uEWYUw6vBGsXPO-(r7B$ec^+4Db!N z5Y=vgMH}n%>HRAc!4GJ=z$FB;2y8tuA$=_jbQ%K@-HvS?6#5ALIf^!!~X?TE(~s@j4ZDP zUFvG9{p@~HxS0#`?iJr71t!erc`VB>=>Y2})Zp63)mo80efdcO%n?Q``5rE|``CWi zEpamuJiG%x-x>LRDk?eWnK?ypFSEOGtKr1bQevXeRpmJMBHjmIo7yQ_6y)OsEY{#K z2u6nD$JZ8s6m~U8`d~1)=k1eH?>nuLfsY=eew$3MwXMef`})ZP6ZL(Z&=m>_bVO{s z3k4;?v2#$hAP~^&F;gulzz|4zC?6P#%n63uxk5G{d7Z6;IA_2R@z2U#>`@g-Ik(%L z8|)5j>S!%4;7CTh%Sk$`=^sX0tBj7fNcU{OO4(=)S9oLQ&)&9KE_$E+)K2dfl>JrbXNU`;|iU zWdIdyUAFvoFsAv7n89!C5R(my-vT556`e}*3(ooQm!$f|Od;gM;^MIXWXvPDDE%LZ z$T@|cRh}Y@1a+oLLG|E&1t^|NLS|S@5jzF3{&TN?bpIb4P!!--!;W2+Dg0OHp_g=z zHx;@8@sqOp5O~p;f1rU4~AG6 z|0;n0(;0<7c-=6h>c1wQv3|&KzbvLgnEwv|{yz)Nf8TZ^12a&;+3o)T!T;+>{C_#) zCFKVOPX&&Y{OkDmx6bsSgBj?a*KhS#KcFb2^U4*eoWwtmz<-g{|K{dduwVwF6M`|% ze@z64!GP&3WFyv3f2}*(xfqy%S%>Ff(Dy&-wST{T9}AXXFW3a7zbZl0e;6nh0!F?6 zCCqzI$R7z7O^FQuOWpmT_)x(N{Qp(J|CcJ@{0*WEYF_|YTfzDc{3k6gFIFXD81Vl9 D@PZdq literal 0 HcmV?d00001 diff --git a/site/voyager.md b/site/voyager.md new file mode 100644 index 00000000000..f7d99cc944b --- /dev/null +++ b/site/voyager.md @@ -0,0 +1,130 @@ +# Load balancing with Voyager/HAProxy + +This document explains how to set up Voyager to use HAProxy as load balancer to WebLogic domain(s) running in Kubernetes. + +Voyager is a HAProxy backed ingress controller for Kubernetes. More information about Voyager ingress controller can be found at: https://appscode.com/products/voyager/6.0.0/concepts/. + +## Set up Voyager/HAProxy automatically +To set up Voyager/HAProxy automatically you can run the create-weblogic-domain.sh script to create a WebLogic domain in Kubernetes. You need to change the domain inputs YAML file first. There are three related properties. + +``` +... +# Load balancer to deploy. Supported values are: APACHE, TRAEFIK, VOYAGER, NONE +loadBalancer: VOYAGER +... +# Load balancer web port +loadBalancerWebPort: 30305 + +# Load balancer dashboard port +loadBalancerDashboardPort: 30315 +``` + +* The `loadBalancer` property needs to be changed to `VOYAGER`. +* The `loadBalancerWebPort` property is to specify the `NodePort` number to access the HAProxy load balancer itself. +* The `loadBalancerDashboardPort` property is to specify the `NodePort` number to access the HAPRoxy stats webpage. + +Then after running create-weblogic-domain.sh script, the WebLogic domain is created and the Voyager/HAProxy is also installed and configured properly. +You can access the HAProxy stats webpage via `http://:30315/` and you'll get a webpage like below. + +![HAProxy stats](images/haproxy-stats.png) + +After you deploy some application to the WebLogic cluster, you can send requests to the application via `http://:30305//`. The requests are sent to the HAProxy load balancer and the HAProxy will distribute the requests to managed servers of the WebLogic cluster. + +### What happens underground? +#### 1. Install the Voyager operator +The Voyager operator is installed in `voyager` namespace if it hasn't been done before. + +You should have a voyager-operator-*** pod running in the `voyager` namespace. +``` +$ kubectl get pod -n voyager +NAME READY STATUS RESTARTS AGE +voyager-operator-86bcd6f656-jj2t6 1/1 Running 1 8d +``` +And new CRD groups are registered by the Voyager operator. +``` +$ kubectl get crd -l app=voyager +NAME AGE +certificates.voyager.appscode.com 8d +ingresses.voyager.appscode.com 8d +``` +Now the Voyager operator watches Ingress resources in any namespace. + +#### 2. Create an Voyager Ingress resource +An Ingress resource is generated based on the WebLogic domain configuration and deployed to Kubernetes. Here is an example of what the Voyager Ingress resource might look like for a WebLogic cluster named `cluster-1`, in a domain named `base_domain` with domainUID `domain1`. + +```yaml +apiVersion: voyager.appscode.com/v1beta1 +kind: Ingress +metadata: + name: domain1-voyager + namespace: default + labels: + weblogic.domainUID: domain1 + weblogic.domainName: base_domain + annotations: + ingress.appscode.com/type: 'NodePort' + ingress.appscode.com/stats: 'true' +spec: + rules: + - host: domain1.cluster-1 + http: + nodePort: '30305' + paths: + - backend: + serviceName: domain1-cluster-cluster-1 + servicePort: '8001' +``` + +The Kubernetes service named `domain1-cluster-cluster-1` referred by the Ingress resource is also created by create-weblogic-domain.sh which dynamically includes all the managed servers in the cluster. + +```bash +$ kubectl get endpoints domain1-cluster-cluster-1 +NAME ENDPOINTS AGE +domain1-cluster-cluster-1 10.244.0.170:8001,10.244.0.171:8001,10.244.0.172:8001 2d +``` +When the Ingress resource is deployed, following Kubernetes resources are created by the Voyager operator: +```bash +$ kubectl get deploy,pod,svc | awk '/voyager/ || /NAME/' +NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE +deploy/voyager-domain1-voyager 1 1 1 1 1d + +NAME READY STATUS RESTARTS AGE +po/voyager-domain1-voyager-577bcfb4b-kbnhp 1/1 Running 0 1d + +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +svc/voyager-domain1-voyager NodePort 10.109.137.41 80:30305/TCP 1d +svc/voyager-domain1-voyager-stats ClusterIP 10.105.160.124 56789/TCP 1d +``` +#### 3. Create a NodePort service +A NodePort service is created to expose the HAProxy stats. Following are the example yaml file for the service. +```yaml +apiVersion: v1 +kind: Service +metadata: + name: domain1-voyager-stats + namespace: default + labels: + app: domain1-voyager-stats + weblogic.domainUID: domain1 + weblogic.domainName: base_domain +spec: + type: NodePort + ports: + - name: client + protocol: TCP + port: 56789 + targetPort: 56789 + nodePort: 30315 + selector: + origin: voyager + origin-name: domain1-voyager +``` + +## Set up Voyager manually to WebLogic domain running in Kubernetes +You may want to manually set up Voyager to WebLogic domain running in Kubernetes, in cases when some of the default settings of the automatic steps don't meet your requirements, or your WebLogic domain is not created by the WebLogic Operator. +You need to refer to Voyager documents to do the setup: https://appscode.com/products/voyager/6.0.0/setup/. +### 1. Install the Voyager operator +Refer to install guide of Voyager here: install guide: https://appscode.com/products/voyager/6.0.0/setup/install/. +### 2. Create an Voyager Ingress resource +Refer to the detail here: https://appscode.com/products/voyager/6.0.0/concepts/overview/. You can choose different types for your Ingress: LoadBalancer, NodePort, HostPort and Internal. + From d483a6d6382a001be1105b9a642ac93ba78b86a5 Mon Sep 17 00:00:00 2001 From: Lily He Date: Fri, 4 May 2018 08:18:16 -0700 Subject: [PATCH 172/186] handle uppercase of clusterName when setting up Voyager --- kubernetes/internal/create-weblogic-domain.sh | 80 ++++++++++++------- .../internal/voyager-ingress-template.yaml | 2 +- 2 files changed, 50 insertions(+), 32 deletions(-) diff --git a/kubernetes/internal/create-weblogic-domain.sh b/kubernetes/internal/create-weblogic-domain.sh index 4e551e1fdad..cdc5f0d0ed8 100755 --- a/kubernetes/internal/create-weblogic-domain.sh +++ b/kubernetes/internal/create-weblogic-domain.sh @@ -581,6 +581,7 @@ function createYamlFiles { sed -i -e "s:%DOMAIN_UID%:${domainUID}:g" ${voyagerOutput} sed -i -e "s:%DOMAIN_NAME%:${domainName}:g" ${voyagerOutput} sed -i -e "s:%CLUSTER_NAME%:${clusterName}:g" ${voyagerOutput} + sed -i -e "s:%CLUSTER_NAME_LC%:${clusterNameLC}:g" ${voyagerOutput} sed -i -e "s:%MANAGED_SERVER_PORT%:${managedServerPort}:g" ${voyagerOutput} sed -i -e "s:%LOAD_BALANCER_WEB_PORT%:$loadBalancerWebPort:g" ${voyagerOutput} sed -i -e "s:%LOAD_BALANCER_DASHBOARD_PORT%:$loadBalancerDashboardPort:g" ${voyagerOutput} @@ -698,38 +699,55 @@ function setupVoyagerLoadBalancer { # deploy Voyager Ingress resource kubectl apply -f ${voyagerOutput} - echo Checking Voyager Ingress resource - local maxwaitsecs=100 - local mstart=`date +%s` - while : ; do - local mnow=`date +%s` - local vdep=`kubectl get ingresses.voyager.appscode.com -n ${namespace} | grep ${domainUID}-voyager | wc | awk ' { print $1; } '` - if [ "$vdep" = "1" ]; then - echo 'The Voyager Ingress resource ${domainUID}-voyager is created successfully.' - break - fi - if [ $((mnow - mstart)) -gt $((maxwaitsecs)) ]; then - fail "The Voyager Ingress resource ${domainUID}-voyager was not created." - fi - sleep 5 - done - - echo Checking Voyager service - local maxwaitsecs=100 - local mstart=`date +%s` - while : ; do - local mnow=`date +%s` - local vscv=`kubectl get service ${domainUID}-voyager-stats -n ${namespace} | grep ${domainUID}-voyager-stats | wc | awk ' { print $1; } '` - if [ "$vscv" = "1" ]; then - echo 'The service ${domainUID}-voyager-stats is created successfully.' - break - fi - if [ $((mnow - mstart)) -gt $((maxwaitsecs)) ]; then - fail "The service ${domainUID}-voyager-stats was not created." - fi - sleep 5 - done + echo Checking Voyager Ingress resource + local maxwaitsecs=100 + local mstart=`date +%s` + while : ; do + local mnow=`date +%s` + local vdep=`kubectl get ingresses.voyager.appscode.com -n ${namespace} | grep ${domainUID}-voyager | wc | awk ' { print $1; } '` + if [ "$vdep" = "1" ]; then + echo 'The Voyager Ingress resource ${domainUID}-voyager is created successfully.' + break + fi + if [ $((mnow - mstart)) -gt $((maxwaitsecs)) ]; then + fail "The Voyager Ingress resource ${domainUID}-voyager was not created." + fi + sleep 5 + done + + echo Checking HAProxy pod is running + local maxwaitsecs=100 + local mstart=`date +%s` + while : ; do + local mnow=`date +%s` + local st=`kubectl get pod -n ${namespace} | grep ^voyager-${domainUID}-voyager- | awk ' { print $3; } '` + if [ "$st" = "Running" ]; then + echo 'The HAProxy pod for Voyaer Ingress ${domainUID}-voyager is created successfully.' + break + fi + if [ $((mnow - mstart)) -gt $((maxwaitsecs)) ]; then + fail "The HAProxy pod for Voyaer Ingress ${domainUID}-voyager was not created or running." + fi + sleep 5 + done + + echo Checking Voyager service + local maxwaitsecs=100 + local mstart=`date +%s` + while : ; do + local mnow=`date +%s` + local vscv=`kubectl get service ${domainUID}-voyager-stats -n ${namespace} | grep ${domainUID}-voyager-stats | wc | awk ' { print $1; } '` + if [ "$vscv" = "1" ]; then + echo 'The service ${domainUID}-voyager-stats is created successfully.' + break + fi + if [ $((mnow - mstart)) -gt $((maxwaitsecs)) ]; then + fail "The service ${domainUID}-voyager-stats was not created." + fi + sleep 5 + done } + # # Deploy traefik load balancer # diff --git a/kubernetes/internal/voyager-ingress-template.yaml b/kubernetes/internal/voyager-ingress-template.yaml index 4f9bed4e5e6..4b8f42316a4 100644 --- a/kubernetes/internal/voyager-ingress-template.yaml +++ b/kubernetes/internal/voyager-ingress-template.yaml @@ -16,7 +16,7 @@ spec: nodePort: '%LOAD_BALANCER_WEB_PORT%' paths: - backend: - serviceName: %DOMAIN_UID%-cluster-%CLUSTER_NAME% + serviceName: %DOMAIN_UID%-cluster-%CLUSTER_NAME_LC% servicePort: '%MANAGED_SERVER_PORT%' --- From f28bad4cb8f70e2f232101dd413c05db09c63d7c Mon Sep 17 00:00:00 2001 From: anthony_lai Date: Fri, 4 May 2018 10:58:40 -0700 Subject: [PATCH 173/186] better handling of FileSystemAlreadyExistsException --- .../kubernetes/operator/helpers/ConfigMapHelper.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java index d153e4e9b63..fea853b7ee0 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java @@ -8,6 +8,7 @@ import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.nio.file.FileSystem; +import java.nio.file.FileSystemAlreadyExistsException; import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; @@ -173,15 +174,16 @@ private synchronized Map loadScripts() { } else { return walkScriptsPath(Paths.get(uri)); } - } catch (IOException e) { - LOGGER.warning(MessageKeys.EXCEPTION, e); - e.printStackTrace(); // xyz- - LOGGER.warning(MessageKeys.EXCEPTION, new IOException("xyz- uri is " + uri)); + } catch (FileSystemAlreadyExistsException ale) { + LOGGER.warning(MessageKeys.EXCEPTION, new IOException("xyz-FileSystemAlreadyExistsException uri is " + uri)); try (FileSystem fileSystem = FileSystems.getFileSystem(uri)) { return walkScriptsPath(fileSystem.getPath(SCRIPTS)); - } catch(IOException ioe) { + } catch(IOException e) { throw new RuntimeException(e); } + } catch (IOException e) { + LOGGER.warning(MessageKeys.EXCEPTION, e); + throw new RuntimeException(e); } } From 950392d5a61c9661b3549ec54178d52433ee7b27 Mon Sep 17 00:00:00 2001 From: Rosemary Marano Date: Fri, 4 May 2018 14:59:08 -0400 Subject: [PATCH 174/186] review/edit Voyager doc --- site/voyager.md | 55 +++++++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/site/voyager.md b/site/voyager.md index f7d99cc944b..77b3b94d496 100644 --- a/site/voyager.md +++ b/site/voyager.md @@ -1,11 +1,12 @@ # Load balancing with Voyager/HAProxy -This document explains how to set up Voyager to use HAProxy as load balancer to WebLogic domain(s) running in Kubernetes. +This document explains how to set up Voyager to use the HAProxy as a load balancer for WebLogic domains running in Kubernetes. -Voyager is a HAProxy backed ingress controller for Kubernetes. More information about Voyager ingress controller can be found at: https://appscode.com/products/voyager/6.0.0/concepts/. +Voyager is an HAProxy-backed Ingress controller for Kubernetes. For more information about the Voyager Ingress controller, see https://appscode.com/products/voyager/6.0.0/concepts/. -## Set up Voyager/HAProxy automatically -To set up Voyager/HAProxy automatically you can run the create-weblogic-domain.sh script to create a WebLogic domain in Kubernetes. You need to change the domain inputs YAML file first. There are three related properties. +## Set up the Voyager/HAProxy automatically + +To set up the Voyager/HAProxy automatically, run the `create-weblogic-domain.sh` script to create a WebLogic domain in Kubernetes. However, first you need to change the domain inputs YAML file. There are three load balancer related properties: ``` ... @@ -19,37 +20,37 @@ loadBalancerWebPort: 30305 loadBalancerDashboardPort: 30315 ``` -* The `loadBalancer` property needs to be changed to `VOYAGER`. -* The `loadBalancerWebPort` property is to specify the `NodePort` number to access the HAProxy load balancer itself. -* The `loadBalancerDashboardPort` property is to specify the `NodePort` number to access the HAPRoxy stats webpage. +* The `loadBalancer` property needs to be changed to `VOYAGER`. +* The `loadBalancerWebPort` property is used to specify the `NodePort` number to access the HAProxy load balancer itself. +* The `loadBalancerDashboardPort` property is used to specify the `NodePort` number to access the HAPRoxy stats web page. -Then after running create-weblogic-domain.sh script, the WebLogic domain is created and the Voyager/HAProxy is also installed and configured properly. -You can access the HAProxy stats webpage via `http://:30315/` and you'll get a webpage like below. +Then, after running the `create-weblogic-domain.sh script`, the WebLogic domain is created and the Voyager/HAProxy is also installed and configured properly. +You can access the HAProxy stats web page using `http://:30315/` and will see a web page like the following: ![HAProxy stats](images/haproxy-stats.png) -After you deploy some application to the WebLogic cluster, you can send requests to the application via `http://:30305//`. The requests are sent to the HAProxy load balancer and the HAProxy will distribute the requests to managed servers of the WebLogic cluster. +After you deploy an application to the WebLogic cluster, you can send requests to the application using `http://:30305//`. The requests are sent to the HAProxy load balancer and HAProxy will distribute the requests to the Managed Servers of the WebLogic cluster. ### What happens underground? #### 1. Install the Voyager operator -The Voyager operator is installed in `voyager` namespace if it hasn't been done before. +If it hasn't been done before, the Voyager operator is installed in the `voyager` namespace. -You should have a voyager-operator-*** pod running in the `voyager` namespace. +You should have a `voyager-operator-***` pod running in the `voyager` namespace. ``` $ kubectl get pod -n voyager NAME READY STATUS RESTARTS AGE voyager-operator-86bcd6f656-jj2t6 1/1 Running 1 8d ``` -And new CRD groups are registered by the Voyager operator. +New CRD groups are registered by the Voyager operator. ``` $ kubectl get crd -l app=voyager NAME AGE certificates.voyager.appscode.com 8d ingresses.voyager.appscode.com 8d ``` -Now the Voyager operator watches Ingress resources in any namespace. +Now the Voyager operator watches the Ingress resources in any namespace. -#### 2. Create an Voyager Ingress resource +#### 2. Create a Voyager Ingress resource An Ingress resource is generated based on the WebLogic domain configuration and deployed to Kubernetes. Here is an example of what the Voyager Ingress resource might look like for a WebLogic cluster named `cluster-1`, in a domain named `base_domain` with domainUID `domain1`. ```yaml @@ -75,14 +76,14 @@ spec: servicePort: '8001' ``` -The Kubernetes service named `domain1-cluster-cluster-1` referred by the Ingress resource is also created by create-weblogic-domain.sh which dynamically includes all the managed servers in the cluster. +The Kubernetes service named `domain1-cluster-cluster-1`, referred to by the Ingress resource, is also created by the `create-weblogic-domain.sh` script, which dynamically includes all the Managed Servers in the cluster. ```bash -$ kubectl get endpoints domain1-cluster-cluster-1 +$ kubectl get endpoints domain1-cluster-cluster-1 NAME ENDPOINTS AGE domain1-cluster-cluster-1 10.244.0.170:8001,10.244.0.171:8001,10.244.0.172:8001 2d ``` -When the Ingress resource is deployed, following Kubernetes resources are created by the Voyager operator: +When the Ingress resource is deployed, the following Kubernetes resources are created by the Voyager operator: ```bash $ kubectl get deploy,pod,svc | awk '/voyager/ || /NAME/' NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE @@ -96,7 +97,8 @@ svc/voyager-domain1-voyager NodePort 10.109.137.41 56789/TCP 1d ``` #### 3. Create a NodePort service -A NodePort service is created to expose the HAProxy stats. Following are the example yaml file for the service. +A NodePort service is created to expose the HAProxy stats. The following is an example YAML file for the service. + ```yaml apiVersion: v1 kind: Service @@ -119,12 +121,11 @@ spec: origin: voyager origin-name: domain1-voyager ``` - -## Set up Voyager manually to WebLogic domain running in Kubernetes -You may want to manually set up Voyager to WebLogic domain running in Kubernetes, in cases when some of the default settings of the automatic steps don't meet your requirements, or your WebLogic domain is not created by the WebLogic Operator. -You need to refer to Voyager documents to do the setup: https://appscode.com/products/voyager/6.0.0/setup/. -### 1. Install the Voyager operator -Refer to install guide of Voyager here: install guide: https://appscode.com/products/voyager/6.0.0/setup/install/. -### 2. Create an Voyager Ingress resource -Refer to the detail here: https://appscode.com/products/voyager/6.0.0/concepts/overview/. You can choose different types for your Ingress: LoadBalancer, NodePort, HostPort and Internal. +## Set up Voyager manually to load balance a WebLogic domain running in Kubernetes +You may want to manually set up Voyager to load balance a WebLogic domain running in Kubernetes, in cases when some of the default settings of the automatic steps don't meet your requirements, or your WebLogic domain is not created by the WebLogic Operator. +To do the setup, refer to the Voyager documents at https://appscode.com/products/voyager/6.0.0/setup/. +### 1. Install the Voyager operator +Refer to Voyager install guide at https://appscode.com/products/voyager/6.0.0/setup/install/. +### 2. Create a Voyager Ingress resource +Refer to the details at https://appscode.com/products/voyager/6.0.0/concepts/overview/. You can choose different types for your Ingress: LoadBalancer, NodePort, HostPort, and Internal. From d53b340b0f1c4a67d6b84735eb6544d736ded90b Mon Sep 17 00:00:00 2001 From: Rosemary Marano Date: Fri, 4 May 2018 15:12:43 -0400 Subject: [PATCH 175/186] added x-ref to Voyager doc --- README.md | 2 +- site/installation.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fa1f4377aa1..012622cb328 100644 --- a/README.md +++ b/README.md @@ -148,7 +148,7 @@ For more information, see [Scaling a WebLogic cluster](site/scaling.md). ## Load balancing with an Ingress controller or a web server -You can choose a load balancer provider for your WebLogic domains running in a Kubernetes cluster. Please refer to Load balancing with Voyager Ingress Controller, [Load balancing with Traefik Ingress Controller](site/traefik.md), and [Load balancing with Apache HTTP Server](site/apache.md) for information about the current capabilities and setup instructions for each of the supported load balancers. +You can choose a load balancer provider for your WebLogic domains running in a Kubernetes cluster. Please refer to [Load balancing with Voyager/HAProxy](site/voyager.md), [Load balancing with Traefik](site/traefik.md), and [Load balancing with the Apache HTTP Server](site/apache.md) for information about the current capabilities and setup instructions for each of the supported load balancers. [comment]: # (Exporting operator logs to ELK. The operator provides an option to export its log files to the ELK stack. Please refer to [ELK integration]site/elk.md for information about this capability.) diff --git a/site/installation.md b/site/installation.md index cb7c04efb00..1aec8d2a543 100644 --- a/site/installation.md +++ b/site/installation.md @@ -140,7 +140,7 @@ The operator provides some optional features that can be enabled in the configur ### Load balancing with an Ingress controller or a web server -You can choose a load balancer provider for your WebLogic domains running in a Kubernetes cluster. Please refer to Load balancing with Voyager Ingress Controller, [Load balancing with Traefik Ingress Controller](site/traefik.md), and [Load balancing with the Apache HTTP Server](site/apache.md) for information about the current capabilities and setup instructions for each of the supported load balancers. +You can choose a load balancer provider for your WebLogic domains running in a Kubernetes cluster. Please refer to [Load balancing with Voyager/HAProxy](site/voyager.md), [Load balancing with Traefik](site/traefik.md), and [Load balancing with the Apache HTTP Server](site/apache.md) for information about the current capabilities and setup instructions for each of the supported load balancers. Note these limitations: From 7dbd47913bc9e9973e169ae44f92bab02c825c9c Mon Sep 17 00:00:00 2001 From: anthony_lai Date: Fri, 4 May 2018 12:33:07 -0700 Subject: [PATCH 176/186] load scripts only once --- .../operator/helpers/ConfigMapHelper.java | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java index fea853b7ee0..82f86ec7569 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java @@ -152,12 +152,18 @@ protected V1ConfigMap computeDomainConfigMap() { metadata.setLabels(labels); cm.setMetadata(metadata); + LOGGER.warning("xyz- computeDomainConfigMap called, cm.getData() is " + cm.getData()); cm.setData(loadScripts()); + LOGGER.warning("xyz- computeDomainConfigMap called, cm.getData() after setData().size() is " + cm.getData().size()); return cm; } - + Map scripts; private synchronized Map loadScripts() { + if (scripts != null) { + LOGGER.warning("xyz- computeDomainConfigMap loadScripts() returning previously loaded scripts"); + return scripts; + } URI uri = null; try { uri = getClass().getResource(SCRIPT_LOCATION).toURI(); @@ -169,18 +175,19 @@ private synchronized Map loadScripts() { try { if ("jar".equals(uri.getScheme())) { try (FileSystem fileSystem = FileSystems.newFileSystem(uri, Collections.emptyMap())) { - return walkScriptsPath(fileSystem.getPath(SCRIPTS)); + return scripts = walkScriptsPath(fileSystem.getPath(SCRIPTS)); } } else { - return walkScriptsPath(Paths.get(uri)); - } - } catch (FileSystemAlreadyExistsException ale) { - LOGGER.warning(MessageKeys.EXCEPTION, new IOException("xyz-FileSystemAlreadyExistsException uri is " + uri)); - try (FileSystem fileSystem = FileSystems.getFileSystem(uri)) { - return walkScriptsPath(fileSystem.getPath(SCRIPTS)); - } catch(IOException e) { - throw new RuntimeException(e); + return scripts = walkScriptsPath(Paths.get(uri)); } +// } catch (FileSystemAlreadyExistsException ale) { +// LOGGER.warning(MessageKeys.EXCEPTION, new IOException("xyz-FileSystemAlreadyExistsException uri is " + uri)); +// try (FileSystem fileSystem = FileSystems.getFileSystem(uri)) { +// LOGGER.warning(MessageKeys.EXCEPTION, new IOException("xyz-FileSystem " + fileSystem + ", isOpen()=" + fileSystem.isOpen())); +// return walkScriptsPath(fileSystem.getPath(SCRIPTS)); +// } catch(IOException e) { +// throw new RuntimeException(e); +// } } catch (IOException e) { LOGGER.warning(MessageKeys.EXCEPTION, e); throw new RuntimeException(e); From d385d37563ab45972cff1b09ef50b9e3011e1637 Mon Sep 17 00:00:00 2001 From: anthony_lai Date: Fri, 4 May 2018 13:46:31 -0700 Subject: [PATCH 177/186] another attempt --- .../operator/helpers/ConfigMapHelper.java | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java index 82f86ec7569..90f5d278e55 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java @@ -7,6 +7,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; +import java.nio.file.FileAlreadyExistsException; import java.nio.file.FileSystem; import java.nio.file.FileSystemAlreadyExistsException; import java.nio.file.FileSystems; @@ -152,33 +153,30 @@ protected V1ConfigMap computeDomainConfigMap() { metadata.setLabels(labels); cm.setMetadata(metadata); - LOGGER.warning("xyz- computeDomainConfigMap called, cm.getData() is " + cm.getData()); - cm.setData(loadScripts()); - LOGGER.warning("xyz- computeDomainConfigMap called, cm.getData() after setData().size() is " + cm.getData().size()); + LOGGER.warning("xyz- computeDomainConfigMap called for domainNamespace:" + domainNamespace + ", cm.getData() is " + cm.getData()); + cm.setData(loadScripts(domainNamespace)); + LOGGER.warning("xyz- computeDomainConfigMap called for domainNamespace:" + domainNamespace + ", cm.getData() after setData().size() is " + cm.getData().size()); return cm; } - Map scripts; - private synchronized Map loadScripts() { - if (scripts != null) { - LOGGER.warning("xyz- computeDomainConfigMap loadScripts() returning previously loaded scripts"); - return scripts; - } + + private static synchronized Map loadScripts(String domainNamespace) { URI uri = null; try { - uri = getClass().getResource(SCRIPT_LOCATION).toURI(); + uri = ScriptConfigMapStep.class.getResource(SCRIPT_LOCATION).toURI(); } catch (URISyntaxException e) { LOGGER.warning(MessageKeys.EXCEPTION, e); throw new RuntimeException(e); } - + LOGGER.warning("xyz- loadScripts() domainNamespace:" + domainNamespace + ", uri is " + uri + ", scheme=" + uri.getScheme()); try { if ("jar".equals(uri.getScheme())) { try (FileSystem fileSystem = FileSystems.newFileSystem(uri, Collections.emptyMap())) { - return scripts = walkScriptsPath(fileSystem.getPath(SCRIPTS)); + LOGGER.warning("xyz- walkScriptsPath for domainNamespace=" + domainNamespace + ", fileSystem=" + fileSystem); + return walkScriptsPath(fileSystem.getPath(SCRIPTS), domainNamespace); } } else { - return scripts = walkScriptsPath(Paths.get(uri)); + return walkScriptsPath(Paths.get(uri), domainNamespace); } // } catch (FileSystemAlreadyExistsException ale) { // LOGGER.warning(MessageKeys.EXCEPTION, new IOException("xyz-FileSystemAlreadyExistsException uri is " + uri)); @@ -189,12 +187,13 @@ private synchronized Map loadScripts() { // throw new RuntimeException(e); // } } catch (IOException e) { + LOGGER.warning(MessageKeys.EXCEPTION, new FileAlreadyExistsException("xyz- uri," + uri)); LOGGER.warning(MessageKeys.EXCEPTION, e); throw new RuntimeException(e); } } - private Map walkScriptsPath(Path scriptsDir) throws IOException { + private static Map walkScriptsPath(Path scriptsDir, String domainNamespace) throws IOException { try (Stream walk = Files.walk(scriptsDir, 1)) { Map data = walk.filter(i -> !Files.isDirectory(i)).collect(Collectors.toMap( i -> i.getFileName().toString(), @@ -204,7 +203,7 @@ private Map walkScriptsPath(Path scriptsDir) throws IOException } } - private byte[] read(Path path) { + private static byte[] read(Path path) { try { return Files.readAllBytes(path); } catch (IOException io) { From 6f8729e0cb2ed4c682ac7edbaef4fa4997853e5a Mon Sep 17 00:00:00 2001 From: anthony_lai Date: Fri, 4 May 2018 15:18:25 -0700 Subject: [PATCH 178/186] fix concurrent reading of config map scripts --- .../kubernetes/operator/helpers/ConfigMapHelper.java | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java index 90f5d278e55..bbd125859ad 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java @@ -168,24 +168,14 @@ private static synchronized Map loadScripts(String domainNamespa LOGGER.warning(MessageKeys.EXCEPTION, e); throw new RuntimeException(e); } - LOGGER.warning("xyz- loadScripts() domainNamespace:" + domainNamespace + ", uri is " + uri + ", scheme=" + uri.getScheme()); try { if ("jar".equals(uri.getScheme())) { try (FileSystem fileSystem = FileSystems.newFileSystem(uri, Collections.emptyMap())) { - LOGGER.warning("xyz- walkScriptsPath for domainNamespace=" + domainNamespace + ", fileSystem=" + fileSystem); return walkScriptsPath(fileSystem.getPath(SCRIPTS), domainNamespace); } } else { return walkScriptsPath(Paths.get(uri), domainNamespace); } -// } catch (FileSystemAlreadyExistsException ale) { -// LOGGER.warning(MessageKeys.EXCEPTION, new IOException("xyz-FileSystemAlreadyExistsException uri is " + uri)); -// try (FileSystem fileSystem = FileSystems.getFileSystem(uri)) { -// LOGGER.warning(MessageKeys.EXCEPTION, new IOException("xyz-FileSystem " + fileSystem + ", isOpen()=" + fileSystem.isOpen())); -// return walkScriptsPath(fileSystem.getPath(SCRIPTS)); -// } catch(IOException e) { -// throw new RuntimeException(e); -// } } catch (IOException e) { LOGGER.warning(MessageKeys.EXCEPTION, new FileAlreadyExistsException("xyz- uri," + uri)); LOGGER.warning(MessageKeys.EXCEPTION, e); From 8c9c322fc9b24b75a5af2da7cb7e9560eac87d42 Mon Sep 17 00:00:00 2001 From: anthony_lai Date: Fri, 4 May 2018 15:20:12 -0700 Subject: [PATCH 179/186] fix concurrent reading of config map scripts --- .../oracle/kubernetes/operator/helpers/ConfigMapHelper.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java index bbd125859ad..2ec9b13ed00 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java @@ -7,9 +7,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; -import java.nio.file.FileAlreadyExistsException; import java.nio.file.FileSystem; -import java.nio.file.FileSystemAlreadyExistsException; import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; @@ -153,9 +151,7 @@ protected V1ConfigMap computeDomainConfigMap() { metadata.setLabels(labels); cm.setMetadata(metadata); - LOGGER.warning("xyz- computeDomainConfigMap called for domainNamespace:" + domainNamespace + ", cm.getData() is " + cm.getData()); cm.setData(loadScripts(domainNamespace)); - LOGGER.warning("xyz- computeDomainConfigMap called for domainNamespace:" + domainNamespace + ", cm.getData() after setData().size() is " + cm.getData().size()); return cm; } @@ -177,7 +173,6 @@ private static synchronized Map loadScripts(String domainNamespa return walkScriptsPath(Paths.get(uri), domainNamespace); } } catch (IOException e) { - LOGGER.warning(MessageKeys.EXCEPTION, new FileAlreadyExistsException("xyz- uri," + uri)); LOGGER.warning(MessageKeys.EXCEPTION, e); throw new RuntimeException(e); } From 34961a93f56d0b6b4af344563fcc60980563aacf Mon Sep 17 00:00:00 2001 From: Ryan Eberhard Date: Fri, 4 May 2018 21:44:00 -0400 Subject: [PATCH 180/186] Retry and recheck mechanism --- .../java/oracle/kubernetes/operator/Main.java | 73 ++++++++++++------- .../operator/ProcessingConstants.java | 3 - .../kubernetes/operator/TuningParameters.java | 7 +- .../operator/TuningParametersImpl.java | 2 + .../operator/calls/AsyncRequestStep.java | 19 +---- .../operator/helpers/DomainPresenceInfo.java | 47 ++++++++++++ .../operator/helpers/PodHelper.java | 33 +++++---- .../operator/logging/MessageKeys.java | 3 - .../operator/steps/ManagedServersUpStep.java | 10 +++ .../src/main/resources/Operator.properties | 6 +- 10 files changed, 133 insertions(+), 70 deletions(-) diff --git a/operator/src/main/java/oracle/kubernetes/operator/Main.java b/operator/src/main/java/oracle/kubernetes/operator/Main.java index 58243519671..29c18883287 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/Main.java +++ b/operator/src/main/java/oracle/kubernetes/operator/Main.java @@ -259,6 +259,14 @@ public void onThrowable(Packet packet, Throwable throwable) { } } } + + // start periodic retry and recheck + MainTuning main = tuningAndConfig.getMainTuning(); + engine.getExecutor().scheduleWithFixedDelay(() -> { + for (DomainPresenceInfo info : domains.values()) { + checkAndCreateDomainPresence(info, false); + } + }, main.domainPresenceRecheckIntervalSeconds, main.domainPresenceRecheckIntervalSeconds, TimeUnit.SECONDS); } catch (Throwable e) { LOGGER.warning(MessageKeys.EXCEPTION, e); } finally { @@ -405,6 +413,7 @@ private static void doCheckAndCreateDomainPresence(Domain dom) { private static void doCheckAndCreateDomainPresence(Domain dom, boolean explicitRecheck) { doCheckAndCreateDomainPresence(dom, explicitRecheck, false, null, null); } + private static void doCheckAndCreateDomainPresence(Domain dom, boolean explicitRecheck, boolean explicitRestartAdmin, List explicitRestartServers, List explicitRestartClusters) { @@ -433,6 +442,31 @@ private static void doCheckAndCreateDomainPresence(Domain dom, boolean explicitR } info.setDomain(dom); } + + if (explicitRestartAdmin) { + LOGGER.info(MessageKeys.RESTART_ADMIN_STARTING, domainUID); + info.getExplicitRestartAdmin().set(true); + } + if (explicitRestartServers != null) { + LOGGER.info(MessageKeys.RESTART_SERVERS_STARTING, domainUID, explicitRestartServers); + info.getExplicitRestartServers().addAll(explicitRestartServers); + } + if (explicitRestartClusters != null) { + LOGGER.info(MessageKeys.ROLLING_CLUSTERS_STARTING, domainUID, explicitRestartClusters); + info.getExplicitRestartClusters().addAll(explicitRestartClusters); + } + + checkAndCreateDomainPresence(info); + } + + private static void checkAndCreateDomainPresence(DomainPresenceInfo info) { + checkAndCreateDomainPresence(info, true); + } + + private static void checkAndCreateDomainPresence(DomainPresenceInfo info, boolean isCausedByWatch) { + Domain dom = info.getDomain(); + DomainSpec spec = dom.getSpec(); + String domainUID = spec.getDomainUID(); String ns = dom.getMetadata().getNamespace(); if (initialized.getOrDefault(ns, Boolean.FALSE) && !stopping.get()) { @@ -449,34 +483,10 @@ private static void doCheckAndCreateDomainPresence(Domain dom, boolean explicitR p.getComponents().put(ProcessingConstants.DOMAIN_COMPONENT_NAME, Component.createFor(info, version, pw)); p.put(ProcessingConstants.PRINCIPAL, principal); - if (explicitRestartAdmin) { - p.put(ProcessingConstants.EXPLICIT_RESTART_ADMIN, Boolean.TRUE); - } - p.put(ProcessingConstants.EXPLICIT_RESTART_SERVERS, explicitRestartServers); - p.put(ProcessingConstants.EXPLICIT_RESTART_CLUSTERS, explicitRestartClusters); - - if (explicitRestartAdmin) { - LOGGER.info(MessageKeys.RESTART_ADMIN_STARTING, domainUID); - } - if (explicitRestartServers != null) { - LOGGER.info(MessageKeys.RESTART_SERVERS_STARTING, domainUID, explicitRestartServers); - } - if (explicitRestartClusters != null) { - LOGGER.info(MessageKeys.ROLLING_CLUSTERS_STARTING, domainUID, explicitRestartClusters); - } - - domainUpdaters.startFiber(domainUID, strategy, p, new CompletionCallback() { + CompletionCallback cc = new CompletionCallback() { @Override public void onCompletion(Packet packet) { - if (explicitRestartAdmin) { - LOGGER.info(MessageKeys.RESTART_ADMIN_COMPLETE, domainUID); - } - if (explicitRestartServers != null) { - LOGGER.info(MessageKeys.RESTART_SERVERS_COMPLETE, domainUID, explicitRestartServers); - } - if (explicitRestartClusters != null) { - LOGGER.info(MessageKeys.ROLLING_CLUSTERS_COMPLETE, domainUID, explicitRestartClusters); - } + info.complete(); } @Override @@ -496,9 +506,16 @@ public void onThrowable(Packet packet, Throwable throwable) { } }); - // TODO: consider retrying domain update after a delay + engine.getExecutor().schedule(() -> { checkAndCreateDomainPresence(info, false); }, + tuningAndConfig.getMainTuning().domainPresenceFailureRetrySeconds, TimeUnit.SECONDS); } - }); + }; + + if (isCausedByWatch) { + domainUpdaters.startFiber(domainUID, strategy, p, cc); + } else { + domainUpdaters.startFiberIfNoCurrentFiber(domainUID, strategy, p, cc); + } scheduleDomainStatusUpdating(info); } diff --git a/operator/src/main/java/oracle/kubernetes/operator/ProcessingConstants.java b/operator/src/main/java/oracle/kubernetes/operator/ProcessingConstants.java index fec5c18378a..e8c31b4527f 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/ProcessingConstants.java +++ b/operator/src/main/java/oracle/kubernetes/operator/ProcessingConstants.java @@ -26,9 +26,6 @@ public interface ProcessingConstants { public static final String NETWORK_ACCESS_POINT = "nap"; public static final String SERVERS_TO_ROLL = "roll"; - public static final String EXPLICIT_RESTART_ADMIN = "explicitRestartAdmin"; - public static final String EXPLICIT_RESTART_SERVERS = "explicitRestartServers"; - public static final String EXPLICIT_RESTART_CLUSTERS = "explicitRestartClusters"; public static final String SCRIPT_CONFIG_MAP = "scriptConfigMap"; public static final String SERVER_STATE_MAP = "serverStateMap"; diff --git a/operator/src/main/java/oracle/kubernetes/operator/TuningParameters.java b/operator/src/main/java/oracle/kubernetes/operator/TuningParameters.java index 73c8aa1f28d..e44741fce22 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/TuningParameters.java +++ b/operator/src/main/java/oracle/kubernetes/operator/TuningParameters.java @@ -19,13 +19,18 @@ public static TuningParameters getInstance() { } public static class MainTuning { + public final int domainPresenceFailureRetrySeconds; + public final int domainPresenceRecheckIntervalSeconds; public final int statusUpdateTimeoutSeconds; public final int unchangedCountToDelayStatusRecheck; public final long initialShortDelay; public final long eventualLongDelay; - public MainTuning(int statusUpdateTimeoutSeconds, int unchangedCountToDelayStatusRecheck, + public MainTuning(int domainPresenceFailureRetrySeconds, int domainPresenceRecheckIntervalSeconds, + int statusUpdateTimeoutSeconds, int unchangedCountToDelayStatusRecheck, long initialShortDelay, long eventualLongDelay) { + this.domainPresenceFailureRetrySeconds = domainPresenceFailureRetrySeconds; + this.domainPresenceRecheckIntervalSeconds = domainPresenceRecheckIntervalSeconds; this.statusUpdateTimeoutSeconds = statusUpdateTimeoutSeconds; this.unchangedCountToDelayStatusRecheck = unchangedCountToDelayStatusRecheck; this.initialShortDelay = initialShortDelay; diff --git a/operator/src/main/java/oracle/kubernetes/operator/TuningParametersImpl.java b/operator/src/main/java/oracle/kubernetes/operator/TuningParametersImpl.java index 840a830409d..8297343e3f4 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/TuningParametersImpl.java +++ b/operator/src/main/java/oracle/kubernetes/operator/TuningParametersImpl.java @@ -49,6 +49,8 @@ private void update() { LOGGER.info(MessageKeys.TUNING_PARAMETERS); MainTuning main = new MainTuning( + (int) readTuningParameter("domainPresenceFailureRetrySeconds", 30), + (int) readTuningParameter("domainPresenceRecheckIntervalSeconds", 300), (int) readTuningParameter("statusUpdateTimeoutSeconds", 10), (int) readTuningParameter("statusUpdateUnchangedCountToDelayStatusRecheck", 10), readTuningParameter("statusUpdateInitialShortDelay", 3), diff --git a/operator/src/main/java/oracle/kubernetes/operator/calls/AsyncRequestStep.java b/operator/src/main/java/oracle/kubernetes/operator/calls/AsyncRequestStep.java index e5d2272ae77..4517a2601c3 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/calls/AsyncRequestStep.java +++ b/operator/src/main/java/oracle/kubernetes/operator/calls/AsyncRequestStep.java @@ -86,7 +86,6 @@ public NextAction apply(Packet packet) { LOGGER.fine(MessageKeys.ASYNC_REQUEST, requestParams.call, requestParams.namespace, requestParams.name, requestParams.body, fieldSelector, labelSelector, resourceVersion); AtomicBoolean didResume = new AtomicBoolean(false); - AtomicBoolean didRecycle = new AtomicBoolean(false); ApiClient client = helper.take(); return doSuspend((fiber) -> { ApiCallback callback = new BaseApiCallback() { @@ -97,9 +96,7 @@ public void onFailure(ApiException e, int statusCode, Map> LOGGER.info(MessageKeys.ASYNC_FAILURE, e, statusCode, responseHeaders, requestParams.call, requestParams.namespace, requestParams.name, requestParams.body, fieldSelector, labelSelector, resourceVersion); } - if (didRecycle.compareAndSet(false, true)) { - helper.recycle(client); - } + helper.recycle(client); packet.getComponents().put(RESPONSE_COMPONENT_NAME, Component.createFor(RetryStrategy.class, _retry, new CallResponse(null, e, statusCode, responseHeaders))); fiber.resume(packet); } @@ -110,24 +107,18 @@ public void onSuccess(T result, int statusCode, Map> respon if (didResume.compareAndSet(false, true)) { LOGGER.fine(MessageKeys.ASYNC_SUCCESS, result, statusCode, responseHeaders); - if (didRecycle.compareAndSet(false, true)) { - helper.recycle(client); - } + helper.recycle(client); packet.getComponents().put(RESPONSE_COMPONENT_NAME, Component.createFor(new CallResponse<>(result, null, statusCode, responseHeaders))); fiber.resume(packet); } } }; - + try { CancellableCall c = factory.generate(requestParams, client, _continue, callback); // timeout handling fiber.owner.getExecutor().schedule(() -> { - if (didRecycle.compareAndSet(false, true)) { - // don't recycle on timeout because state is unknown - // usage.recycle(); - } if (didResume.compareAndSet(false, true)) { try { c.cancel(); @@ -140,10 +131,6 @@ public void onSuccess(T result, int statusCode, Map> respon }, timeoutSeconds, TimeUnit.SECONDS); } catch (Throwable t) { LOGGER.warning(MessageKeys.ASYNC_FAILURE, t, 0, null, requestParams, requestParams.namespace, requestParams.name, requestParams.body, fieldSelector, labelSelector, resourceVersion); - if (didRecycle.compareAndSet(false, true)) { - // don't recycle on throwable because state is unknown - // usage.recycle(); - } if (didResume.compareAndSet(false, true)) { packet.getComponents().put(RESPONSE_COMPONENT_NAME, Component.createFor(RetryStrategy.class, _retry)); fiber.resume(packet); diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/DomainPresenceInfo.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/DomainPresenceInfo.java index 6977321f48c..ae71189af4f 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/DomainPresenceInfo.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/DomainPresenceInfo.java @@ -5,9 +5,12 @@ import java.util.Collection; import java.util.List; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CopyOnWriteArraySet; import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import org.joda.time.DateTime; @@ -37,11 +40,16 @@ public class DomainPresenceInfo { private final ConcurrentMap servers = new ConcurrentHashMap<>(); private final ConcurrentMap clusters = new ConcurrentHashMap<>(); private final ConcurrentMap ingresses = new ConcurrentHashMap<>(); + + private final AtomicBoolean explicitRestartAdmin = new AtomicBoolean(false); + private final Set explicitRestartServers = new CopyOnWriteArraySet<>(); + private final Set explicitRestartClusters = new CopyOnWriteArraySet<>(); private V1PersistentVolumeClaimList claims = null; private WlsDomainConfig domainConfig; private DateTime lastScanTime; + private DateTime lastCompletionTime; /** * Create presence for a domain @@ -113,6 +121,21 @@ public void setLastScanTime(DateTime lastScanTime) { this.lastScanTime = lastScanTime; } + /** + * Last completion time + * @return Last completion time + */ + public DateTime getLastCompletionTime() { + return lastCompletionTime; + } + + /** + * Sets the last completion time to now + */ + public void complete() { + this.lastCompletionTime = new DateTime(); + } + /** * Gets the domain. Except the instance to change frequently based on status updates * @return Domain @@ -161,6 +184,30 @@ public ConcurrentMap getIngresses() { return ingresses; } + /** + * Control for if domain has outstanding restart admin server pending + * @return Control for pending admin server restart + */ + public AtomicBoolean getExplicitRestartAdmin() { + return explicitRestartAdmin; + } + + /** + * Control list for outstanding server restarts + * @return Control list for outstanding server restarts + */ + public Set getExplicitRestartServers() { + return explicitRestartServers; + } + + /** + * Control list for outstanding cluster restarts + * @return Control list for outstanding cluster restarts + */ + public Set getExplicitRestartClusters() { + return explicitRestartClusters; + } + /** * Server objects (Pods and Services) for admin server * @return Server objects for admin server diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java index 6c8b2bf22a0..c650c10c18a 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java @@ -91,13 +91,8 @@ public NextAction apply(Packet packet) { DomainPresenceInfo info = packet.getSPI(DomainPresenceInfo.class); - Boolean explicitRestartAdmin = (Boolean) packet.get(ProcessingConstants.EXPLICIT_RESTART_ADMIN); - @SuppressWarnings("unchecked") - List explicitRestartServers = (List) packet.get(ProcessingConstants.EXPLICIT_RESTART_SERVERS); - boolean isExplicitRestartThisServer = - (Boolean.TRUE.equals(explicitRestartAdmin)) || - (explicitRestartServers != null && explicitRestartServers.contains(asName)); + info.getExplicitRestartAdmin().get() || info.getExplicitRestartServers().contains(asName); ServerKubernetesObjects sko = skoFactory.getOrCreate(info, asName); @@ -116,6 +111,8 @@ public NextAction onFailure(Packet packet, ApiException e, int statusCode, public NextAction onSuccess(Packet packet, V1Pod result, int statusCode, Map> responseHeaders) { if (result == null) { + info.getExplicitRestartAdmin().set(false); + info.getExplicitRestartServers().remove(asName); Step create = factory.create().createPodAsync(namespace, adminPod, new ResponseStep(next) { @Override public NextAction onFailure(Packet packet, ApiException e, int statusCode, @@ -145,7 +142,7 @@ public NextAction onSuccess(Packet packet, V1Pod result, int statusCode, Step replace = new CyclePodStep( AdminPodStep.this, podName, namespace, adminPod, MessageKeys.ADMIN_POD_REPLACED, - weblogicDomainUID, asName, sko, next); + weblogicDomainUID, asName, info, sko, next); return doNext(replace, packet); } } @@ -332,9 +329,12 @@ private static class CyclePodStep extends Step { private final String messageKey; private final String weblogicDomainUID; private final String serverName; + private final DomainPresenceInfo info; private final ServerKubernetesObjects sko; - public CyclePodStep(Step conflictStep, String podName, String namespace, V1Pod newPod, String messageKey, String weblogicDomainUID, String serverName, ServerKubernetesObjects sko, Step next) { + public CyclePodStep(Step conflictStep, String podName, String namespace, V1Pod newPod, + String messageKey, String weblogicDomainUID, String serverName, DomainPresenceInfo info, + ServerKubernetesObjects sko, Step next) { super(next); this.conflictStep = conflictStep; this.podName = podName; @@ -343,6 +343,7 @@ public CyclePodStep(Step conflictStep, String podName, String namespace, V1Pod n this.messageKey = messageKey; this.weblogicDomainUID = weblogicDomainUID; this.serverName = serverName; + this.info = info; this.sko = sko; } @@ -365,6 +366,10 @@ public NextAction onFailure(Packet packet, ApiException e, int statusCode, @Override public NextAction onSuccess(Packet packet, V1Status result, int statusCode, Map> responseHeaders) { + if (conflictStep instanceof AdminPodStep) { + info.getExplicitRestartAdmin().set(false); + } + info.getExplicitRestartServers().contains(serverName); Step create = factory.create().createPodAsync(namespace, newPod, new ResponseStep(next) { @Override public NextAction onFailure(Packet packet, ApiException e, int statusCode, @@ -497,14 +502,9 @@ public NextAction apply(Packet packet) { DomainPresenceInfo info = packet.getSPI(DomainPresenceInfo.class); - @SuppressWarnings("unchecked") - List explicitRestartServers = (List) packet.get(ProcessingConstants.EXPLICIT_RESTART_SERVERS); - @SuppressWarnings("unchecked") - List explicitRestartClusters = (List) packet.get(ProcessingConstants.EXPLICIT_RESTART_CLUSTERS); - boolean isExplicitRestartThisServer = - (explicitRestartServers != null && explicitRestartServers.contains(weblogicServerName)) || - (explicitRestartClusters != null && weblogicClusterName != null && explicitRestartClusters.contains(weblogicClusterName)); + info.getExplicitRestartServers().contains(weblogicServerName) || + (weblogicClusterName != null && info.getExplicitRestartClusters().contains(weblogicClusterName)); ServerKubernetesObjects sko = skoFactory.getOrCreate(info, weblogicServerName); @@ -523,6 +523,7 @@ public NextAction onFailure(Packet packet, ApiException e, int statusCode, public NextAction onSuccess(Packet packet, V1Pod result, int statusCode, Map> responseHeaders) { if (result == null) { + info.getExplicitRestartServers().remove(weblogicServerName); Step create = factory.create().createPodAsync(namespace, pod, new ResponseStep(next) { @Override public NextAction onFailure(Packet packet, ApiException e, int statusCode, @@ -553,7 +554,7 @@ public NextAction onSuccess(Packet packet, V1Pod result, int statusCode, Step replace = new CyclePodStep( ManagedPodStep.this, podName, namespace, pod, MessageKeys.MANAGED_POD_REPLACED, - weblogicDomainUID, weblogicServerName, sko, next); + weblogicDomainUID, weblogicServerName, info, sko, next); synchronized (packet) { @SuppressWarnings("unchecked") Map rolling = (Map) packet.get(ProcessingConstants.SERVERS_TO_ROLL); diff --git a/operator/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java b/operator/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java index 18d35ec8e43..0bc1a8d8c9e 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java +++ b/operator/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java @@ -119,9 +119,6 @@ private MessageKeys() {} public static final String NULL_TOKEN_REVIEW_STATUS = "WLSKO-0109"; public static final String NULL_USER_INFO = "WLSKO-0110"; public static final String RESOURCE_BUNDLE_NOT_FOUND = "WLSKO-0111"; - public static final String RESTART_ADMIN_COMPLETE = "WLSKO-0112"; - public static final String RESTART_SERVERS_COMPLETE = "WLSKO-0113"; - public static final String ROLLING_CLUSTERS_COMPLETE = "WLSKO-0114"; public static final String RESTART_ADMIN_STARTING = "WLSKO-0115"; public static final String RESTART_SERVERS_STARTING = "WLSKO-0116"; public static final String ROLLING_CLUSTERS_STARTING = "WLSKO-0117"; diff --git a/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServersUpStep.java b/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServersUpStep.java index 4e374556e06..1303d07246b 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServersUpStep.java +++ b/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServersUpStep.java @@ -65,6 +65,16 @@ public NextAction apply(Packet packet) { Collection ssic = new ArrayList(); String asName = spec.getAsName(); + + for (String clusterName : info.getExplicitRestartClusters()) { + WlsClusterConfig cluster = scan.getClusterConfig(clusterName); + if (cluster != null) { + for (WlsServerConfig server : cluster.getServerConfigs()) { + info.getExplicitRestartServers().add(server.getName()); + } + } + } + info.getExplicitRestartClusters().clear(); boolean startAll = false; Collection servers = new ArrayList(); diff --git a/operator/src/main/resources/Operator.properties b/operator/src/main/resources/Operator.properties index 8d349849c42..d2da81eb117 100644 --- a/operator/src/main/resources/Operator.properties +++ b/operator/src/main/resources/Operator.properties @@ -110,9 +110,9 @@ WLSKO-0108=Null domainUID WLSKO-0109=Null V1TokenReviewStatus WLSKO-0110=Null userInfo {0} WLSKO-0111=Could not find the resource bundle -WLSKO-0112=Restart of administration server for Domain with UID {0} has completed -WLSKO-0113=Restart of servers for Domain with UID {0} in the list {1} has completed -WLSKO-0114=Rolling restart of servers for Domain with UID {0} in the list of clusters {1} has completed +WLSKO-0112= +WLSKO-0113= +WLSKO-0114= WLSKO-0115=Restart of administration server for Domain with UID {0} is starting WLSKO-0116=Restart of servers for Domain with UID {0} in the list {1} is starting WLSKO-0117=Rolling restart of servers for Domain with UID {0} in the list of clusters {1} is starting From f8824744cee0efaa813e71677e3258c1e91cb718 Mon Sep 17 00:00:00 2001 From: Lily He Date: Sat, 5 May 2018 01:14:48 -0700 Subject: [PATCH 181/186] update voyager template --- kubernetes/internal/voyager-ingress-template.yaml | 3 ++- src/integration-tests/bash/run.sh | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/kubernetes/internal/voyager-ingress-template.yaml b/kubernetes/internal/voyager-ingress-template.yaml index affe0485d89..bfe39021dcd 100644 --- a/kubernetes/internal/voyager-ingress-template.yaml +++ b/kubernetes/internal/voyager-ingress-template.yaml @@ -10,9 +10,10 @@ metadata: annotations: ingress.appscode.com/type: 'NodePort' ingress.appscode.com/stats: 'true' + ingress.appscode.com/affinity: 'cookie' spec: rules: - - host: %DOMAIN_UID%.%CLUSTER_NAME% + - host: '*' http: nodePort: '%LOAD_BALANCER_WEB_PORT%' paths: diff --git a/src/integration-tests/bash/run.sh b/src/integration-tests/bash/run.sh index 00e87de8e3d..584693fa614 100755 --- a/src/integration-tests/bash/run.sh +++ b/src/integration-tests/bash/run.sh @@ -1161,12 +1161,11 @@ function verify_webapp_load_balancing { local max_count=30 local wait_time=6 local count=0 - local vheader="host: $DOMAIN_UID.$WL_CLUSTER_NAME" # this is only needed for voyager but it does no harm to traefik etc while [ "${HTTP_RESPONSE}" != "200" -a $count -lt $max_count ] ; do local count=`expr $count + 1` echo "NO_DATA" > $CURL_RESPONSE_BODY - local HTTP_RESPONSE=$(eval "curl --silent --show-error -H '${vheader}' --noproxy ${NODEPORT_HOST} ${TEST_APP_URL} \ + local HTTP_RESPONSE=$(eval "curl --silent --show-error --noproxy ${NODEPORT_HOST} ${TEST_APP_URL} \ --write-out '%{http_code}' \ -o ${CURL_RESPONSE_BODY}" \ ) From abae32dec137f07f0d5fdaf8e096029fa5ef2d37 Mon Sep 17 00:00:00 2001 From: Lily He Date: Sat, 5 May 2018 16:21:15 +0800 Subject: [PATCH 182/186] Update voyager.md --- site/voyager.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/site/voyager.md b/site/voyager.md index 77b3b94d496..cdc0e7af591 100644 --- a/site/voyager.md +++ b/site/voyager.md @@ -65,9 +65,10 @@ metadata: annotations: ingress.appscode.com/type: 'NodePort' ingress.appscode.com/stats: 'true' + ingress.appscode.com/affinity: 'cookie' spec: rules: - - host: domain1.cluster-1 + - host: '*' http: nodePort: '30305' paths: From 2d6dba8d3ee5f4f0fb1d73e1ef0852edda82e791 Mon Sep 17 00:00:00 2001 From: Lily He Date: Sun, 6 May 2018 20:24:23 -0700 Subject: [PATCH 183/186] minor change --- kubernetes/internal/create-weblogic-domain.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/internal/create-weblogic-domain.sh b/kubernetes/internal/create-weblogic-domain.sh index b87b97cfaea..a96aa2071e8 100755 --- a/kubernetes/internal/create-weblogic-domain.sh +++ b/kubernetes/internal/create-weblogic-domain.sh @@ -718,7 +718,7 @@ function setupVoyagerLoadBalancer { local mnow=`date +%s` local vdep=`kubectl get ingresses.voyager.appscode.com -n ${namespace} | grep ${domainUID}-voyager | wc | awk ' { print $1; } '` if [ "$vdep" = "1" ]; then - echo 'The Voyager Ingress resource ${domainUID}-voyager is created successfully.' + echo "The Voyager Ingress resource ${domainUID}-voyager is created successfully." break fi if [ $((mnow - mstart)) -gt $((maxwaitsecs)) ]; then @@ -734,7 +734,7 @@ function setupVoyagerLoadBalancer { local mnow=`date +%s` local st=`kubectl get pod -n ${namespace} | grep ^voyager-${domainUID}-voyager- | awk ' { print $3; } '` if [ "$st" = "Running" ]; then - echo 'The HAProxy pod for Voyaer Ingress ${domainUID}-voyager is created successfully.' + echo "The HAProxy pod for Voyaer Ingress ${domainUID}-voyager is created successfully." break fi if [ $((mnow - mstart)) -gt $((maxwaitsecs)) ]; then From 47de69d480a0052d754393b4e0997530fbd5b072 Mon Sep 17 00:00:00 2001 From: Ryan Eberhard Date: Mon, 7 May 2018 07:02:03 -0400 Subject: [PATCH 184/186] Release 1.0 --- Dockerfile | 20 +- docs/apidocs/allclasses-frame.html | 62 +- docs/apidocs/allclasses-noframe.html | 62 +- docs/apidocs/constant-values.html | 1226 ++- docs/apidocs/deprecated-list.html | 37 +- docs/apidocs/help-doc.html | 39 +- docs/apidocs/index-all.html | 1971 ++-- docs/apidocs/index.html | 41 +- docs/apidocs/jquery/external/jquery/jquery.js | 9789 +++++++++++++++++ .../images/ui-bg_flat_0_aaaaaa_40x100.png | Bin 0 -> 212 bytes .../images/ui-bg_flat_75_ffffff_40x100.png | Bin 0 -> 208 bytes .../images/ui-bg_glass_55_fbf9ee_1x400.png | Bin 0 -> 335 bytes .../images/ui-bg_glass_65_ffffff_1x400.png | Bin 0 -> 207 bytes .../images/ui-bg_glass_75_dadada_1x400.png | Bin 0 -> 262 bytes .../images/ui-bg_glass_75_e6e6e6_1x400.png | Bin 0 -> 262 bytes .../images/ui-bg_glass_95_fef1ec_1x400.png | Bin 0 -> 332 bytes .../ui-bg_highlight-soft_75_cccccc_1x100.png | Bin 0 -> 280 bytes .../jquery/images/ui-icons_222222_256x240.png | Bin 0 -> 6922 bytes .../jquery/images/ui-icons_2e83ff_256x240.png | Bin 0 -> 4549 bytes .../jquery/images/ui-icons_454545_256x240.png | Bin 0 -> 6992 bytes .../jquery/images/ui-icons_888888_256x240.png | Bin 0 -> 6999 bytes .../jquery/images/ui-icons_cd0a0a_256x240.png | Bin 0 -> 4549 bytes docs/apidocs/jquery/jquery-1.10.2.js | 9789 +++++++++++++++++ docs/apidocs/jquery/jquery-ui.css | 544 + docs/apidocs/jquery/jquery-ui.js | 2610 +++++ docs/apidocs/jquery/jquery-ui.min.css | 7 + docs/apidocs/jquery/jquery-ui.min.js | 7 + docs/apidocs/jquery/jquery-ui.structure.css | 152 + .../jquery/jquery-ui.structure.min.css | 5 + .../jquery/jszip-utils/dist/jszip-utils-ie.js | 56 + .../jszip-utils/dist/jszip-utils-ie.min.js | 10 + .../jquery/jszip-utils/dist/jszip-utils.js | 118 + .../jszip-utils/dist/jszip-utils.min.js | 10 + docs/apidocs/jquery/jszip/dist/jszip.js | 9155 +++++++++++++++ docs/apidocs/jquery/jszip/dist/jszip.min.js | 14 + docs/apidocs/member-search-index.js | 1 + docs/apidocs/member-search-index.zip | Bin 0 -> 13266 bytes .../kubernetes/operator/ConfigMapWatcher.html | 74 +- .../operator/DomainStatusUpdater.html | 125 +- .../kubernetes/operator/DomainWatcher.html | 82 +- .../kubernetes/operator/EventWatcher.html | 78 +- .../kubernetes/operator/IngressWatcher.html | 74 +- .../operator/KubernetesConstants.html | 149 +- .../kubernetes/operator/LabelConstants.html | 118 +- .../oracle/kubernetes/operator/Main.html | 147 +- .../kubernetes/operator/OperatorLiveness.html | 93 +- .../kubernetes/operator/PodWatcher.html | 88 +- .../operator/ProcessingConstants.html | 221 +- .../operator/ServerStatusReader.html | 70 +- .../kubernetes/operator/ServiceWatcher.html | 74 +- .../operator/StartupControlConstants.html | 83 +- .../operator/SwaggerBuildHelper.html | 282 - .../TuningParameters.CallBuilderTuning.html | 73 +- .../operator/TuningParameters.MainTuning.html | 110 +- .../operator/TuningParameters.PodTuning.html | 82 +- .../TuningParameters.WatchTuning.html | 65 +- .../kubernetes/operator/TuningParameters.html | 134 +- .../operator/TuningParametersImpl.html | 137 +- .../kubernetes/operator/VersionConstants.html | 358 + .../operator/WebLogicConstants.html | 125 +- .../authentication/Authenticator.html | 116 +- .../operator/authentication/Helpers.html | 57 +- .../class-use/Authenticator.html | 49 +- .../authentication/class-use/Helpers.html | 49 +- .../authentication/package-frame.html | 14 +- .../authentication/package-summary.html | 53 +- .../operator/authentication/package-tree.html | 43 +- .../operator/authentication/package-use.html | 52 +- .../operator/builders/CallParams.html | 103 +- .../builders/WatchBuilder.WatchFactory.html | 57 +- .../operator/builders/WatchBuilder.html | 154 +- .../kubernetes/operator/builders/WatchI.html | 66 +- .../operator/builders/WatchImpl.html | 104 +- .../builders/class-use/CallParams.html | 65 +- .../class-use/WatchBuilder.WatchFactory.html | 37 +- .../builders/class-use/WatchBuilder.html | 107 +- .../operator/builders/class-use/WatchI.html | 112 +- .../builders/class-use/WatchImpl.html | 37 +- .../operator/builders/package-frame.html | 14 +- .../operator/builders/package-summary.html | 67 +- .../operator/builders/package-tree.html | 65 +- .../operator/builders/package-use.html | 72 +- .../operator/calls/AsyncRequestStep.html | 403 + .../operator/calls/CallFactory.html | 268 + .../operator/calls/CallResponse.html | 356 + .../operator/calls/CallWrapper.html | 318 + .../operator/calls/CancellableCall.html | 260 + .../operator/calls/RequestParams.html | 354 + .../RetryStrategy.html} | 95 +- .../calls/class-use/AsyncRequestStep.html | 153 + .../operator/calls/class-use/CallFactory.html | 235 + .../calls/class-use/CallResponse.html | 153 + .../operator/calls/class-use/CallWrapper.html | 153 + .../calls/class-use/CancellableCall.html | 215 + .../calls/class-use/RequestParams.html | 253 + .../calls/class-use/RetryStrategy.html | 153 + .../operator/calls/package-frame.html | 38 + .../operator/calls/package-summary.html | 214 + .../operator/calls/package-tree.html | 179 + .../operator/calls/package-use.html | 225 + .../operator/class-use/ConfigMapWatcher.html | 81 +- .../class-use/DomainStatusUpdater.html | 37 +- .../operator/class-use/DomainWatcher.html | 57 +- .../operator/class-use/EventWatcher.html | 57 +- .../operator/class-use/IngressWatcher.html | 55 +- .../class-use/KubernetesConstants.html | 37 +- .../operator/class-use/LabelConstants.html | 37 +- .../kubernetes/operator/class-use/Main.html | 37 +- .../operator/class-use/OperatorLiveness.html | 37 +- .../operator/class-use/PodWatcher.html | 78 +- .../class-use/ProcessingConstants.html | 37 +- .../class-use/ServerStatusReader.html | 37 +- .../operator/class-use/ServiceWatcher.html | 55 +- .../class-use/StartupControlConstants.html | 37 +- .../TuningParameters.CallBuilderTuning.html | 52 +- .../TuningParameters.MainTuning.html | 52 +- .../class-use/TuningParameters.PodTuning.html | 52 +- .../TuningParameters.WatchTuning.html | 52 +- .../operator/class-use/TuningParameters.html | 86 +- .../class-use/TuningParametersImpl.html | 37 +- ...BuildHelper.html => VersionConstants.html} | 53 +- .../operator/class-use/WebLogicConstants.html | 37 +- .../operator/helpers/AnnotationHelper.html | 184 +- .../helpers/AsyncRequestStepFactory.html | 268 + .../operator/helpers/AuthenticationProxy.html | 75 +- .../helpers/AuthorizationProxy.Operation.html | 116 +- .../helpers/AuthorizationProxy.Resource.html | 425 +- .../helpers/AuthorizationProxy.Scope.html | 89 +- .../operator/helpers/AuthorizationProxy.html | 132 +- .../operator/helpers/CRDHelper.html | 57 +- ...allBuilder.SynchronousCallFactoryImpl.html | 479 + .../operator/helpers/CallBuilder.html | 2056 +--- .../operator/helpers/CallBuilderFactory.html | 77 +- .../operator/helpers/ClientFactory.html | 222 + .../operator/helpers/ClientPool.html | 67 +- .../operator/helpers/ConfigMapConsumer.html | 199 +- .../ConfigMapHelper.ScriptConfigMapStep.html | 79 +- .../operator/helpers/ConfigMapHelper.html | 71 +- .../DomainPresenceInfo.ServerStartupInfo.html | 90 +- .../operator/helpers/DomainPresenceInfo.html | 302 +- .../HealthCheckHelper.KubernetesVersion.html | 70 +- .../operator/helpers/HealthCheckHelper.html | 93 +- .../operator/helpers/IngressHelper.html | 57 +- .../helpers/PodHelper.AdminPodStep.html | 73 +- .../helpers/PodHelper.ManagedPodStep.html | 73 +- .../operator/helpers/PodHelper.html | 80 +- .../kubernetes/operator/helpers/Pool.html | 97 +- .../operator/helpers/ResponseStep.html | 116 +- .../operator/helpers/RollingHelper.html | 59 +- .../helpers/SecretHelper.SecretType.html | 86 +- .../operator/helpers/SecretHelper.html | 122 +- .../helpers/ServerKubernetesObjects.html | 80 +- .../ServerKubernetesObjectsFactory.html | 89 +- .../operator/helpers/ServiceHelper.html | 78 +- .../helpers/SynchronousCallFactory.html | 410 + .../operator/helpers/VersionHelper.html | 317 + .../helpers/class-use/AnnotationHelper.html | 37 +- .../class-use/AsyncRequestStepFactory.html | 153 + .../class-use/AuthenticationProxy.html | 37 +- .../AuthorizationProxy.Operation.html | 82 +- .../AuthorizationProxy.Resource.html | 82 +- .../class-use/AuthorizationProxy.Scope.html | 82 +- .../helpers/class-use/AuthorizationProxy.html | 37 +- .../operator/helpers/class-use/CRDHelper.html | 37 +- ...allBuilder.SynchronousCallFactoryImpl.html | 153 + .../helpers/class-use/CallBuilder.html | 60 +- .../helpers/class-use/CallBuilderFactory.html | 37 +- ....RetryStrategy.html => ClientFactory.html} | 53 +- .../helpers/class-use/ClientPool.html | 102 +- .../helpers/class-use/ConfigMapConsumer.html | 49 +- .../ConfigMapHelper.ScriptConfigMapStep.html | 37 +- .../helpers/class-use/ConfigMapHelper.html | 37 +- .../DomainPresenceInfo.ServerStartupInfo.html | 82 +- .../helpers/class-use/DomainPresenceInfo.html | 99 +- .../HealthCheckHelper.KubernetesVersion.html | 66 +- .../helpers/class-use/HealthCheckHelper.html | 37 +- .../helpers/class-use/IngressHelper.html | 37 +- .../class-use/PodHelper.AdminPodStep.html | 37 +- .../class-use/PodHelper.ManagedPodStep.html | 37 +- .../operator/helpers/class-use/PodHelper.html | 37 +- .../operator/helpers/class-use/Pool.html | 63 +- .../helpers/class-use/ResponseStep.html | 365 +- .../helpers/class-use/RollingHelper.html | 37 +- .../class-use/SecretHelper.SecretType.html | 73 +- .../helpers/class-use/SecretHelper.html | 37 +- .../class-use/ServerKubernetesObjects.html | 149 +- .../ServerKubernetesObjectsFactory.html | 37 +- .../helpers/class-use/ServiceHelper.html | 37 +- .../class-use/SynchronousCallFactory.html | 197 + .../helpers/class-use/VersionHelper.html | 153 + .../operator/helpers/package-frame.html | 20 +- .../operator/helpers/package-summary.html | 171 +- .../operator/helpers/package-tree.html | 117 +- .../operator/helpers/package-use.html | 175 +- .../operator/http/HTTPException.html | 83 +- .../kubernetes/operator/http/HttpClient.html | 150 +- .../kubernetes/operator/http/Result.html | 90 +- .../http/class-use/HTTPException.html | 55 +- .../operator/http/class-use/HttpClient.html | 56 +- .../operator/http/class-use/Result.html | 67 +- .../operator/http/package-frame.html | 14 +- .../operator/http/package-summary.html | 57 +- .../operator/http/package-tree.html | 49 +- .../kubernetes/operator/http/package-use.html | 55 +- .../operator/logging/LoggingFacade.html | 388 +- .../operator/logging/LoggingFactory.html | 66 +- .../operator/logging/LoggingFormatter.html | 77 +- .../operator/logging/MessageKeys.html | 1079 +- .../logging/class-use/LoggingFacade.html | 51 +- .../logging/class-use/LoggingFactory.html | 37 +- .../logging/class-use/LoggingFormatter.html | 37 +- .../logging/class-use/MessageKeys.html | 37 +- .../operator/logging/package-frame.html | 14 +- .../operator/logging/package-summary.html | 57 +- .../operator/logging/package-tree.html | 49 +- .../operator/logging/package-use.html | 49 +- .../kubernetes/operator/package-frame.html | 16 +- .../kubernetes/operator/package-summary.html | 106 +- .../kubernetes/operator/package-tree.html | 89 +- .../kubernetes/operator/package-use.html | 105 +- .../operator/rest/AuthenticationFilter.html | 86 +- .../operator/rest/BaseDebugLoggingFilter.html | 59 +- .../kubernetes/operator/rest/ErrorFilter.html | 76 +- .../operator/rest/ExceptionMapper.html | 72 +- .../operator/rest/FilterPriorities.html | 67 +- .../rest/RequestDebugLoggingFilter.html | 72 +- .../rest/ResponseDebugLoggingFilter.html | 74 +- .../operator/rest/RestBackendImpl.html | 111 +- .../kubernetes/operator/rest/RestConfig.html | 136 +- .../operator/rest/RestConfigImpl.html | 146 +- .../kubernetes/operator/rest/RestServer.html | 90 +- .../operator/rest/backend/RestBackend.html | 85 +- .../operator/rest/backend/VersionUtils.html | 83 +- .../rest/backend/class-use/RestBackend.html | 60 +- .../rest/backend/class-use/VersionUtils.html | 37 +- .../operator/rest/backend/package-frame.html | 14 +- .../rest/backend/package-summary.html | 55 +- .../operator/rest/backend/package-tree.html | 43 +- .../operator/rest/backend/package-use.html | 49 +- .../rest/class-use/AuthenticationFilter.html | 37 +- .../class-use/BaseDebugLoggingFilter.html | 55 +- .../operator/rest/class-use/ErrorFilter.html | 37 +- .../rest/class-use/ExceptionMapper.html | 37 +- .../rest/class-use/FilterPriorities.html | 37 +- .../class-use/RequestDebugLoggingFilter.html | 37 +- .../class-use/ResponseDebugLoggingFilter.html | 37 +- .../rest/class-use/RestBackendImpl.html | 37 +- .../operator/rest/class-use/RestConfig.html | 57 +- .../rest/class-use/RestConfigImpl.html | 37 +- .../operator/rest/class-use/RestServer.html | 37 +- .../operator/rest/model/BaseModel.html | 72 +- .../operator/rest/model/ClusterModel.html | 89 +- .../operator/rest/model/CollectionModel.html | 89 +- .../operator/rest/model/DomainModel.html | 89 +- .../operator/rest/model/ErrorModel.html | 115 +- .../operator/rest/model/ItemModel.html | 69 +- .../rest/model/LinkContainerModel.html | 119 +- .../operator/rest/model/LinkModel.html | 118 +- .../rest/model/ScaleClusterParamsModel.html | 70 +- .../operator/rest/model/VersionModel.html | 115 +- .../rest/model/class-use/BaseModel.html | 73 +- .../rest/model/class-use/ClusterModel.html | 57 +- .../rest/model/class-use/CollectionModel.html | 67 +- .../rest/model/class-use/DomainModel.html | 57 +- .../rest/model/class-use/ErrorModel.html | 37 +- .../rest/model/class-use/ItemModel.html | 63 +- .../model/class-use/LinkContainerModel.html | 61 +- .../rest/model/class-use/LinkModel.html | 59 +- .../class-use/ScaleClusterParamsModel.html | 49 +- .../rest/model/class-use/VersionModel.html | 57 +- .../operator/rest/model/package-frame.html | 14 +- .../operator/rest/model/package-summary.html | 69 +- .../operator/rest/model/package-tree.html | 59 +- .../operator/rest/model/package-use.html | 80 +- .../operator/rest/package-frame.html | 14 +- .../operator/rest/package-summary.html | 73 +- .../operator/rest/package-tree.html | 61 +- .../kubernetes/operator/rest/package-use.html | 52 +- .../operator/rest/resource/BaseResource.html | 49 +- .../rest/resource/ClusterResource.html | 78 +- .../rest/resource/ClustersResource.html | 80 +- .../rest/resource/DomainResource.html | 78 +- .../rest/resource/DomainsResource.html | 80 +- .../rest/resource/ScaleClusterResource.html | 71 +- .../rest/resource/SwaggerResource.html | 73 +- .../rest/resource/VersionResource.html | 85 +- .../rest/resource/VersionsResource.html | 78 +- .../rest/resource/class-use/BaseResource.html | 110 +- .../resource/class-use/ClusterResource.html | 49 +- .../resource/class-use/ClustersResource.html | 49 +- .../resource/class-use/DomainResource.html | 49 +- .../resource/class-use/DomainsResource.html | 49 +- .../class-use/ScaleClusterResource.html | 49 +- .../resource/class-use/SwaggerResource.html | 49 +- .../resource/class-use/VersionResource.html | 49 +- .../resource/class-use/VersionsResource.html | 37 +- .../operator/rest/resource/package-frame.html | 14 +- .../rest/resource/package-summary.html | 71 +- .../operator/rest/resource/package-tree.html | 61 +- .../operator/rest/resource/package-use.html | 70 +- .../steps/BeforeAdminServiceStep.html | 344 + .../operator/steps/ClusterServicesStep.html | 346 + .../operator/steps/ConfigMapAfterStep.html | 352 + .../operator/steps/DeleteDomainStep.html | 346 + .../operator/steps/DeleteIngressListStep.html | 346 + .../operator/steps/DeleteServiceListStep.html | 346 + .../operator/steps/DomainPrescenceStep.html | 346 + .../ExternalAdminChannelIteratorStep.html | 348 + .../steps/ExternalAdminChannelsStep.html | 373 + .../steps/ListPersistentVolumeClaimStep.html | 344 + .../steps/ManagedServerUpAfterStep.html | 344 + .../steps/ManagedServerUpIteratorStep.html | 346 + .../operator/steps/ManagedServersUpStep.html | 344 + .../steps/ServerDownFinalizeStep.html | 346 + .../steps/ServerDownIteratorStep.html | 346 + .../operator/steps/ServerDownStep.html | 348 + .../steps/WatchPodReadyAdminStep.html | 346 + .../class-use/BeforeAdminServiceStep.html | 153 + .../steps/class-use/ClusterServicesStep.html | 153 + .../steps/class-use/ConfigMapAfterStep.html | 153 + .../steps/class-use/DeleteDomainStep.html | 153 + .../class-use/DeleteIngressListStep.html | 153 + .../class-use/DeleteServiceListStep.html | 153 + .../steps/class-use/DomainPrescenceStep.html | 153 + .../ExternalAdminChannelIteratorStep.html | 153 + .../class-use/ExternalAdminChannelsStep.html | 153 + .../ListPersistentVolumeClaimStep.html | 153 + .../class-use/ManagedServerUpAfterStep.html | 153 + .../ManagedServerUpIteratorStep.html | 153 + .../steps/class-use/ManagedServersUpStep.html | 153 + .../class-use/ServerDownFinalizeStep.html | 153 + .../class-use/ServerDownIteratorStep.html | 153 + .../steps/class-use/ServerDownStep.html | 153 + .../class-use/WatchPodReadyAdminStep.html | 153 + .../operator/steps/package-frame.html | 45 + .../operator/steps/package-summary.html | 235 + .../operator/steps/package-tree.html | 186 + .../api => operator/steps}/package-use.html | 69 +- .../operator/utils/ConcurrentWeakHashMap.html | 355 +- .../class-use/ConcurrentWeakHashMap.html | 37 +- .../operator/utils/package-frame.html | 14 +- .../operator/utils/package-summary.html | 45 +- .../operator/utils/package-tree.html | 47 +- .../operator/utils/package-use.html | 37 +- .../operator/watcher/WatchListener.html | 53 +- .../watcher/class-use/WatchListener.html | 150 +- .../operator/watcher/package-frame.html | 14 +- .../operator/watcher/package-summary.html | 51 +- .../operator/watcher/package-tree.html | 39 +- .../operator/watcher/package-use.html | 72 +- .../operator/wlsconfig/ConfigUpdate.html | 263 + .../operator/wlsconfig/MacroSubstitutor.html | 307 + .../wlsconfig/NetworkAccessPoint.html | 115 +- .../operator/wlsconfig/WlsClusterConfig.html | 366 +- .../operator/wlsconfig/WlsDomainConfig.html | 269 +- .../wlsconfig/WlsDynamicServerConfig.html | 308 + .../wlsconfig/WlsDynamicServersConfig.html | 443 + .../operator/wlsconfig/WlsMachineConfig.html | 387 + .../operator/wlsconfig/WlsRetriever.html | 131 +- .../operator/wlsconfig/WlsServerConfig.html | 245 +- .../wlsconfig/class-use/ConfigUpdate.html | 218 + .../wlsconfig/class-use/MacroSubstitutor.html | 153 + .../class-use/NetworkAccessPoint.html | 111 +- .../wlsconfig/class-use/WlsClusterConfig.html | 85 +- .../wlsconfig/class-use/WlsDomainConfig.html | 150 +- .../class-use/WlsDynamicServerConfig.html | 153 + .../class-use/WlsDynamicServersConfig.html | 199 + .../wlsconfig/class-use/WlsMachineConfig.html | 216 + .../wlsconfig/class-use/WlsRetriever.html | 53 +- .../wlsconfig/class-use/WlsServerConfig.html | 169 +- .../operator/wlsconfig/package-frame.html | 22 +- .../operator/wlsconfig/package-summary.html | 105 +- .../operator/wlsconfig/package-tree.html | 60 +- .../operator/wlsconfig/package-use.html | 145 +- .../kubernetes/operator/work/Component.html | 58 +- .../kubernetes/operator/work/ComponentEx.html | 61 +- .../operator/work/ComponentRegistry.html | 55 +- .../kubernetes/operator/work/Container.html | 103 +- .../operator/work/ContainerResolver.html | 84 +- .../kubernetes/operator/work/Engine.html | 96 +- .../work/Fiber.CompletionCallback.html | 58 +- .../operator/work/Fiber.ExitCallback.html | 49 +- .../kubernetes/operator/work/Fiber.html | 269 +- .../kubernetes/operator/work/FiberGate.html | 83 +- .../operator/work/NextAction.Kind.html | 92 +- .../kubernetes/operator/work/NextAction.html | 143 +- .../kubernetes/operator/work/Packet.html | 148 +- .../operator/work/Step.MultiThrowable.html | 77 +- .../operator/work/Step.StepAndPacket.html | 70 +- .../oracle/kubernetes/operator/work/Step.html | 85 +- .../work/ThreadLocalContainerResolver.html | 77 +- .../operator/work/class-use/Component.html | 103 +- .../operator/work/class-use/ComponentEx.html | 52 +- .../work/class-use/ComponentRegistry.html | 55 +- .../operator/work/class-use/Container.html | 91 +- .../work/class-use/ContainerResolver.html | 67 +- .../operator/work/class-use/Engine.html | 57 +- .../class-use/Fiber.CompletionCallback.html | 82 +- .../work/class-use/Fiber.ExitCallback.html | 51 +- .../operator/work/class-use/Fiber.html | 96 +- .../operator/work/class-use/FiberGate.html | 37 +- .../work/class-use/NextAction.Kind.html | 52 +- .../operator/work/class-use/NextAction.html | 238 +- .../operator/work/class-use/Packet.html | 326 +- .../work/class-use/Step.MultiThrowable.html | 37 +- .../work/class-use/Step.StepAndPacket.html | 51 +- .../operator/work/class-use/Step.html | 1008 +- .../ThreadLocalContainerResolver.html | 49 +- .../operator/work/package-frame.html | 14 +- .../operator/work/package-summary.html | 95 +- .../operator/work/package-tree.html | 89 +- .../kubernetes/operator/work/package-use.html | 223 +- .../weblogic/domain/v1/ClusterStartup.html | 563 - .../kubernetes/weblogic/domain/v1/Domain.html | 622 -- .../weblogic/domain/v1/DomainCondition.html | 690 -- .../weblogic/domain/v1/DomainList.html | 563 - .../weblogic/domain/v1/DomainSpec.html | 1074 -- .../weblogic/domain/v1/DomainStatus.html | 622 -- .../weblogic/domain/v1/ServerHealth.html | 498 - .../weblogic/domain/v1/ServerStartup.html | 563 - .../weblogic/domain/v1/ServerStatus.html | 628 -- .../weblogic/domain/v1/SubsystemHealth.html | 504 - .../weblogic/domain/v1/api/WeblogicApi.html | 4370 -------- .../domain/v1/api/class-use/WeblogicApi.html | 126 - .../weblogic/domain/v1/api/package-frame.html | 21 - .../domain/v1/api/package-summary.html | 144 - .../weblogic/domain/v1/api/package-tree.html | 139 - .../domain/v1/class-use/ClusterStartup.html | 250 - .../weblogic/domain/v1/class-use/Domain.html | 796 -- .../domain/v1/class-use/DomainCondition.html | 236 - .../domain/v1/class-use/DomainList.html | 355 - .../domain/v1/class-use/DomainSpec.html | 290 - .../domain/v1/class-use/DomainStatus.html | 221 - .../domain/v1/class-use/ServerHealth.html | 209 - .../domain/v1/class-use/ServerStartup.html | 264 - .../domain/v1/class-use/ServerStatus.html | 230 - .../domain/v1/class-use/SubsystemHealth.html | 218 - .../weblogic/domain/v1/package-frame.html | 30 - .../weblogic/domain/v1/package-summary.html | 209 - .../weblogic/domain/v1/package-tree.html | 148 - .../weblogic/domain/v1/package-use.html | 341 - docs/apidocs/overview-frame.html | 24 +- docs/apidocs/overview-summary.html | 109 +- docs/apidocs/overview-tree.html | 371 +- docs/apidocs/package-list | 4 +- docs/apidocs/package-search-index.js | 1 + docs/apidocs/package-search-index.zip | Bin 0 -> 304 bytes docs/apidocs/resources/glass.png | Bin 0 -> 499 bytes docs/apidocs/resources/x.png | Bin 0 -> 394 bytes docs/apidocs/script.js | 145 +- docs/apidocs/search.js | 349 + docs/apidocs/serialized-form.html | 71 +- docs/apidocs/stylesheet.css | 384 +- docs/apidocs/type-search-index.js | 1 + docs/apidocs/type-search-index.zip | Bin 0 -> 1554 bytes model/pom.xml | 4 +- operator/pom.xml | 4 +- .../operator/helpers/HealthCheckHelper.java | 17 - .../operator/helpers/VersionHelper.java | 1 + .../operator/logging/MessageKeys.java | 1 - .../wlsconfig/WlsDynamicServerConfig.java | 1 - .../wlsconfig/WlsDynamicServersConfig.java | 1 - .../operator/wlsconfig/WlsMachineConfig.java | 1 - operator/src/main/javadoc/overview.html | 1 - .../src/main/resources/Operator.properties | 2 +- pom.xml | 6 +- swagger/pom.xml | 4 +- wercker.yml | 2 +- 468 files changed, 72437 insertions(+), 26435 deletions(-) create mode 100644 docs/apidocs/jquery/external/jquery/jquery.js create mode 100644 docs/apidocs/jquery/images/ui-bg_flat_0_aaaaaa_40x100.png create mode 100644 docs/apidocs/jquery/images/ui-bg_flat_75_ffffff_40x100.png create mode 100644 docs/apidocs/jquery/images/ui-bg_glass_55_fbf9ee_1x400.png create mode 100644 docs/apidocs/jquery/images/ui-bg_glass_65_ffffff_1x400.png create mode 100644 docs/apidocs/jquery/images/ui-bg_glass_75_dadada_1x400.png create mode 100644 docs/apidocs/jquery/images/ui-bg_glass_75_e6e6e6_1x400.png create mode 100644 docs/apidocs/jquery/images/ui-bg_glass_95_fef1ec_1x400.png create mode 100644 docs/apidocs/jquery/images/ui-bg_highlight-soft_75_cccccc_1x100.png create mode 100644 docs/apidocs/jquery/images/ui-icons_222222_256x240.png create mode 100644 docs/apidocs/jquery/images/ui-icons_2e83ff_256x240.png create mode 100644 docs/apidocs/jquery/images/ui-icons_454545_256x240.png create mode 100644 docs/apidocs/jquery/images/ui-icons_888888_256x240.png create mode 100644 docs/apidocs/jquery/images/ui-icons_cd0a0a_256x240.png create mode 100644 docs/apidocs/jquery/jquery-1.10.2.js create mode 100644 docs/apidocs/jquery/jquery-ui.css create mode 100644 docs/apidocs/jquery/jquery-ui.js create mode 100644 docs/apidocs/jquery/jquery-ui.min.css create mode 100644 docs/apidocs/jquery/jquery-ui.min.js create mode 100644 docs/apidocs/jquery/jquery-ui.structure.css create mode 100644 docs/apidocs/jquery/jquery-ui.structure.min.css create mode 100644 docs/apidocs/jquery/jszip-utils/dist/jszip-utils-ie.js create mode 100644 docs/apidocs/jquery/jszip-utils/dist/jszip-utils-ie.min.js create mode 100644 docs/apidocs/jquery/jszip-utils/dist/jszip-utils.js create mode 100644 docs/apidocs/jquery/jszip-utils/dist/jszip-utils.min.js create mode 100644 docs/apidocs/jquery/jszip/dist/jszip.js create mode 100644 docs/apidocs/jquery/jszip/dist/jszip.min.js create mode 100644 docs/apidocs/member-search-index.js create mode 100644 docs/apidocs/member-search-index.zip delete mode 100644 docs/apidocs/oracle/kubernetes/operator/SwaggerBuildHelper.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/VersionConstants.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/calls/AsyncRequestStep.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/calls/CallFactory.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/calls/CallResponse.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/calls/CallWrapper.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/calls/CancellableCall.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/calls/RequestParams.html rename docs/apidocs/oracle/kubernetes/operator/{helpers/CallBuilder.RetryStrategy.html => calls/RetryStrategy.html} (66%) create mode 100644 docs/apidocs/oracle/kubernetes/operator/calls/class-use/AsyncRequestStep.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/calls/class-use/CallFactory.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/calls/class-use/CallResponse.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/calls/class-use/CallWrapper.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/calls/class-use/CancellableCall.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/calls/class-use/RequestParams.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/calls/class-use/RetryStrategy.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/calls/package-frame.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/calls/package-summary.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/calls/package-tree.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/calls/package-use.html rename docs/apidocs/oracle/kubernetes/operator/class-use/{SwaggerBuildHelper.html => VersionConstants.html} (62%) create mode 100644 docs/apidocs/oracle/kubernetes/operator/helpers/AsyncRequestStepFactory.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/helpers/CallBuilder.SynchronousCallFactoryImpl.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/helpers/ClientFactory.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/helpers/SynchronousCallFactory.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/helpers/VersionHelper.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/helpers/class-use/AsyncRequestStepFactory.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/helpers/class-use/CallBuilder.SynchronousCallFactoryImpl.html rename docs/apidocs/oracle/kubernetes/operator/helpers/class-use/{CallBuilder.RetryStrategy.html => ClientFactory.html} (66%) create mode 100644 docs/apidocs/oracle/kubernetes/operator/helpers/class-use/SynchronousCallFactory.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/helpers/class-use/VersionHelper.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/steps/BeforeAdminServiceStep.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/steps/ClusterServicesStep.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/steps/ConfigMapAfterStep.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/steps/DeleteDomainStep.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/steps/DeleteIngressListStep.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/steps/DeleteServiceListStep.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/steps/DomainPrescenceStep.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/steps/ExternalAdminChannelIteratorStep.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/steps/ExternalAdminChannelsStep.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/steps/ListPersistentVolumeClaimStep.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/steps/ManagedServerUpAfterStep.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/steps/ManagedServerUpIteratorStep.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/steps/ManagedServersUpStep.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/steps/ServerDownFinalizeStep.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/steps/ServerDownIteratorStep.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/steps/ServerDownStep.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/steps/WatchPodReadyAdminStep.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/steps/class-use/BeforeAdminServiceStep.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/steps/class-use/ClusterServicesStep.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/steps/class-use/ConfigMapAfterStep.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/steps/class-use/DeleteDomainStep.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/steps/class-use/DeleteIngressListStep.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/steps/class-use/DeleteServiceListStep.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/steps/class-use/DomainPrescenceStep.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/steps/class-use/ExternalAdminChannelIteratorStep.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/steps/class-use/ExternalAdminChannelsStep.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/steps/class-use/ListPersistentVolumeClaimStep.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/steps/class-use/ManagedServerUpAfterStep.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/steps/class-use/ManagedServerUpIteratorStep.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/steps/class-use/ManagedServersUpStep.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/steps/class-use/ServerDownFinalizeStep.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/steps/class-use/ServerDownIteratorStep.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/steps/class-use/ServerDownStep.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/steps/class-use/WatchPodReadyAdminStep.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/steps/package-frame.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/steps/package-summary.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/steps/package-tree.html rename docs/apidocs/oracle/kubernetes/{weblogic/domain/v1/api => operator/steps}/package-use.html (53%) create mode 100644 docs/apidocs/oracle/kubernetes/operator/wlsconfig/ConfigUpdate.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/wlsconfig/MacroSubstitutor.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/wlsconfig/WlsDynamicServerConfig.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/wlsconfig/WlsDynamicServersConfig.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/wlsconfig/WlsMachineConfig.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/wlsconfig/class-use/ConfigUpdate.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/wlsconfig/class-use/MacroSubstitutor.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/wlsconfig/class-use/WlsDynamicServerConfig.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/wlsconfig/class-use/WlsDynamicServersConfig.html create mode 100644 docs/apidocs/oracle/kubernetes/operator/wlsconfig/class-use/WlsMachineConfig.html delete mode 100644 docs/apidocs/oracle/kubernetes/weblogic/domain/v1/ClusterStartup.html delete mode 100644 docs/apidocs/oracle/kubernetes/weblogic/domain/v1/Domain.html delete mode 100644 docs/apidocs/oracle/kubernetes/weblogic/domain/v1/DomainCondition.html delete mode 100644 docs/apidocs/oracle/kubernetes/weblogic/domain/v1/DomainList.html delete mode 100644 docs/apidocs/oracle/kubernetes/weblogic/domain/v1/DomainSpec.html delete mode 100644 docs/apidocs/oracle/kubernetes/weblogic/domain/v1/DomainStatus.html delete mode 100644 docs/apidocs/oracle/kubernetes/weblogic/domain/v1/ServerHealth.html delete mode 100644 docs/apidocs/oracle/kubernetes/weblogic/domain/v1/ServerStartup.html delete mode 100644 docs/apidocs/oracle/kubernetes/weblogic/domain/v1/ServerStatus.html delete mode 100644 docs/apidocs/oracle/kubernetes/weblogic/domain/v1/SubsystemHealth.html delete mode 100644 docs/apidocs/oracle/kubernetes/weblogic/domain/v1/api/WeblogicApi.html delete mode 100644 docs/apidocs/oracle/kubernetes/weblogic/domain/v1/api/class-use/WeblogicApi.html delete mode 100644 docs/apidocs/oracle/kubernetes/weblogic/domain/v1/api/package-frame.html delete mode 100644 docs/apidocs/oracle/kubernetes/weblogic/domain/v1/api/package-summary.html delete mode 100644 docs/apidocs/oracle/kubernetes/weblogic/domain/v1/api/package-tree.html delete mode 100644 docs/apidocs/oracle/kubernetes/weblogic/domain/v1/class-use/ClusterStartup.html delete mode 100644 docs/apidocs/oracle/kubernetes/weblogic/domain/v1/class-use/Domain.html delete mode 100644 docs/apidocs/oracle/kubernetes/weblogic/domain/v1/class-use/DomainCondition.html delete mode 100644 docs/apidocs/oracle/kubernetes/weblogic/domain/v1/class-use/DomainList.html delete mode 100644 docs/apidocs/oracle/kubernetes/weblogic/domain/v1/class-use/DomainSpec.html delete mode 100644 docs/apidocs/oracle/kubernetes/weblogic/domain/v1/class-use/DomainStatus.html delete mode 100644 docs/apidocs/oracle/kubernetes/weblogic/domain/v1/class-use/ServerHealth.html delete mode 100644 docs/apidocs/oracle/kubernetes/weblogic/domain/v1/class-use/ServerStartup.html delete mode 100644 docs/apidocs/oracle/kubernetes/weblogic/domain/v1/class-use/ServerStatus.html delete mode 100644 docs/apidocs/oracle/kubernetes/weblogic/domain/v1/class-use/SubsystemHealth.html delete mode 100644 docs/apidocs/oracle/kubernetes/weblogic/domain/v1/package-frame.html delete mode 100644 docs/apidocs/oracle/kubernetes/weblogic/domain/v1/package-summary.html delete mode 100644 docs/apidocs/oracle/kubernetes/weblogic/domain/v1/package-tree.html delete mode 100644 docs/apidocs/oracle/kubernetes/weblogic/domain/v1/package-use.html create mode 100644 docs/apidocs/package-search-index.js create mode 100644 docs/apidocs/package-search-index.zip create mode 100644 docs/apidocs/resources/glass.png create mode 100644 docs/apidocs/resources/x.png create mode 100644 docs/apidocs/search.js create mode 100644 docs/apidocs/type-search-index.js create mode 100644 docs/apidocs/type-search-index.zip diff --git a/Dockerfile b/Dockerfile index 36b6c070e02..60528137cb7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,26 @@ -# Copyright 2017, 2018, Oracle Corporation and/or its affiliates. All rights reserved. - -# using JRE 8 with support for container heap management +# Copyright 2017, 2018, Oracle Corporation and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. +# +# HOW TO BUILD THIS IMAGE +# ----------------------- +# Run: +# $ docker build -t weblogic-kubernetes-operator:latest . +# +# Pull base image +# From the Docker store +# ------------------------- FROM store/oracle/serverjre:8 +# Maintainer +# ---------- +MAINTAINER Ryan Eberhard + RUN mkdir /operator RUN mkdir /operator/lib ENV PATH=$PATH:/operator COPY src/scripts/* /operator/ -COPY operator/target/weblogic-kubernetes-operator-0.2.jar /operator/weblogic-kubernetes-operator.jar +COPY operator/target/weblogic-kubernetes-operator-1.0.jar /operator/weblogic-kubernetes-operator.jar COPY operator/target/lib/*.jar /operator/lib/ HEALTHCHECK --interval=1m --timeout=10s \ diff --git a/docs/apidocs/allclasses-frame.html b/docs/apidocs/allclasses-frame.html index f4339597974..12ec94f0be3 100644 --- a/docs/apidocs/allclasses-frame.html +++ b/docs/apidocs/allclasses-frame.html @@ -2,18 +2,28 @@ - + +All Classes (operator-runtime 1.0 API) -All Classes (weblogic-kubernetes-operator 0.2 API) - + + + + + + +

All Classes

diff --git a/docs/apidocs/allclasses-noframe.html b/docs/apidocs/allclasses-noframe.html index ee6a248e3d1..dab52ef3abf 100644 --- a/docs/apidocs/allclasses-noframe.html +++ b/docs/apidocs/allclasses-noframe.html @@ -2,18 +2,28 @@ - + +All Classes (operator-runtime 1.0 API) -All Classes (weblogic-kubernetes-operator 0.2 API) - + + + + + + +

All Classes

diff --git a/docs/apidocs/constant-values.html b/docs/apidocs/constant-values.html index 4cdeb9451d3..38f963577b1 100644 --- a/docs/apidocs/constant-values.html +++ b/docs/apidocs/constant-values.html @@ -2,27 +2,36 @@ - + +Constant Field Values (operator-runtime 1.0 API) -Constant Field Values (weblogic-kubernetes-operator 0.2 API) - + + + + + + + +var pathtoroot = "./";loadScripts(document, 'script'); +
@@ -54,6 +63,12 @@ +
    +
  • SEARCH:  + + +
    • +
+
+
+
 
+

Constant Field Values

Contents

@@ -83,520 +107,567 @@

Contents

oracle.kubernetes.*

+
  • - +
    - + - +public static final java.lang.String +
    oracle.kubernetes.operator.rest.AuthenticationFilter 
    Modifier and TypeConstant FieldConstant Field Value
    -public static final StringREST_BACKEND_PROPERTYREST_BACKEND_PROPERTY "RestBackend"
  • - +
    - + @@ -1652,54 +1739,54 @@

    oracle.kubernetes.*

    - + - + - + - + - +
    oracle.kubernetes.operator.rest.FilterPriorities 
    Modifier and TypeConstant FieldConstant Field Value
    public static final intAUTHENTICATION_FILTER_PRIORITYAUTHENTICATION_FILTER_PRIORITY 1000
    public static final intCSRF_PROTECTION_FILTER_PRIORITYCSRF_PROTECTION_FILTER_PRIORITY 1100
    public static final intERROR_FILTER_PRIORITYERROR_FILTER_PRIORITY 4000
    public static final intREQUEST_DEBUG_LOGGING_FILTER_PRIORITYREQUEST_DEBUG_LOGGING_FILTER_PRIORITY 1300
    public static final intRESPONSE_DEBUG_LOGGING_FILTER_PRIORITYRESPONSE_DEBUG_LOGGING_FILTER_PRIORITY 4200
  • - +
    - + - +public static final java.lang.String + @@ -1708,19 +1795,19 @@

    oracle.kubernetes.*

    • -
    oracle.kubernetes.operator.rest.RestConfig 
    Modifier and TypeConstant FieldConstant Field Value
    -public static final StringREST_CONFIG_PROPERTYREST_CONFIG_PROPERTY "RestConfig"
    +
    - + - +public static final java.lang.String + @@ -1770,6 +1857,9 @@

    oracle.kubernetes.*

    } //--> + diff --git a/docs/apidocs/deprecated-list.html b/docs/apidocs/deprecated-list.html index 82cb6091528..b8612de223f 100644 --- a/docs/apidocs/deprecated-list.html +++ b/docs/apidocs/deprecated-list.html @@ -2,27 +2,36 @@ - + +Deprecated List (operator-runtime 1.0 API) -Deprecated List (weblogic-kubernetes-operator 0.2 API) - + + + + + + + +var pathtoroot = "./";loadScripts(document, 'script'); +
    @@ -54,6 +63,12 @@ +
      +
    • SEARCH:  + + +
      • +
    +
    +
    +
     
    +

    Deprecated API

    Contents

    @@ -116,6 +140,9 @@

    Contents

    } //--> +
    diff --git a/docs/apidocs/help-doc.html b/docs/apidocs/help-doc.html index 144a84bc930..74de6a65337 100644 --- a/docs/apidocs/help-doc.html +++ b/docs/apidocs/help-doc.html @@ -2,27 +2,36 @@ - + +API Help (operator-runtime 1.0 API) -API Help (weblogic-kubernetes-operator 0.2 API) - + + + + + + + +var pathtoroot = "./";loadScripts(document, 'script'); +
    @@ -54,6 +63,12 @@ +
      +
    • SEARCH:  + + +
      • +
    +
    +
    +
     
    +

    How This API Document Is Organized

    This API (Application Programming Interface) document has pages corresponding to the items in the navigation bar, described as follows.
    @@ -166,7 +190,7 @@

    Frames/No Frames

    These links show and hide the HTML frames. All pages are available with or without frames.

  • -

    All Classes

    +

    All Classes

    The All Classes link shows all classes and interfaces except non-static nested types.

  • @@ -221,6 +245,9 @@

    Constant Field Values

    } //--> +
    • diff --git a/docs/apidocs/index-all.html b/docs/apidocs/index-all.html index e28be65dca2..92b3b2fc219 100644 --- a/docs/apidocs/index-all.html +++ b/docs/apidocs/index-all.html @@ -2,27 +2,36 @@ - + +Index (operator-runtime 1.0 API) -Index (weblogic-kubernetes-operator 0.2 API) - + + + + + + + +var pathtoroot = "./";loadScripts(document, 'script'); +
    @@ -54,6 +63,12 @@ +
      +
    • SEARCH:  + + +
      • +
    +
    +
    +
     
    +
    A B C D E F G H I J K L M N O P R S T U V W  @@ -121,11 +145,13 @@

    A

     
    ADMIN_STATE - Static variable in interface oracle.kubernetes.operator.WebLogicConstants
     
    -
    adminChannelsToCreate(WlsDomainConfig, Domain) - Static method in class oracle.kubernetes.operator.Main
    +
    adminChannelsToCreate(WlsDomainConfig, Domain) - Static method in class oracle.kubernetes.operator.steps.ExternalAdminChannelsStep
    This method checks the domain spec against any configured network access points defined for the domain.
    +
    AdminCredentials - oracle.kubernetes.operator.helpers.SecretHelper.SecretType
    +
     
    AdminPodStep(Step) - Constructor for class oracle.kubernetes.operator.helpers.PodHelper.AdminPodStep
     
    ALL_STARTUPCONTROL - Static variable in interface oracle.kubernetes.operator.StartupControlConstants
    @@ -139,22 +165,22 @@

    A

    Marks metadata with annotations that let Prometheus know how to retrieve metrics from the wls-exporter web-app.
    -
    annotateWithFormat(V1ObjectMeta) - Static method in class oracle.kubernetes.operator.helpers.AnnotationHelper
    -
    -
    Marks metadata object with an annotation saying that it was created for this format version
    -
    AnnotationHelper - Class in oracle.kubernetes.operator.helpers
    Annotates pods, services with details about the Domain instance and checks these annotations.
    AnnotationHelper() - Constructor for class oracle.kubernetes.operator.helpers.AnnotationHelper
     
    +
    APACHE_LOAD_BALANCER_V1 - Static variable in interface oracle.kubernetes.operator.VersionConstants
    +
     
    APIEXCEPTION_FROM_SUBJECT_ACCESS_REVIEW - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
     
    APIEXCEPTION_FROM_TOKEN_REVIEW - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
     
    APP_LABEL - Static variable in interface oracle.kubernetes.operator.LabelConstants
     
    +
    apply(Packet) - Method in class oracle.kubernetes.operator.calls.AsyncRequestStep
    +
     
    apply(Packet) - Method in class oracle.kubernetes.operator.helpers.ConfigMapHelper.ScriptConfigMapStep
     
    apply(Packet) - Method in class oracle.kubernetes.operator.helpers.PodHelper.AdminPodStep
    @@ -163,6 +189,40 @@

    A

     
    apply(Packet) - Method in class oracle.kubernetes.operator.helpers.ResponseStep
     
    +
    apply(Packet) - Method in class oracle.kubernetes.operator.steps.BeforeAdminServiceStep
    +
     
    +
    apply(Packet) - Method in class oracle.kubernetes.operator.steps.ClusterServicesStep
    +
     
    +
    apply(Packet) - Method in class oracle.kubernetes.operator.steps.ConfigMapAfterStep
    +
     
    +
    apply(Packet) - Method in class oracle.kubernetes.operator.steps.DeleteDomainStep
    +
     
    +
    apply(Packet) - Method in class oracle.kubernetes.operator.steps.DeleteIngressListStep
    +
     
    +
    apply(Packet) - Method in class oracle.kubernetes.operator.steps.DeleteServiceListStep
    +
     
    +
    apply(Packet) - Method in class oracle.kubernetes.operator.steps.DomainPrescenceStep
    +
     
    +
    apply(Packet) - Method in class oracle.kubernetes.operator.steps.ExternalAdminChannelIteratorStep
    +
     
    +
    apply(Packet) - Method in class oracle.kubernetes.operator.steps.ExternalAdminChannelsStep
    +
     
    +
    apply(Packet) - Method in class oracle.kubernetes.operator.steps.ListPersistentVolumeClaimStep
    +
     
    +
    apply(Packet) - Method in class oracle.kubernetes.operator.steps.ManagedServersUpStep
    +
     
    +
    apply(Packet) - Method in class oracle.kubernetes.operator.steps.ManagedServerUpAfterStep
    +
     
    +
    apply(Packet) - Method in class oracle.kubernetes.operator.steps.ManagedServerUpIteratorStep
    +
     
    +
    apply(Packet) - Method in class oracle.kubernetes.operator.steps.ServerDownFinalizeStep
    +
     
    +
    apply(Packet) - Method in class oracle.kubernetes.operator.steps.ServerDownIteratorStep
    +
     
    +
    apply(Packet) - Method in class oracle.kubernetes.operator.steps.ServerDownStep
    +
     
    +
    apply(Packet) - Method in class oracle.kubernetes.operator.steps.WatchPodReadyAdminStep
    +
     
    apply(Packet) - Method in class oracle.kubernetes.operator.work.Step
    Invokes step using the packet as input/output context.
    @@ -179,6 +239,14 @@

    A

     
    ASYNC_TIMEOUT - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
     
    +
    AsyncRequestStep<T> - Class in oracle.kubernetes.operator.calls
    +
    +
    A Step driven by an asynchronous call to the Kubernetes API, which results in a series of callbacks until canceled.
    +
    +
    AsyncRequestStep(ResponseStep<T>, RequestParams, CallFactory<T>, ClientPool, int, int, String, String, String) - Constructor for class oracle.kubernetes.operator.calls.AsyncRequestStep
    +
     
    +
    AsyncRequestStepFactory - Interface in oracle.kubernetes.operator.helpers
    +
     
    AUTHENTICATION_FILTER_PRIORITY - Static variable in class oracle.kubernetes.operator.rest.FilterPriorities
    The authentication filter's priority.
    @@ -249,34 +317,58 @@

    B

    BaseResource is the base resource of all the WebLogic operator's REST resources.
    +
    BeforeAdminServiceStep - Class in oracle.kubernetes.operator.steps
    +
     
    +
    BeforeAdminServiceStep(Step) - Constructor for class oracle.kubernetes.operator.steps.BeforeAdminServiceStep
    +
     
    +
    body - Variable in class oracle.kubernetes.operator.calls.RequestParams
    +
     

    C

    +
    call - Variable in class oracle.kubernetes.operator.calls.RequestParams
    +
     
    CallBuilder - Class in oracle.kubernetes.operator.helpers
    Simplifies synchronous and asynchronous call patterns to the Kubernetes API Server.
    -
    CallBuilder.RetryStrategy - Interface in oracle.kubernetes.operator.helpers
    -
    -
    Failed or timed-out call retry strategy
    -
    +
    CallBuilder.SynchronousCallFactoryImpl - Class in oracle.kubernetes.operator.helpers
    +
     
    CallBuilderFactory - Class in oracle.kubernetes.operator.helpers
     
    -
    CallBuilderFactory(TuningParameters) - Constructor for class oracle.kubernetes.operator.helpers.CallBuilderFactory
    +
    CallBuilderFactory() - Constructor for class oracle.kubernetes.operator.helpers.CallBuilderFactory
     
    CallBuilderTuning(int, int, int) - Constructor for class oracle.kubernetes.operator.TuningParameters.CallBuilderTuning
     
    +
    CallFactory<T> - Interface in oracle.kubernetes.operator.calls
    +
     
    callMaxRetryCount - Variable in class oracle.kubernetes.operator.TuningParameters.CallBuilderTuning
     
    CallParams - Interface in oracle.kubernetes.operator.builders
     
    callRequestLimit - Variable in class oracle.kubernetes.operator.TuningParameters.CallBuilderTuning
     
    +
    CallResponse<T> - Class in oracle.kubernetes.operator.calls
    +
     
    +
    CallResponse(T, ApiException, int, Map<String, List<String>>) - Constructor for class oracle.kubernetes.operator.calls.CallResponse
    +
     
    callTimeoutSeconds - Variable in class oracle.kubernetes.operator.TuningParameters.CallBuilderTuning
     
    +
    CallWrapper - Class in oracle.kubernetes.operator.calls
    +
    +
    A wrapper for an OKHttp call to isolate its own callers.
    +
    +
    CallWrapper(Call) - Constructor for class oracle.kubernetes.operator.calls.CallWrapper
    +
     
    +
    cancel() - Method in class oracle.kubernetes.operator.calls.CallWrapper
    +
     
    +
    cancel() - Method in interface oracle.kubernetes.operator.calls.CancellableCall
    +
    +
    Cancels the active call.
    +
    cancel(boolean) - Method in class oracle.kubernetes.operator.work.Fiber
    Marks this Fiber as cancelled.
    @@ -286,6 +378,10 @@

    C

    Cancels the current thread and accepts a callback for when the current thread, if any, exits processing this fiber.
    +
    CancellableCall - Interface in oracle.kubernetes.operator.calls
    +
    +
    An interface for an asynchronous API invocation that can be canceled
    +
    CANNOT_CREATE_NETWORK_POLICY - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
     
    CANNOT_CREATE_POD - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
    @@ -310,24 +406,22 @@

    C

    Check if the specified access token can be authenticated
    -
    check(String, AuthorizationProxy.Operation, AuthorizationProxy.Resource, String, AuthorizationProxy.Scope, String) - Method in class oracle.kubernetes.operator.helpers.AuthorizationProxy
    +
    check(String, List<String>, AuthorizationProxy.Operation, AuthorizationProxy.Resource, String, AuthorizationProxy.Scope, String) - Method in class oracle.kubernetes.operator.helpers.AuthorizationProxy
    Check if the specified principal is allowed to perform the specified operation on the specified resource in the specified scope.
    -
    check(String, List<String>, AuthorizationProxy.Operation, AuthorizationProxy.Resource, String, AuthorizationProxy.Scope, String) - Method in class oracle.kubernetes.operator.helpers.AuthorizationProxy
    +
    check(String, AuthorizationProxy.Operation, AuthorizationProxy.Resource, String, AuthorizationProxy.Scope, String) - Method in class oracle.kubernetes.operator.helpers.AuthorizationProxy
    Check if the specified principal is allowed to perform the specified operation on the specified resource in the specified scope.
    +
    check(AuthorizationProxy.Operation, AuthorizationProxy.Resource, String, AuthorizationProxy.Scope, String) - Method in class oracle.kubernetes.operator.helpers.AuthorizationProxy
    +
     
    checkAndCreateCustomResourceDefinition() - Static method in class oracle.kubernetes.operator.helpers.CRDHelper
    Validates and, if necessary, created domains.weblogic.oracle CRD.
    -
    checkFormatAnnotation(V1ObjectMeta) - Static method in class oracle.kubernetes.operator.helpers.AnnotationHelper
    -
    -
    Check the metadata object for the presence of an annotation matching the expected format version.
    -
    CLASS_INGRESS - Static variable in interface oracle.kubernetes.operator.KubernetesConstants
     
    CLASS_INGRESS_VALUE - Static variable in interface oracle.kubernetes.operator.KubernetesConstants
    @@ -338,6 +432,8 @@

    C

    Removes all of the mappings from this map.
    +
    ClientFactory - Interface in oracle.kubernetes.operator.helpers
    +
     
    ClientPool - Class in oracle.kubernetes.operator.helpers
     
    clone() - Method in class oracle.kubernetes.operator.work.Packet
    @@ -350,6 +446,8 @@

    C

    Close the ApiClient to make sure any open connection is cleaned up.
    +
    cluster - oracle.kubernetes.operator.helpers.AuthorizationProxy.Scope
    +
     
    CLUSTER_NAME - Static variable in interface oracle.kubernetes.operator.ProcessingConstants
     
    CLUSTER_SCAN - Static variable in interface oracle.kubernetes.operator.ProcessingConstants
    @@ -387,6 +485,10 @@

    C

    Construct a ClusterResource.
    +
    ClusterServicesStep - Class in oracle.kubernetes.operator.steps
    +
     
    +
    ClusterServicesStep(DomainPresenceInfo, Step) - Constructor for class oracle.kubernetes.operator.steps.ClusterServicesStep
    +
     
    ClustersResource - Class in oracle.kubernetes.operator.rest.resource
    ClustersResource is a jaxrs resource that implements the REST api for the @@ -396,12 +498,6 @@

    C

    Construct a ClustersResource.
    -
    ClusterStartup - Class in oracle.kubernetes.weblogic.domain.v1
    -
    -
    ClusterStarup describes the desired startup state and passed environment variables for a specific cluster.
    -
    -
    ClusterStartup() - Constructor for class oracle.kubernetes.weblogic.domain.v1.ClusterStartup
    -
     
    CM_CREATED - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
     
    CM_EXISTS - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
    @@ -414,6 +510,10 @@

    C

    CollectionModel() - Constructor for class oracle.kubernetes.operator.rest.model.CollectionModel
     
    +
    complete() - Method in class oracle.kubernetes.operator.helpers.DomainPresenceInfo
    +
    +
    Sets the last completion time to now
    +
    Component - Interface in oracle.kubernetes.operator.work
    Interface that allows components to hook up with each other.
    @@ -431,25 +531,25 @@

    C

    A hash table with weak keys, full concurrency of retrievals, and adjustable expected concurrency for updates.
    -
    ConcurrentWeakHashMap(int, float, int) - Constructor for class oracle.kubernetes.operator.utils.ConcurrentWeakHashMap
    -
    -
    Creates a new, empty map with the specified initial - capacity, load factor and concurrency level.
    -
    -
    ConcurrentWeakHashMap(int, float) - Constructor for class oracle.kubernetes.operator.utils.ConcurrentWeakHashMap
    +
    ConcurrentWeakHashMap() - Constructor for class oracle.kubernetes.operator.utils.ConcurrentWeakHashMap
    -
    Creates a new, empty map with the specified initial capacity - and load factor and with the default concurrencyLevel (16).
    +
    Creates a new, empty map with a default initial capacity (16), + load factor (0.75) and concurrencyLevel (16).
    ConcurrentWeakHashMap(int) - Constructor for class oracle.kubernetes.operator.utils.ConcurrentWeakHashMap
    Creates a new, empty map with the specified initial capacity, and with default load factor (0.75) and concurrencyLevel (16).
    -
    ConcurrentWeakHashMap() - Constructor for class oracle.kubernetes.operator.utils.ConcurrentWeakHashMap
    +
    ConcurrentWeakHashMap(int, float) - Constructor for class oracle.kubernetes.operator.utils.ConcurrentWeakHashMap
    -
    Creates a new, empty map with a default initial capacity (16), - load factor (0.75) and concurrencyLevel (16).
    +
    Creates a new, empty map with the specified initial capacity + and load factor and with the default concurrencyLevel (16).
    +
    +
    ConcurrentWeakHashMap(int, float, int) - Constructor for class oracle.kubernetes.operator.utils.ConcurrentWeakHashMap
    +
    +
    Creates a new, empty map with the specified initial + capacity, load factor and concurrency level.
    ConcurrentWeakHashMap(Map<? extends K, ? extends V>) - Constructor for class oracle.kubernetes.operator.utils.ConcurrentWeakHashMap
    @@ -467,6 +567,10 @@

    C

    Logs a message which accompanies a Throwable at the CONFIG level.
    +
    ConfigMapAfterStep - Class in oracle.kubernetes.operator.steps
    +
     
    +
    ConfigMapAfterStep(String, Map<String, ConfigMapWatcher>, AtomicBoolean, WatchListener<V1ConfigMap>, Step) - Constructor for class oracle.kubernetes.operator.steps.ConfigMapAfterStep
    +
     
    ConfigMapConsumer - Class in oracle.kubernetes.operator.helpers
    Kubernetes mounts ConfigMaps in the Pod's file-system as directories where the contained @@ -478,10 +582,17 @@

    C

     
    ConfigMapHelper.ScriptConfigMapStep - Class in oracle.kubernetes.operator.helpers
     
    +
    CONFIGMAPS - oracle.kubernetes.operator.helpers.AuthorizationProxy.Resource
    +
     
    ConfigMapWatcher - Class in oracle.kubernetes.operator
    This class handles ConfigMap watching.
    +
    ConfigUpdate - Interface in oracle.kubernetes.operator.wlsconfig
    +
    +
    Each ConfigUpdate contains a suggested WebLogic configuration update that is necessary + to make the WebLogic configuration to be compatible with the DomainSpec configuration.
    +
    CONFLICT - Static variable in class oracle.kubernetes.operator.helpers.CallBuilder
    HTTP status code for "Conflict"
    @@ -528,28 +639,38 @@

    C

    Helper class to ensure Domain CRD is created
    -
    create(ThreadFactory, String, String, WatchListener<V1ConfigMap>, AtomicBoolean) - Static method in class oracle.kubernetes.operator.ConfigMapWatcher
    -
     
    -
    create(ThreadFactory, String, String, WatchListener<Domain>, AtomicBoolean) - Static method in class oracle.kubernetes.operator.DomainWatcher
    +
    CRDS - oracle.kubernetes.operator.helpers.AuthorizationProxy.Resource
     
    -
    create(ThreadFactory, String, String, String, WatchListener<V1Event>, AtomicBoolean) - Static method in class oracle.kubernetes.operator.EventWatcher
    +
    create - oracle.kubernetes.operator.helpers.AuthorizationProxy.Operation
     
    create() - Method in class oracle.kubernetes.operator.helpers.CallBuilderFactory
     
    +
    create(String) - Static method in class oracle.kubernetes.operator.wlsconfig.WlsDomainConfig
    +
    +
    Create a new WlsDomainConfig object using the json result from the WLS REST call
    +
    +
    create(String, String, String) - Static method in class oracle.kubernetes.operator.wlsconfig.WlsRetriever
    +
    +
    Constructor.
    +
    +
    create(ThreadFactory, String, String, String, WatchListener<V1Event>, AtomicBoolean) - Static method in class oracle.kubernetes.operator.EventWatcher
    +
     
    create(ThreadFactory, String, String, WatchListener<V1beta1Ingress>, AtomicBoolean) - Static method in class oracle.kubernetes.operator.IngressWatcher
     
    +
    create(ThreadFactory, String, String, WatchListener<V1ConfigMap>, AtomicBoolean) - Static method in class oracle.kubernetes.operator.ConfigMapWatcher
    +
     
    create(ThreadFactory, String, String, WatchListener<V1Pod>, AtomicBoolean) - Static method in class oracle.kubernetes.operator.PodWatcher
    Factory for PodWatcher
    create(ThreadFactory, String, String, WatchListener<V1Service>, AtomicBoolean) - Static method in class oracle.kubernetes.operator.ServiceWatcher
     
    -
    create(String) - Static method in class oracle.kubernetes.operator.wlsconfig.WlsClusterConfig
    +
    create(ThreadFactory, String, String, WatchListener<Domain>, AtomicBoolean) - Static method in class oracle.kubernetes.operator.DomainWatcher
    +
     
    +
    CREATE_WEBLOGIC_DOMAIN_INPUTS_V1 - Static variable in interface oracle.kubernetes.operator.VersionConstants
    +
     
    +
    CREATE_WEBLOGIC_OPERATOR_INPUTS_V1 - Static variable in interface oracle.kubernetes.operator.VersionConstants
     
    -
    create(String, String, String) - Static method in class oracle.kubernetes.operator.wlsconfig.WlsRetriever
    -
    -
    Constructor.
    -
    createAdminPodStep(Step) - Static method in class oracle.kubernetes.operator.helpers.PodHelper
    Factory for Step that creates admin server pod
    @@ -558,14 +679,14 @@

    C

    Create authenticated HTTP client
    -
    createAuthenticatedClientForServer(String, String, Step) - Static method in class oracle.kubernetes.operator.http.HttpClient
    -
    -
    Asynchronous Step for creating an authenticated HTTP client targeted at a server instance
    -
    createAuthenticatedClientForServer(String, String) - Static method in class oracle.kubernetes.operator.http.HttpClient
    Create authenticated client specifically targeted at an admin server
    +
    createAuthenticatedClientForServer(String, String, Step) - Static method in class oracle.kubernetes.operator.http.HttpClient
    +
    +
    Asynchronous Step for creating an authenticated HTTP client targeted at a server instance
    +
    createAvailableStep(String, Step) - Static method in class oracle.kubernetes.operator.DomainStatusUpdater
    Asynchronous step to set Domain condition to Available
    @@ -593,10 +714,6 @@

    C

    Creates asynchronous step to create or verify Ingress for cluster
    -
    createConfigMap(String, V1ConfigMap) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    -
    -
    Create config map
    -
    createConfigMapAsync(String, V1ConfigMap, ResponseStep<V1ConfigMap>) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    Asynchronous step for creating config map
    @@ -605,14 +722,14 @@

    C

    Creates a web hook object to track config map calls
    +
    createCustomResourceDefinition(ApiClient, V1beta1CustomResourceDefinition, String) - Method in class oracle.kubernetes.operator.helpers.CallBuilder.SynchronousCallFactoryImpl
    +
     
    +
    createCustomResourceDefinition(ApiClient, V1beta1CustomResourceDefinition, String) - Method in interface oracle.kubernetes.operator.helpers.SynchronousCallFactory
    +
     
    createCustomResourceDefinition(V1beta1CustomResourceDefinition) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    Create custom resource definition
    -
    createCustomResourceDefinitionAsync(String, V1beta1CustomResourceDefinition, ResponseStep<V1beta1CustomResourceDefinition>) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    -
    -
    Asynchronous step for creating custom resource definition
    -
    CREATED_NETWORK_POLICY - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
     
    CREATED_SERVICE - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
    @@ -629,14 +746,6 @@

    C

    Asynchronous step to set Domain condition end Progressing and set Available, if needed
    -
    createEvent(String, V1Event) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    -
    -
    Create event
    -
    -
    createEventAsync(String, V1Service, ResponseStep<V1Event>) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    -
    -
    Asynchronous step for creating event
    -
    createEventWatch(String) - Method in class oracle.kubernetes.operator.builders.WatchBuilder
    Creates a web hook object to track events
    @@ -665,10 +774,6 @@

    C

    Create asynchronous step for internal cluster service
    -
    createIngress(String, V1beta1Ingress) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    -
    -
    Create ingress
    -
    createIngressAsync(String, V1beta1Ingress, ResponseStep<V1beta1Ingress>) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    Asynchronous step for creating ingress
    @@ -677,6 +782,10 @@

    C

    Creates a web hook object to track changes to the cluster ingress
    +
    createJobAsync(String, V1Job, ResponseStep<V1Job>) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    +
    +
    Asynchronous step for creating job
    +
    createManagedPodStep(Step) - Static method in class oracle.kubernetes.operator.helpers.PodHelper
    Factory for Step that creates managed server pod
    @@ -685,14 +794,6 @@

    C

    Create namespace
    -
    createNamespaceAsync(V1Namespace, ResponseStep<V1Namespace>) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    -
    -
    Asynchronous step for creating namespace
    -
    -
    createPod(String, V1Pod) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    -
    -
    Create pod
    -
    createPodAsync(String, V1Pod, ResponseStep<V1Pod>) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    Asynchronous step for creating pod
    @@ -705,6 +806,8 @@

    C

    Asynchronous step to set Domain condition to Progressing
    +
    createRequestAsync(ResponseStep<T>, RequestParams, CallFactory<T>, ClientPool, int, int, String, String, String) - Method in interface oracle.kubernetes.operator.helpers.AsyncRequestStepFactory
    +
     
    createScriptConfigMapStep(String, String, Step) - Static method in class oracle.kubernetes.operator.helpers.ConfigMapHelper
    Factory for Step that creates config map containing scripts
    @@ -713,13 +816,29 @@

    C

    Create secret
    -
    createServerStatusReaderStep(ServerKubernetesObjects, V1Pod, String, long, Step) - Static method in class oracle.kubernetes.operator.ServerStatusReader
    +
    createSelfSubjectAccessReview(V1SelfSubjectAccessReview) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    -
    Creates asynchronous step to read WebLogic server state from a particular pod
    +
    Create self subject access review
    +
    +
    createSelfSubjectAccessReviewAsync(V1SelfSubjectAccessReview, ResponseStep<V1SelfSubjectAccessReview>) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    +
    +
    Asynchronous step for creating self subject access review
    +
    +
    createSelfSubjectRulesReview(ApiClient, V1SelfSubjectRulesReview, String) - Method in class oracle.kubernetes.operator.helpers.CallBuilder.SynchronousCallFactoryImpl
    +
     
    +
    createSelfSubjectRulesReview(ApiClient, V1SelfSubjectRulesReview, String) - Method in interface oracle.kubernetes.operator.helpers.SynchronousCallFactory
    +
     
    +
    createSelfSubjectRulesReview(V1SelfSubjectRulesReview) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    +
    +
    Create self subject rules review
    +
    +
    createSelfSubjectRulesReviewAsync(V1SelfSubjectRulesReview, ResponseStep<V1SelfSubjectRulesReview>) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    +
    +
    Asynchronous step for creating self subject rules review
    -
    createService(String, V1Service) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    +
    createServerStatusReaderStep(ServerKubernetesObjects, V1Pod, String, long, Step) - Static method in class oracle.kubernetes.operator.ServerStatusReader
    -
    Create service
    +
    Creates asynchronous step to read WebLogic server state from a particular pod
    createServiceAsync(String, V1Service, ResponseStep<V1Service>) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    @@ -733,6 +852,10 @@

    C

    Asynchronous step to set Domain status to indicate WebLogic server status
    +
    createStep(Step) - Method in interface oracle.kubernetes.operator.wlsconfig.ConfigUpdate
    +
    +
    Create a Step to perform the suggested WebLogic configuration update
    +
    createStepline(List<Class<? extends Step>>) - Static method in class oracle.kubernetes.operator.work.Step
    Simplifies creation of stepline.
    @@ -749,34 +872,16 @@

    C

    Create token review
    -
    createTokenReviewAsync(V1TokenReview, ResponseStep<V1TokenReview>) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    -
    -
    Asynchronous step for creating token review
    -
    createWatch(Pool<ApiClient>, CallParams, Class<?>, BiFunction<ApiClient, CallParams, Call>) - Method in interface oracle.kubernetes.operator.builders.WatchBuilder.WatchFactory
     
    -
    createWebLogicOracleV1NamespacedDomain(String, Domain, String) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    create a Domain
    -
    -
    createWebLogicOracleV1NamespacedDomainAsync(String, Domain, String, ApiCallback<Domain>) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    (asynchronously) create a Domain
    -
    -
    createWebLogicOracleV1NamespacedDomainCall(String, Domain, String, ProgressResponseBody.ProgressListener, ProgressRequestBody.ProgressRequestListener) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    Build call for createWebLogicOracleV1NamespacedDomain
    -
    -
    createWebLogicOracleV1NamespacedDomainWithHttpInfo(String, Domain, String) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    create a Domain
    -
    CREATING_API_CLIENT - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
     
    CREATING_CRD - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
     
    CREATING_SERVICE - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
     
    +
    CRONJOBS - oracle.kubernetes.operator.helpers.AuthorizationProxy.Resource
    +
     
    CSRF_PROTECTION_FILTER_PRIORITY - Static variable in class oracle.kubernetes.operator.rest.FilterPriorities
    The CSRF protection filter's priority.
    @@ -800,30 +905,18 @@

    D

    Indicates that the fiber should be suspended for the indicated delay duration and then automatically resumed.
    -
    deleteCollectionPod(String) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    -
    -
    Delete collection of pods
    -
    +
    delete - oracle.kubernetes.operator.helpers.AuthorizationProxy.Operation
    +
     
    +
    deletecollection - oracle.kubernetes.operator.helpers.AuthorizationProxy.Operation
    +
     
    deleteCollectionPodAsync(String, ResponseStep<V1Status>) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    Asynchronous step for deleting collection of pods
    -
    deleteConfigMap(String, String, V1DeleteOptions) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    -
    -
    Delete config map
    -
    -
    deleteConfigMapAsync(String, String, ResponseStep<V1Status>) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    -
    -
    Asynchronous step for deleting config map
    -
    -
    deleteEvent(String, String, V1DeleteOptions) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    -
    -
    Delete event
    -
    -
    deleteEventAsync(String, String, ResponseStep<V1Status>) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    -
    -
    Asynchronous step for deleting event
    -
    +
    DeleteDomainStep - Class in oracle.kubernetes.operator.steps
    +
     
    +
    DeleteDomainStep(String, String) - Constructor for class oracle.kubernetes.operator.steps.DeleteDomainStep
    +
     
    deleteIngress(String, String, V1DeleteOptions) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    Delete ingress
    @@ -832,17 +925,13 @@

    D

    Asynchronous step for deleting ingress
    -
    deleteNamespace(String, V1DeleteOptions) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    -
    -
    Delete namespace
    -
    -
    deleteNamespaceAsync(String, ResponseStep<V1Status>) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    -
    -
    Asynchronous step for deleting service
    -
    -
    deletePod(String, String, V1DeleteOptions) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    +
    DeleteIngressListStep - Class in oracle.kubernetes.operator.steps
    +
     
    +
    DeleteIngressListStep(Collection<V1beta1Ingress>, Step) - Constructor for class oracle.kubernetes.operator.steps.DeleteIngressListStep
    +
     
    +
    deleteJobAsync(String, String, ResponseStep<V1Status>) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    -
    Delete pod
    +
    Asynchronous step for deleting job
    deletePodAsync(String, String, V1DeleteOptions, ResponseStep<V1Status>) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    @@ -864,56 +953,20 @@

    D

    Asynchronous step for deleting service
    +
    DeleteServiceListStep - Class in oracle.kubernetes.operator.steps
    +
     
    +
    DeleteServiceListStep(Collection<V1Service>, Step) - Constructor for class oracle.kubernetes.operator.steps.DeleteServiceListStep
    +
     
    deleteServiceStep(ServerKubernetesObjects, Step) - Static method in class oracle.kubernetes.operator.helpers.ServiceHelper
    Factory for Step that deletes per-managed server service
    -
    deleteWebLogicOracleV1CollectionNamespacedDomain(String, String, String, String, Boolean, String, Integer, String, Integer, Boolean) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    delete collection of Domain
    -
    -
    deleteWebLogicOracleV1CollectionNamespacedDomainAsync(String, String, String, String, Boolean, String, Integer, String, Integer, Boolean, ApiCallback<V1Status>) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    (asynchronously) delete collection of Domain
    -
    -
    deleteWebLogicOracleV1CollectionNamespacedDomainCall(String, String, String, String, Boolean, String, Integer, String, Integer, Boolean, ProgressResponseBody.ProgressListener, ProgressRequestBody.ProgressRequestListener) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    Build call for deleteWebLogicOracleV1CollectionNamespacedDomain
    -
    -
    deleteWebLogicOracleV1CollectionNamespacedDomainWithHttpInfo(String, String, String, String, Boolean, String, Integer, String, Integer, Boolean) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    delete collection of Domain
    -
    -
    deleteWebLogicOracleV1NamespacedDomain(String, String, V1DeleteOptions, String, Integer, Boolean, String) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    delete a Domain
    -
    -
    deleteWebLogicOracleV1NamespacedDomainAsync(String, String, V1DeleteOptions, String, Integer, Boolean, String, ApiCallback<V1Status>) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    (asynchronously) delete a Domain
    -
    -
    deleteWebLogicOracleV1NamespacedDomainCall(String, String, V1DeleteOptions, String, Integer, Boolean, String, ProgressResponseBody.ProgressListener, ProgressRequestBody.ProgressRequestListener) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    Build call for deleteWebLogicOracleV1NamespacedDomain
    -
    -
    deleteWebLogicOracleV1NamespacedDomainWithHttpInfo(String, String, V1DeleteOptions, String, Integer, Boolean, String) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    delete a Domain
    -
    -
    Domain - Class in oracle.kubernetes.weblogic.domain.v1
    -
    -
    Domain represents a WebLogic domain and how it will be realized in the Kubernetes cluster.
    -
    -
    Domain() - Constructor for class oracle.kubernetes.weblogic.domain.v1.Domain
    -
     
    DOMAIN_COMPONENT_NAME - Static variable in interface oracle.kubernetes.operator.ProcessingConstants
     
    DOMAIN_CONFIG_MAP_NAME - Static variable in interface oracle.kubernetes.operator.KubernetesConstants
     
    DOMAIN_GROUP - Static variable in interface oracle.kubernetes.operator.KubernetesConstants
     
    -
    DOMAIN_IMAGE_FAILED - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
    -
     
    DOMAIN_PLURAL - Static variable in interface oracle.kubernetes.operator.KubernetesConstants
     
    DOMAIN_REPLICAS_IGNORED - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
    @@ -926,19 +979,9 @@

    D

     
    DOMAIN_UID_UNIQUENESS_FAILED - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
     
    -
    DOMAIN_VERSION - Static variable in interface oracle.kubernetes.operator.KubernetesConstants
    -
     
    -
    DomainCondition - Class in oracle.kubernetes.weblogic.domain.v1
    -
    -
    DomainCondition contains details for the current condition of this domain.
    -
    -
    DomainCondition() - Constructor for class oracle.kubernetes.weblogic.domain.v1.DomainCondition
    +
    DOMAIN_V1 - Static variable in interface oracle.kubernetes.operator.VersionConstants
     
    -
    DomainList - Class in oracle.kubernetes.weblogic.domain.v1
    -
    -
    DomainList is a list of Domains.
    -
    -
    DomainList() - Constructor for class oracle.kubernetes.weblogic.domain.v1.DomainList
    +
    DOMAIN_VERSION - Static variable in interface oracle.kubernetes.operator.KubernetesConstants
     
    DomainModel - Class in oracle.kubernetes.operator.rest.model
    @@ -955,16 +998,22 @@

    D

    DOMAINNAME_LABEL - Static variable in interface oracle.kubernetes.operator.LabelConstants
     
    +
    DomainPrescenceStep - Class in oracle.kubernetes.operator.steps
    +
     
    +
    DomainPrescenceStep(Step, Step) - Constructor for class oracle.kubernetes.operator.steps.DomainPrescenceStep
    +
     
    +
    domainPresenceFailureRetrySeconds - Variable in class oracle.kubernetes.operator.TuningParameters.MainTuning
    +
     
    DomainPresenceInfo - Class in oracle.kubernetes.operator.helpers
    Operator's mapping between custom resource Domain and runtime details about that domain, including the scan and the Pods and Services for servers.
    -
    DomainPresenceInfo(Domain) - Constructor for class oracle.kubernetes.operator.helpers.DomainPresenceInfo
    +
    DomainPresenceInfo(String) - Constructor for class oracle.kubernetes.operator.helpers.DomainPresenceInfo
    Create presence for a domain
    -
    DomainPresenceInfo(String) - Constructor for class oracle.kubernetes.operator.helpers.DomainPresenceInfo
    +
    DomainPresenceInfo(Domain) - Constructor for class oracle.kubernetes.operator.helpers.DomainPresenceInfo
    Create presence for a domain
    @@ -972,6 +1021,8 @@

    D

    Details about a specific managed server that will be started up
    +
    domainPresenceRecheckIntervalSeconds - Variable in class oracle.kubernetes.operator.TuningParameters.MainTuning
    +
     
    DomainResource - Class in oracle.kubernetes.operator.rest.resource
    DomainResource is a jaxrs resource that implements the REST api for the @@ -981,11 +1032,7 @@

    D

    Construct a DomainResource.
    -
    DomainSpec - Class in oracle.kubernetes.weblogic.domain.v1
    -
    -
    DomainSpec is a description of a domain.
    -
    -
    DomainSpec() - Constructor for class oracle.kubernetes.weblogic.domain.v1.DomainSpec
    +
    DOMAINS - oracle.kubernetes.operator.helpers.AuthorizationProxy.Resource
     
    DomainsResource - Class in oracle.kubernetes.operator.rest.resource
    @@ -996,11 +1043,7 @@

    D

    Construct a DomainsResource.
    -
    DomainStatus - Class in oracle.kubernetes.weblogic.domain.v1
    -
    -
    DomainStatus represents information about the status of a domain.
    -
    -
    DomainStatus() - Constructor for class oracle.kubernetes.weblogic.domain.v1.DomainStatus
    +
    DOMAINSTATUSS - oracle.kubernetes.operator.helpers.AuthorizationProxy.Resource
     
    DomainStatusUpdater - Class in oracle.kubernetes.operator
    @@ -1012,7 +1055,7 @@

    D

    This class handles Domain watching.
    -
    doPotentialRetry(Step, Packet, ApiException, int, Map<String, List<String>>) - Method in interface oracle.kubernetes.operator.helpers.CallBuilder.RetryStrategy
    +
    doPotentialRetry(Step, Packet, ApiException, int, Map<String, List<String>>) - Method in interface oracle.kubernetes.operator.calls.RetryStrategy
    Called during ResponseStep.onFailure(Packet, ApiException, int, Map) to decide if another retry attempt will occur.
    @@ -1029,12 +1072,18 @@

    D

    Restarts the listed servers, if already running.
    +
    drain() - Method in class oracle.kubernetes.operator.helpers.Pool
    +
    +
    Drains pool of all entries; useful for unit-testing
    +

    E

    +
    e - Variable in class oracle.kubernetes.operator.calls.CallResponse
    +
     
    elements() - Method in class oracle.kubernetes.operator.utils.ConcurrentWeakHashMap
    Returns an enumeration of the values in this table.
    @@ -1043,14 +1092,14 @@

    E

    Collection of Fibers.
    -
    Engine(ScheduledExecutorService) - Constructor for class oracle.kubernetes.operator.work.Engine
    -
    -
    Creates engine with the specified executor
    -
    Engine(String) - Constructor for class oracle.kubernetes.operator.work.Engine
    Creates engine with the specified id and default container and executor
    +
    Engine(ScheduledExecutorService) - Constructor for class oracle.kubernetes.operator.work.Engine
    +
    +
    Creates engine with the specified executor
    +
    enterContainer(Container) - Method in class oracle.kubernetes.operator.work.ThreadLocalContainerResolver
    Enters container
    @@ -1067,7 +1116,7 @@

    E

     
    entrySet() - Method in class oracle.kubernetes.operator.utils.ConcurrentWeakHashMap
    -
    Returns a Set view of the mappings contained in this map.
    +
    Returns a Set view of the mappings contained in this map.
    entrySet() - Method in class oracle.kubernetes.operator.work.Packet
     
    @@ -1075,26 +1124,6 @@

    E

     
    ENVVARS - Static variable in interface oracle.kubernetes.operator.ProcessingConstants
     
    -
    equals(Object) - Method in class oracle.kubernetes.weblogic.domain.v1.ClusterStartup
    -
     
    -
    equals(Object) - Method in class oracle.kubernetes.weblogic.domain.v1.Domain
    -
     
    -
    equals(Object) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainCondition
    -
     
    -
    equals(Object) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainList
    -
     
    -
    equals(Object) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainSpec
    -
     
    -
    equals(Object) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainStatus
    -
     
    -
    equals(Object) - Method in class oracle.kubernetes.weblogic.domain.v1.ServerHealth
    -
     
    -
    equals(Object) - Method in class oracle.kubernetes.weblogic.domain.v1.ServerStartup
    -
     
    -
    equals(Object) - Method in class oracle.kubernetes.weblogic.domain.v1.ServerStatus
    -
     
    -
    equals(Object) - Method in class oracle.kubernetes.weblogic.domain.v1.SubsystemHealth
    -
     
    ERROR_FILTER_PRIORITY - Static variable in class oracle.kubernetes.operator.rest.FilterPriorities
    The error filter's priority.
    @@ -1118,6 +1147,8 @@

    E

    Construct a populated ErrorModel.
    +
    EVENTS - oracle.kubernetes.operator.helpers.AuthorizationProxy.Resource
    +
     
    eventualLongDelay - Variable in class oracle.kubernetes.operator.TuningParameters.MainTuning
     
    EventWatcher - Class in oracle.kubernetes.operator
    @@ -1145,6 +1176,8 @@

    E

     
    EXCH_WRONG_PROTOCOL - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
     
    +
    EXEC - oracle.kubernetes.operator.helpers.AuthorizationProxy.Resource
    +
     
    executeGetOnServiceClusterIP(String, String) - Method in class oracle.kubernetes.operator.http.HttpClient
    Constructs a URL using the provided service URL and request URL, and use the resulting URL to issue a HTTP GET request
    @@ -1171,15 +1204,17 @@

    E

    Logs a method exit, with a result object.
    -
    EXPLICIT_RESTART_ADMIN - Static variable in interface oracle.kubernetes.operator.ProcessingConstants
    +
    export - Variable in class oracle.kubernetes.operator.helpers.CallBuilder
     
    -
    EXPLICIT_RESTART_CLUSTERS - Static variable in interface oracle.kubernetes.operator.ProcessingConstants
    +
    EXTENSIONS_API_VERSION - Static variable in interface oracle.kubernetes.operator.KubernetesConstants
     
    -
    EXPLICIT_RESTART_SERVERS - Static variable in interface oracle.kubernetes.operator.ProcessingConstants
    +
    ExternalAdminChannelIteratorStep - Class in oracle.kubernetes.operator.steps
     
    -
    export - Variable in class oracle.kubernetes.operator.helpers.CallBuilder
    +
    ExternalAdminChannelIteratorStep(DomainPresenceInfo, Collection<NetworkAccessPoint>, Step) - Constructor for class oracle.kubernetes.operator.steps.ExternalAdminChannelIteratorStep
     
    -
    EXTENSIONS_API_VERSION - Static variable in interface oracle.kubernetes.operator.KubernetesConstants
    +
    ExternalAdminChannelsStep - Class in oracle.kubernetes.operator.steps
    +
     
    +
    ExternalAdminChannelsStep(Step) - Constructor for class oracle.kubernetes.operator.steps.ExternalAdminChannelsStep
     
    @@ -1193,6 +1228,8 @@

    F

    User-level thread. Represents the execution of one processing flow.
    +
    FIBER_COMPONENT_NAME - Static variable in interface oracle.kubernetes.operator.ProcessingConstants
    +
     
    Fiber.CompletionCallback - Interface in oracle.kubernetes.operator.work
    Callback to be invoked when a Fiber finishes execution.
    @@ -1201,8 +1238,6 @@

    F

    Callback invoked when a Thread exits processing this fiber
    -
    FIBER_COMPONENT_NAME - Static variable in interface oracle.kubernetes.operator.ProcessingConstants
    -
     
    FiberGate - Class in oracle.kubernetes.operator.work
    Allows at most one running Fiber per key value.
    @@ -1214,8 +1249,8 @@

    F

    fieldSelector - Variable in class oracle.kubernetes.operator.helpers.CallBuilder
     
    filter(ContainerRequestContext) - Method in class oracle.kubernetes.operator.rest.AuthenticationFilter
    -
    filter(ContainerRequestContext, ContainerResponseContext) - Method in class oracle.kubernetes.operator.rest.ErrorFilter
    filter(ContainerRequestContext) - Method in class oracle.kubernetes.operator.rest.RequestDebugLoggingFilter
    +
    filter(ContainerRequestContext, ContainerResponseContext) - Method in class oracle.kubernetes.operator.rest.ErrorFilter
    filter(ContainerRequestContext, ContainerResponseContext) - Method in class oracle.kubernetes.operator.rest.ResponseDebugLoggingFilter
    FilterPriorities - Class in oracle.kubernetes.operator.rest
    @@ -1261,17 +1296,15 @@

    F

     
    format(LogRecord) - Method in class oracle.kubernetes.operator.logging.LoggingFormatter
     
    -
    FORMAT_ANNOTATION - Static variable in class oracle.kubernetes.operator.helpers.AnnotationHelper
    -
     
    -
    FORMAT_VERSION - Static variable in class oracle.kubernetes.operator.helpers.AnnotationHelper
    -
     

    G

    -
    get(Object) - Method in class oracle.kubernetes.operator.helpers.ConfigMapConsumer
    +
    generate(RequestParams, ApiClient, String, ApiCallback<T>) - Method in interface oracle.kubernetes.operator.calls.CallFactory
    +
     
    +
    get - oracle.kubernetes.operator.helpers.AuthorizationProxy.Operation
     
    get() - Method in class oracle.kubernetes.operator.rest.resource.ClusterResource
    @@ -1301,49 +1334,27 @@

    G

    List the supported versions of the WebLogic operator REST api.
    -
    get(Object) - Method in class oracle.kubernetes.operator.utils.ConcurrentWeakHashMap
    -
    -
    Returns the value to which the specified key is mapped, - or null if this map contains no mapping for the key.
    -
    get() - Method in class oracle.kubernetes.operator.work.Fiber
     
    get(long, TimeUnit) - Method in class oracle.kubernetes.operator.work.Fiber
     
    -
    getActivationTime() - Method in class oracle.kubernetes.weblogic.domain.v1.ServerHealth
    +
    get(Object) - Method in class oracle.kubernetes.operator.helpers.ConfigMapConsumer
    +
     
    +
    get(Object) - Method in class oracle.kubernetes.operator.utils.ConcurrentWeakHashMap
    -
    RFC 3339 date and time at which the server started.
    +
    Returns the value to which the specified key is mapped, + or null if this map contains no mapping for the key.
    getAdmin() - Method in class oracle.kubernetes.operator.helpers.DomainPresenceInfo
    Server objects (Pods and Services) for admin server
    -
    getAdminSecret() - Method in class oracle.kubernetes.weblogic.domain.v1.DomainSpec
    -
    -
    Reference to secret containing domain administrator username and password.
    -
    getApiClient() - Method in class oracle.kubernetes.operator.authentication.Authenticator
    Get the API client.
    -
    getApiClient() - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    +
    getAPIGroup() - Method in enum oracle.kubernetes.operator.helpers.AuthorizationProxy.Resource
     
    -
    getApiVersion() - Method in class oracle.kubernetes.weblogic.domain.v1.Domain
    -
    -
    APIVersion defines the versioned schema of this representation of an object.
    -
    -
    getApiVersion() - Method in class oracle.kubernetes.weblogic.domain.v1.DomainList
    -
    -
    APIVersion defines the versioned schema of this representation of an object.
    -
    -
    getAsName() - Method in class oracle.kubernetes.weblogic.domain.v1.DomainSpec
    -
    -
    Admin server name.
    -
    -
    getAsPort() - Method in class oracle.kubernetes.weblogic.domain.v1.DomainSpec
    -
    -
    Administration server port.
    -
    getBackend(String) - Method in interface oracle.kubernetes.operator.rest.RestConfig
    Gets a RestBackend instance that does the real work @@ -1382,14 +1393,6 @@

    G

    Returns the name of the cluster that this WlsClusterConfig is created for
    -
    getClusterName() - Method in class oracle.kubernetes.weblogic.domain.v1.ClusterStartup
    -
    -
    Name of specific cluster to start.
    -
    -
    getClusterName() - Method in class oracle.kubernetes.weblogic.domain.v1.ServerStatus
    -
    -
    WebLogic cluster name, if the server is part of a cluster
    -
    getClusterResource(String) - Method in class oracle.kubernetes.operator.rest.resource.ClustersResource
    Construct and return a 'cluster' jaxrs child resource.
    @@ -1408,16 +1411,12 @@

    G

    getClusterSize() - Method in class oracle.kubernetes.operator.wlsconfig.WlsClusterConfig
    -
    Returns the number of servers configured in this cluster
    +
    Returns the number of servers that are statically configured in this cluster
    getClustersResource() - Method in class oracle.kubernetes.operator.rest.resource.DomainResource
    Construct and return the 'clusters' jaxrs child resource.
    -
    getClusterStartup() - Method in class oracle.kubernetes.weblogic.domain.v1.DomainSpec
    -
    -
    List of server startup details for selected clusters
    -
    getCompletionCallback() - Method in class oracle.kubernetes.operator.work.Fiber
    Returns completion callback associated with this Fiber
    @@ -1432,10 +1431,6 @@

    G

     
    getComponents() - Method in class oracle.kubernetes.operator.work.Packet
     
    -
    getConditions() - Method in class oracle.kubernetes.weblogic.domain.v1.DomainStatus
    -
    -
    Current service state of domain.
    -
    getContainer() - Method in class oracle.kubernetes.operator.work.ContainerResolver
    Returns the Container context in which client is running.
    @@ -1444,7 +1439,7 @@

    G

     
    getContextClassLoader() - Method in class oracle.kubernetes.operator.work.Fiber
    -
    Gets the context ClassLoader of this fiber.
    +
    Gets the context ClassLoader of this fiber.
    getCurrentIfSet() - Static method in class oracle.kubernetes.operator.work.Fiber
    @@ -1455,14 +1450,6 @@

    G

    Returns the default container resolver which can be used to get Container.
    -
    getDesiredState() - Method in class oracle.kubernetes.weblogic.domain.v1.ClusterStartup
    -
    -
    Desired startup state for any managed server started in this cluster.
    -
    -
    getDesiredState() - Method in class oracle.kubernetes.weblogic.domain.v1.ServerStartup
    -
    -
    Desired startup state.
    -
    getDetail() - Method in class oracle.kubernetes.operator.rest.model.ErrorModel
    Get a detailed description of the error.
    @@ -1471,11 +1458,10 @@

    G

    Gets the domain.
    -
    getDomainName() - Method in class oracle.kubernetes.weblogic.domain.v1.DomainSpec
    -
    -
    Domain name - (Required)
    -
    +
    getDomainList(ApiClient, String, String, String, String, Boolean, String, Integer, String, Integer, Boolean) - Method in class oracle.kubernetes.operator.helpers.CallBuilder.SynchronousCallFactoryImpl
    +
     
    +
    getDomainList(ApiClient, String, String, String, String, Boolean, String, Integer, String, Integer, Boolean) - Method in interface oracle.kubernetes.operator.helpers.SynchronousCallFactory
    +
     
    getDomainResource(String) - Method in class oracle.kubernetes.operator.rest.resource.DomainsResource
    Construct and return a 'domain' jaxrs child resource.
    @@ -1488,10 +1474,6 @@

    G

    Get the unique identifier that has been assigned to this WebLogic domain.
    -
    getDomainUID() - Method in class oracle.kubernetes.weblogic.domain.v1.DomainSpec
    -
    -
    Domain unique identifier.
    -
    getDomainUIDs() - Method in interface oracle.kubernetes.operator.rest.backend.RestBackend
    Get the unique identifiers of all the WebLogic domains that have been registered @@ -1502,21 +1484,30 @@

    G

    Get the unique identifiers of all the WebLogic domains that have been registered with the WebLogic operator.
    -
    getEnv() - Method in class oracle.kubernetes.weblogic.domain.v1.ClusterStartup
    +
    getDynamicClusterSize() - Method in class oracle.kubernetes.operator.wlsconfig.WlsClusterConfig
    -
    Environment variables to pass while starting managed servers in this cluster.
    +
    Returns the current size of the dynamic cluster (the number of dynamic server instances allowed + to be created)
    -
    getEnv() - Method in class oracle.kubernetes.weblogic.domain.v1.ServerStartup
    +
    getDynamicClusterSize() - Method in class oracle.kubernetes.operator.wlsconfig.WlsDynamicServersConfig
    -
    Environment variables to pass while starting this managed server.
    +
    Return current size of the dynamic cluster
    getExecutor() - Method in class oracle.kubernetes.operator.work.Engine
    Returns the executor
    -
    getExportT3Channels() - Method in class oracle.kubernetes.weblogic.domain.v1.DomainSpec
    +
    getExplicitRestartAdmin() - Method in class oracle.kubernetes.operator.helpers.DomainPresenceInfo
    +
    +
    Control for if domain has outstanding restart admin server pending
    +
    +
    getExplicitRestartClusters() - Method in class oracle.kubernetes.operator.helpers.DomainPresenceInfo
    +
    +
    Control list for outstanding cluster restarts
    +
    +
    getExplicitRestartServers() - Method in class oracle.kubernetes.operator.helpers.DomainPresenceInfo
    -
    List of specific T3 channels to export.
    +
    Control list for outstanding server restarts
    getExternalHttpsPort() - Method in interface oracle.kubernetes.operator.rest.RestConfig
    @@ -1534,14 +1525,6 @@

    G

    Returns a selector to limit results to those with matching fields.
    -
    getHealth() - Method in class oracle.kubernetes.weblogic.domain.v1.ServerStatus
    -
    -
    ServerHealth describes the current status and health of a specific WebLogic server.
    -
    -
    getHealth() - Method in class oracle.kubernetes.weblogic.domain.v1.SubsystemHealth
    -
    -
    Server health of this WebLogic server.
    -
    getHelper() - Method in class oracle.kubernetes.operator.authentication.Authenticator
    Get a reference to ServiceHelper.
    @@ -1558,14 +1541,6 @@

    G

    Get the link's hypertext reference.
    -
    getImage() - Method in class oracle.kubernetes.weblogic.domain.v1.DomainSpec
    -
    -
    WebLogic Docker image.
    -
    -
    getImagePullPolicy() - Method in class oracle.kubernetes.weblogic.domain.v1.DomainSpec
    -
    -
    Image pull policy.
    -
    getIncludeUninitialized() - Method in interface oracle.kubernetes.operator.builders.CallParams
    Returns a boolean indicating whether partially initialized results should be included in the response.
    @@ -1576,6 +1551,10 @@

    G

    getInstance() - Static method in class oracle.kubernetes.operator.helpers.ClientPool
     
    +
    getInstance() - Static method in interface oracle.kubernetes.operator.TuningParameters
    +
     
    +
    getInstance() - Static method in class oracle.kubernetes.operator.TuningParametersImpl
    +
     
    getInstance() - Static method in class oracle.kubernetes.operator.work.ContainerResolver
    Returns the container resolver which can be used to get client's @@ -1597,58 +1576,42 @@

    G

    Get the items in the collection.
    -
    getItems() - Method in class oracle.kubernetes.weblogic.domain.v1.DomainList
    -
    -
    List of domains.
    -
    -
    getIterableSPI(Class<S>) - Method in interface oracle.kubernetes.operator.work.ComponentEx
    -
    -
    Gets an iterator of implementations of the specified SPI.
    -
    getIterableSPI(Class<E>) - Method in class oracle.kubernetes.operator.work.Container
     
    getIterableSPI(Class<E>) - Method in class oracle.kubernetes.operator.work.Packet
     
    -
    getKind() - Method in class oracle.kubernetes.weblogic.domain.v1.Domain
    -
    -
    Kind is a string value representing the REST resource this object represents.
    -
    -
    getKind() - Method in class oracle.kubernetes.weblogic.domain.v1.DomainList
    +
    getIterableSPI(Class<S>) - Method in interface oracle.kubernetes.operator.work.ComponentEx
    -
    Kind is a string value representing the REST resource this object represents.
    +
    Gets an iterator of implementations of the specified SPI.
    getLabelSelector() - Method in interface oracle.kubernetes.operator.builders.CallParams
    Returns a selector to limit results to those with matching labels.
    -
    getLastKnownStatus() - Method in class oracle.kubernetes.operator.helpers.ServerKubernetesObjects
    +
    getLastCompletionTime() - Method in class oracle.kubernetes.operator.helpers.DomainPresenceInfo
    -
    Managed server status
    +
    Last completion time
    -
    getLastProbeTime() - Method in class oracle.kubernetes.weblogic.domain.v1.DomainCondition
    +
    getLastKnownStatus() - Method in class oracle.kubernetes.operator.helpers.ServerKubernetesObjects
    -
    Last time we probed the condition.
    +
    Managed server status
    getLastScanTime() - Method in class oracle.kubernetes.operator.helpers.DomainPresenceInfo
    Last scan time
    -
    getLastTransitionTime() - Method in class oracle.kubernetes.weblogic.domain.v1.DomainCondition
    -
    -
    Last time the condition transitioned from one status to another.
    -
    getLevel() - Method in class oracle.kubernetes.operator.logging.LoggingFacade
    Returns the level at which the underlying logger operates.
    -
    getLifecycle(String) - Static method in class oracle.kubernetes.operator.rest.backend.VersionUtils
    -
    -
    Gets the lifecycle of a version.
    -
    getLifecycle() - Method in class oracle.kubernetes.operator.rest.model.VersionModel
    Get the lifecycle of this version of the WebLogic operator REST api.
    +
    getLifecycle(String) - Static method in class oracle.kubernetes.operator.rest.backend.VersionUtils
    +
    +
    Gets the lifecycle of a version.
    +
    getLimit() - Method in interface oracle.kubernetes.operator.builders.CallParams
    Returns the limit on the number of updates to send in a single reply.
    @@ -1658,15 +1621,35 @@

    G

    Get the links.
    getListenAddress() - Method in class oracle.kubernetes.operator.wlsconfig.WlsServerConfig
    -
     
    +
    +
    Return the configured listen address of this WLS server
    +
    getListenPort() - Method in class oracle.kubernetes.operator.wlsconfig.NetworkAccessPoint
     
    getListenPort() - Method in class oracle.kubernetes.operator.wlsconfig.WlsServerConfig
    -
     
    +
    +
    Return the configured listen port of this WLS server
    +
    getLogger(String, String) - Static method in class oracle.kubernetes.operator.logging.LoggingFactory
    Obtains a Logger from the underlying logging implementation and wraps it in a LoggingFacade.
    +
    getMachineConfig(String) - Method in class oracle.kubernetes.operator.wlsconfig.WlsDomainConfig
    +
    +
    Returns the configuration for the WLS machine with the given name.
    +
    +
    getMachineConfigs() - Method in class oracle.kubernetes.operator.wlsconfig.WlsDomainConfig
    +
    +
    Returns configuration of machines found in the WLS domain.
    +
    +
    getMachineName() - Method in class oracle.kubernetes.operator.wlsconfig.WlsServerConfig
    +
    +
    Return the machine name configured for this WLS server
    +
    +
    getMachineNameMatchExpression() - Method in class oracle.kubernetes.operator.wlsconfig.WlsDynamicServersConfig
    +
    +
    Return the expression used in matching machine names assigned to dynamic servers
    +
    getMainTuning() - Method in interface oracle.kubernetes.operator.TuningParameters
     
    getMainTuning() - Method in class oracle.kubernetes.operator.TuningParametersImpl
    @@ -1675,21 +1658,13 @@

    G

    Get the desired number of managed servers in the WebLogic cluster.
    -
    getMessage() - Method in class oracle.kubernetes.weblogic.domain.v1.DomainCondition
    +
    getMaxDynamicClusterSize() - Method in class oracle.kubernetes.operator.wlsconfig.WlsClusterConfig
    -
    Human-readable message indicating details about last transition.
    +
    Returns the maximum size of the dynamic cluster
    -
    getMessage() - Method in class oracle.kubernetes.weblogic.domain.v1.DomainStatus
    +
    getMaxDynamicClusterSize() - Method in class oracle.kubernetes.operator.wlsconfig.WlsDynamicServersConfig
    -
    A human readable message indicating details about why the domain is in this condition.
    -
    -
    getMetadata() - Method in class oracle.kubernetes.weblogic.domain.v1.Domain
    -
    -
    Standard object's metadata.
    -
    -
    getMetadata() - Method in class oracle.kubernetes.weblogic.domain.v1.DomainList
    -
    -
    Standard list metadata.
    +
    Return maximum size of the dynamic cluster
    getName() - Method in class oracle.kubernetes.operator.logging.LoggingFacade
    @@ -1697,8 +1672,16 @@

    G

    getName() - Method in class oracle.kubernetes.operator.wlsconfig.NetworkAccessPoint
     
    -
    getName() - Method in class oracle.kubernetes.operator.wlsconfig.WlsServerConfig
    +
    getName() - Method in class oracle.kubernetes.operator.wlsconfig.WlsDomainConfig
    +
    +
    Return the name of the WLS domain
    +
    +
    getName() - Method in class oracle.kubernetes.operator.wlsconfig.WlsMachineConfig
     
    +
    getName() - Method in class oracle.kubernetes.operator.wlsconfig.WlsServerConfig
    +
    +
    Return the name of this WLS server
    +
    getNamespace() - Method in class oracle.kubernetes.operator.helpers.DomainPresenceInfo
    Gets the namespace
    @@ -1711,14 +1694,12 @@

    G

    Returns the next step
    -
    getNodeName() - Method in class oracle.kubernetes.weblogic.domain.v1.ServerStatus
    -
    -
    Name of node that is hosting the Pod containing this WebLogic server.
    -
    -
    getNodePort() - Method in class oracle.kubernetes.weblogic.domain.v1.ServerStartup
    -
    -
    Managed server NodePort port.
    -
    +
    getNodeManagerListenAddress() - Method in class oracle.kubernetes.operator.wlsconfig.WlsMachineConfig
    +
     
    +
    getNodeManagerListenPort() - Method in class oracle.kubernetes.operator.wlsconfig.WlsMachineConfig
    +
     
    +
    getNodeManagerType() - Method in class oracle.kubernetes.operator.wlsconfig.WlsMachineConfig
    +
     
    getOperatorExternalCertificateData() - Method in interface oracle.kubernetes.operator.rest.RestConfig
    Gets the external https port's certificate.
    @@ -1787,10 +1768,6 @@

    G

     
    getOrCreate(DomainPresenceInfo, String, String) - Method in class oracle.kubernetes.operator.helpers.ServerKubernetesObjectsFactory
     
    -
    getOverallHealth() - Method in class oracle.kubernetes.weblogic.domain.v1.ServerHealth
    -
    -
    Server health of this WebLogic server.
    -
    getPacket() - Method in class oracle.kubernetes.operator.work.Fiber
    Gets the current Packet associated with this fiber.
    @@ -1823,26 +1800,12 @@

    G

     
    getPublicPort() - Method in class oracle.kubernetes.operator.wlsconfig.NetworkAccessPoint
     
    -
    getReason() - Method in class oracle.kubernetes.weblogic.domain.v1.DomainCondition
    -
    -
    Unique, one-word, CamelCase reason for the condition's last transition.
    -
    -
    getReason() - Method in class oracle.kubernetes.weblogic.domain.v1.DomainStatus
    -
    -
    A brief CamelCase message indicating details about why the domain is in this state.
    -
    getRel() - Method in class oracle.kubernetes.operator.rest.model.LinkModel
    Get the link's relationship.
    -
    getReplicas() - Method in class oracle.kubernetes.weblogic.domain.v1.ClusterStartup
    -
    -
    Replicas is the desired number of managed servers running for this cluster.
    -
    -
    getReplicas() - Method in class oracle.kubernetes.weblogic.domain.v1.DomainSpec
    -
    -
    Replicas is the desired number of managed servers running in each WebLogic cluster that is not configured under clusterStartup.
    -
    +
    getResource() - Method in enum oracle.kubernetes.operator.helpers.AuthorizationProxy.Resource
    +
     
    getResourceVersion() - Method in interface oracle.kubernetes.operator.builders.CallParams
    On a watch call: when specified, shows changes that occur after that particular version of a resource.
    @@ -1875,41 +1838,35 @@

    G

    getServerConfig(String) - Method in class oracle.kubernetes.operator.wlsconfig.WlsDomainConfig
    -
    Returns the configuration for the WLS server with the given name
    +
    Returns the configuration for the WLS server with the given name.
    getServerConfigs() - Method in class oracle.kubernetes.operator.wlsconfig.WlsClusterConfig
    -
    Returns a list of server configurations for servers that belong to this cluster.
    +
    Returns a list of server configurations for servers that belong to this cluster, which includes + both statically configured servers and dynamic servers
    getServerConfigs() - Method in class oracle.kubernetes.operator.wlsconfig.WlsDomainConfig
    -
    Returns configuration of all servers found in the WLS domain, including admin server, standalone managed servers - that do not belong to any cluster, and managed servers that belong to a cluster.
    +
    Returns configuration of servers found in the WLS domain, including admin server, standalone managed servers + that do not belong to any cluster, and statically configured managed servers that belong to a cluster.
    -
    getServerName() - Method in class oracle.kubernetes.weblogic.domain.v1.ServerStartup
    +
    getServerConfigs() - Method in class oracle.kubernetes.operator.wlsconfig.WlsDynamicServersConfig
    -
    Managed server name of instance to start.
    -
    -
    getServerName() - Method in class oracle.kubernetes.weblogic.domain.v1.ServerStatus
    -
    -
    WebLogic server name.
    +
    Return list of WlsServerConfig objects containing configurations of WLS dynamic server that can be started under + the current cluster size
    getServers() - Method in class oracle.kubernetes.operator.helpers.DomainPresenceInfo
    Map from server name to server objects (Pods and Services)
    -
    getServers() - Method in class oracle.kubernetes.weblogic.domain.v1.DomainStatus
    -
    -
    Status of WebLogic servers in this domain.
    -
    -
    getServerStartup() - Method in class oracle.kubernetes.weblogic.domain.v1.DomainSpec
    -
    -
    List of server startup details for selected servers.
    -
    getServerStartupInfo() - Method in class oracle.kubernetes.operator.helpers.DomainPresenceInfo
    Server startup info
    +
    getServerTemplate() - Method in class oracle.kubernetes.operator.wlsconfig.WlsDynamicServersConfig
    +
    +
    Return the server template associated with this dynamic servers configuration
    +
    getService() - Method in class oracle.kubernetes.operator.helpers.ServerKubernetesObjects
    The Service
    @@ -1918,17 +1875,13 @@

    G

    Get the service token.
    -
    getServiceURL(String, String) - Static method in class oracle.kubernetes.operator.http.HttpClient
    -
    -
    Returns the URL to access the service; using the service clusterIP and port.
    -
    getServiceURL(V1Service) - Static method in class oracle.kubernetes.operator.http.HttpClient
    Returns the URL to access the Service; using the Service clusterIP and port
    -
    getSpec() - Method in class oracle.kubernetes.weblogic.domain.v1.Domain
    +
    getServiceURL(String, String) - Static method in class oracle.kubernetes.operator.http.HttpClient
    -
    DomainSpec is a description of a domain.
    +
    Returns the URL to access the service; using the service clusterIP and port.
    getSPI(Class<S>) - Method in interface oracle.kubernetes.operator.work.Component
    @@ -1940,17 +1893,9 @@

    G

     
    getSPI(Class<S>) - Method in class oracle.kubernetes.operator.work.Packet
     
    -
    getStartTime() - Method in class oracle.kubernetes.weblogic.domain.v1.DomainStatus
    -
    -
    RFC 3339 date and time at which the operator started the domain.
    -
    -
    getStartupControl() - Method in class oracle.kubernetes.weblogic.domain.v1.DomainSpec
    +
    getSslListenPort() - Method in class oracle.kubernetes.operator.wlsconfig.WlsServerConfig
    -
    Controls which managed servers will be started.
    -
    -
    getState() - Method in class oracle.kubernetes.weblogic.domain.v1.ServerStatus
    -
    -
    Current state of this WebLogic server.
    +
    Return the configured SSL listen port of this WLS server
    getStatus() - Method in class oracle.kubernetes.operator.http.Result
     
    @@ -1958,14 +1903,6 @@

    G

    Get the error's HTTP status code.
    -
    getStatus() - Method in class oracle.kubernetes.weblogic.domain.v1.Domain
    -
    -
    DomainStatus represents information about the status of a domain.
    -
    -
    getStatus() - Method in class oracle.kubernetes.weblogic.domain.v1.DomainCondition
    -
    -
    Status is the status of the condition.
    -
    getStatusCode() - Method in exception oracle.kubernetes.operator.http.HTTPException
     
    getStatusUpdater() - Method in class oracle.kubernetes.operator.helpers.DomainPresenceInfo
    @@ -1976,22 +1913,12 @@

    G

    True, if the operator is stopping
    -
    getSubsystemName() - Method in class oracle.kubernetes.weblogic.domain.v1.SubsystemHealth
    -
    -
    Name of subsystem providing symptom information.
    -
    -
    getSubsystems() - Method in class oracle.kubernetes.weblogic.domain.v1.ServerHealth
    -
    -
    Status of unhealthy subsystems, if any.
    -
    +
    getSubResource() - Method in enum oracle.kubernetes.operator.helpers.AuthorizationProxy.Resource
    +
     
    getSwaggerResource() - Method in class oracle.kubernetes.operator.rest.resource.VersionResource
    Construct and return the 'swagger' jaxrs child resource.
    -
    getSymptoms() - Method in class oracle.kubernetes.weblogic.domain.v1.SubsystemHealth
    -
    -
    Symptoms provided by the reporting subsystem.
    -
    getThrowables() - Method in exception oracle.kubernetes.operator.work.Step.MultiThrowable
    The multiple exceptions wrapped by this exception
    @@ -2012,22 +1939,30 @@

    G

    Get the error's type.
    -
    getType() - Method in class oracle.kubernetes.weblogic.domain.v1.DomainCondition
    -
    -
    Type is the type of the condition.
    -
    getUnderlyingLogger() - Method in class oracle.kubernetes.operator.logging.LoggingFacade
    Returns the underlying logger.
    -
    getVersion(String) - Static method in class oracle.kubernetes.operator.rest.backend.VersionUtils
    +
    getUpdateDynamicClusterSizePayload(int) - Method in class oracle.kubernetes.operator.wlsconfig.WlsClusterConfig
    -
    Get the un-aliased name of a version of the WebLogic operator REST api.
    +
    Return the payload used in the REST request for updating the dynamic cluster size.
    +
    +
    getUpdateDynamicClusterSizeUrl() - Method in class oracle.kubernetes.operator.wlsconfig.WlsClusterConfig
    +
    +
    Return the URL path of REST request for updating dynamic cluster size
    getVersion() - Method in class oracle.kubernetes.operator.rest.model.VersionModel
    Get the name of this version of the WebLogic operator REST api.
    +
    getVersion(String) - Static method in class oracle.kubernetes.operator.rest.backend.VersionUtils
    +
    +
    Get the un-aliased name of a version of the WebLogic operator REST api.
    +
    +
    getVersionCode(ApiClient) - Method in class oracle.kubernetes.operator.helpers.CallBuilder.SynchronousCallFactoryImpl
    +
     
    +
    getVersionCode(ApiClient) - Method in interface oracle.kubernetes.operator.helpers.SynchronousCallFactory
    +
     
    getVersionResource(String) - Method in class oracle.kubernetes.operator.rest.resource.VersionsResource
    Construct and return a 'version' jaxrs child resource.
    @@ -2040,6 +1975,10 @@

    G

     
    getWatchTuning() - Method in class oracle.kubernetes.operator.TuningParametersImpl
     
    +
    getWlsDomainConfig() - Method in class oracle.kubernetes.operator.wlsconfig.WlsClusterConfig
    +
    +
    Returns the WlsDomainConfig object for the WLS domain that this cluster belongs to
    +
    gracePeriodSeconds - Variable in class oracle.kubernetes.operator.helpers.CallBuilder
     
    @@ -2048,28 +1987,16 @@

    G

    H

    -
    hashCode() - Method in class oracle.kubernetes.weblogic.domain.v1.ClusterStartup
    -
     
    -
    hashCode() - Method in class oracle.kubernetes.weblogic.domain.v1.Domain
    -
     
    -
    hashCode() - Method in class oracle.kubernetes.weblogic.domain.v1.DomainCondition
    -
     
    -
    hashCode() - Method in class oracle.kubernetes.weblogic.domain.v1.DomainList
    -
     
    -
    hashCode() - Method in class oracle.kubernetes.weblogic.domain.v1.DomainSpec
    -
     
    -
    hashCode() - Method in class oracle.kubernetes.weblogic.domain.v1.DomainStatus
    -
     
    -
    hashCode() - Method in class oracle.kubernetes.weblogic.domain.v1.ServerHealth
    -
     
    -
    hashCode() - Method in class oracle.kubernetes.weblogic.domain.v1.ServerStartup
    -
     
    -
    hashCode() - Method in class oracle.kubernetes.weblogic.domain.v1.ServerStatus
    -
     
    -
    hashCode() - Method in class oracle.kubernetes.weblogic.domain.v1.SubsystemHealth
    -
     
    +
    hasDynamicServers() - Method in class oracle.kubernetes.operator.wlsconfig.WlsClusterConfig
    +
    +
    Whether the cluster contains any dynamic servers
    +
    hasNext() - Method in class oracle.kubernetes.operator.builders.WatchImpl
     
    +
    hasStaticServers() - Method in class oracle.kubernetes.operator.wlsconfig.WlsClusterConfig
    +
    +
    Whether the cluster contains any statically configured servers
    +
    HealthCheckHelper - Class in oracle.kubernetes.operator.helpers
    A Helper Class for checking the health of the WebLogic Operator
    @@ -2125,6 +2052,8 @@

    I

    INGRESS_DELETED - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
     
    +
    INGRESSES - oracle.kubernetes.operator.helpers.AuthorizationProxy.Resource
    +
     
    IngressHelper - Class in oracle.kubernetes.operator.helpers
    Helper class to add/remove server from Ingress.
    @@ -2135,8 +2064,6 @@

    I

    initializeInstance(ThreadFactory, String) - Static method in interface oracle.kubernetes.operator.TuningParameters
     
    -
    initializeInstance(ThreadFactory, String) - Static method in class oracle.kubernetes.operator.TuningParametersImpl
    -
     
    initialShortDelay - Variable in class oracle.kubernetes.operator.TuningParameters.MainTuning
     
    initiateWatch(WatchBuilder) - Method in class oracle.kubernetes.operator.ConfigMapWatcher
    @@ -2162,6 +2089,8 @@

    I

    Indicates that the next action should be to invoke the next step's Step.apply(Packet).
    +
    INVOKE - oracle.kubernetes.operator.work.NextAction.Kind
    +
     
    isCancelled() - Method in class oracle.kubernetes.operator.work.Fiber
     
    isCluster(String, String) - Method in interface oracle.kubernetes.operator.rest.backend.RestBackend
    @@ -2188,6 +2117,14 @@

    I

    isDone() - Method in class oracle.kubernetes.operator.work.Fiber
     
    +
    isDynamicServer() - Method in class oracle.kubernetes.operator.wlsconfig.WlsDynamicServerConfig
    +
    +
    Whether this server is a dynamic server, ie, not statically configured
    +
    +
    isDynamicServer() - Method in class oracle.kubernetes.operator.wlsconfig.WlsServerConfig
    +
    +
    Whether this server is a dynamic server, ie, not statically configured
    +
    isEmpty() - Method in class oracle.kubernetes.operator.helpers.ConfigMapConsumer
     
    isEmpty() - Method in class oracle.kubernetes.operator.utils.ConcurrentWeakHashMap
    @@ -2210,14 +2147,14 @@

    I

    Checks if a message at INFO level would actually be logged.
    -
    isLatest(String) - Static method in class oracle.kubernetes.operator.rest.backend.VersionUtils
    -
    -
    Get whether or not a version is the latest version of the WebLogic operator REST api.
    -
    isLatest() - Method in class oracle.kubernetes.operator.rest.model.VersionModel
    Get whether or not this is the latest version of the WebLogic operator REST api.
    +
    isLatest(String) - Static method in class oracle.kubernetes.operator.rest.backend.VersionUtils
    +
    +
    Get whether or not a version is the latest version of the WebLogic operator REST api.
    +
    isLoggable(Level) - Method in class oracle.kubernetes.operator.logging.LoggingFacade
    Checks if a message at the provided level would actually be logged.
    @@ -2226,6 +2163,10 @@

    I

    Checks if a message at SEVERE level would actually be logged.
    +
    isSslPortEnabled() - Method in class oracle.kubernetes.operator.wlsconfig.WlsServerConfig
    +
    +
    Return whether the SSL listen port is configured to be enabled or not
    +
    isSuccessful() - Method in class oracle.kubernetes.operator.http.Result
     
    isVersion(String) - Static method in class oracle.kubernetes.operator.rest.backend.VersionUtils
    @@ -2250,6 +2191,8 @@

    I

    J

    +
    JOBS - oracle.kubernetes.operator.helpers.AuthorizationProxy.Resource
    +
     
    JSON_PARSING_FAILED - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
     
    @@ -2278,7 +2221,7 @@

    K

     
    keySet() - Method in class oracle.kubernetes.operator.utils.ConcurrentWeakHashMap
    -
    Returns a Set view of the keys contained in this map.
    +
    Returns a Set view of the keys contained in this map.
    KIND_INGRESS - Static variable in interface oracle.kubernetes.operator.KubernetesConstants
     
    @@ -2325,16 +2268,10 @@

    L

    Construct a populated LinkModel.
    +
    list - oracle.kubernetes.operator.helpers.AuthorizationProxy.Operation
    +
     
    LIST_INGRESS_FOR_DOMAIN - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
     
    -
    listConfigMap(String) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    -
    -
    List config maps
    -
    -
    listConfigMapAsync(String, ResponseStep<V1ConfigMapList>) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    -
    -
    Asynchronous step for listing config maps
    -
    listDomain(String) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    List domains
    @@ -2343,52 +2280,32 @@

    L

    Asynchronous step for listing domains
    -
    listEent(String) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    -
    -
    List events
    -
    listEventAsync(String, ResponseStep<V1EventList>) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    Asynchronous step for listing events
    LISTING_DOMAINS - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
     
    -
    listIngress(String) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    -
    -
    List ingress
    -
    listIngressAsync(String, ResponseStep<V1beta1IngressList>) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    Asynchronous step for listing ingress
    -
    listNamespace() - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    -
    -
    List namespaces
    -
    -
    listNamespaceAsync(ResponseStep<V1NamespaceList>) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    -
    -
    Asynchronous step for listing namespaces
    -
    listPersistentVolume() - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    List persistent volumes
    -
    listPersistentVolumeAsync(ResponseStep<V1PersistentVolumeList>) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    -
    -
    Asynchronous step for listing persistent volumes
    -
    -
    listPersistentVolumeClaim(String) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    -
    -
    List persistent volume claims
    -
    listPersistentVolumeClaimAsync(String, ResponseStep<V1PersistentVolumeClaimList>) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    Asynchronous step for listing persistent volume claims
    -
    listPod(String) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    -
    -
    List pods
    -
    +
    ListPersistentVolumeClaimStep - Class in oracle.kubernetes.operator.steps
    +
     
    +
    ListPersistentVolumeClaimStep(Step) - Constructor for class oracle.kubernetes.operator.steps.ListPersistentVolumeClaimStep
    +
     
    +
    listPersistentVolumes(ApiClient, String, String, String, Boolean, String, Integer, String, Integer, Boolean) - Method in class oracle.kubernetes.operator.helpers.CallBuilder.SynchronousCallFactoryImpl
    +
     
    +
    listPersistentVolumes(ApiClient, String, String, String, Boolean, String, Integer, String, Integer, Boolean) - Method in interface oracle.kubernetes.operator.helpers.SynchronousCallFactory
    +
     
    listPodAsync(String, ResponseStep<V1PodList>) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    Asynchronous step for listing pods
    @@ -2401,45 +2318,13 @@

    L

    Asynchronous step for listing services
    -
    listWebLogicOracleV1DomainForAllNamespaces(String, String, Boolean, String, Integer, String, String, Integer, Boolean) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    list or watch objects of kind Domain
    -
    -
    listWebLogicOracleV1DomainForAllNamespacesAsync(String, String, Boolean, String, Integer, String, String, Integer, Boolean, ApiCallback<DomainList>) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    (asynchronously) list or watch objects of kind Domain
    -
    -
    listWebLogicOracleV1DomainForAllNamespacesCall(String, String, Boolean, String, Integer, String, String, Integer, Boolean, ProgressResponseBody.ProgressListener, ProgressRequestBody.ProgressRequestListener) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    Build call for listWebLogicOracleV1DomainForAllNamespaces
    -
    -
    listWebLogicOracleV1DomainForAllNamespacesWithHttpInfo(String, String, Boolean, String, Integer, String, String, Integer, Boolean) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    list or watch objects of kind Domain
    -
    -
    listWebLogicOracleV1NamespacedDomain(String, String, String, String, Boolean, String, Integer, String, Integer, Boolean) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    list or watch objects of kind Domain
    -
    -
    listWebLogicOracleV1NamespacedDomainAsync(String, String, String, String, Boolean, String, Integer, String, Integer, Boolean, ApiCallback<DomainList>) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    (asynchronously) list or watch objects of kind Domain
    -
    -
    listWebLogicOracleV1NamespacedDomainCall(String, String, String, String, Boolean, String, Integer, String, Integer, Boolean, ProgressResponseBody.ProgressListener, ProgressRequestBody.ProgressRequestListener) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    Build call for listWebLogicOracleV1NamespacedDomain
    -
    -
    listWebLogicOracleV1NamespacedDomainWithHttpInfo(String, String, String, String, Boolean, String, Integer, String, Integer, Boolean) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    list or watch objects of kind Domain
    -
    livenessProbeInitialDelaySeconds - Variable in class oracle.kubernetes.operator.TuningParameters.PodTuning
     
    livenessProbePeriodSeconds - Variable in class oracle.kubernetes.operator.TuningParameters.PodTuning
     
    livenessProbeTimeoutSeconds - Variable in class oracle.kubernetes.operator.TuningParameters.PodTuning
     
    -
    load(String) - Method in class oracle.kubernetes.operator.wlsconfig.WlsDomainConfig
    +
    LOCALSUBJECTACCESSREVIEWS - oracle.kubernetes.operator.helpers.AuthorizationProxy.Resource
     
    log(Level, String) - Method in class oracle.kubernetes.operator.logging.LoggingFacade
    @@ -2469,6 +2354,8 @@

    L

    LoggingFormatter() - Constructor for class oracle.kubernetes.operator.logging.LoggingFormatter
     
    +
    LOGS - oracle.kubernetes.operator.helpers.AuthorizationProxy.Resource
    +
     
    lookup(String) - Method in class oracle.kubernetes.operator.helpers.ServerKubernetesObjectsFactory
     
    @@ -2477,25 +2364,29 @@

    L

    M

    -
    Main - Class in oracle.kubernetes.operator
    +
    MacroSubstitutor - Class in oracle.kubernetes.operator.wlsconfig
    -
    A Kubernetes Operator for WebLogic.
    +
    Substitute macro specified in WLS server template.
    +
    +
    MacroSubstitutor(int, String, String, String, String) - Constructor for class oracle.kubernetes.operator.wlsconfig.MacroSubstitutor
    +
    +
    Creates a MacroSubstitutor with the given macro values that will be used in macro substitution
    -
    Main() - Constructor for class oracle.kubernetes.operator.Main
    -
     
    main(String[]) - Static method in class oracle.kubernetes.operator.Main
    Entry point
    -
    main(String[]) - Static method in class oracle.kubernetes.operator.SwaggerBuildHelper
    +
    Main - Class in oracle.kubernetes.operator
    -
    Update the swagger documentation HTML file with the contents of the swagger JSON file.
    +
    A Kubernetes Operator for WebLogic.
    -
    MAIN_COMPONENT_NAME - Static variable in interface oracle.kubernetes.operator.ProcessingConstants
    +
    Main() - Constructor for class oracle.kubernetes.operator.Main
    +
     
    +
    MAIN_COMPONENT_NAME - Static variable in interface oracle.kubernetes.operator.ProcessingConstants
     
    MAIN_THREAD_INTERRUPTED - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
     
    -
    MainTuning(int, int, long, long) - Constructor for class oracle.kubernetes.operator.TuningParameters.MainTuning
    +
    MainTuning(int, int, int, int, long, long) - Constructor for class oracle.kubernetes.operator.TuningParameters.MainTuning
     
    major - Variable in class oracle.kubernetes.operator.helpers.HealthCheckHelper.KubernetesVersion
     
    @@ -2515,6 +2406,22 @@

    M

     
    ManagedPodStep(Step) - Constructor for class oracle.kubernetes.operator.helpers.PodHelper.ManagedPodStep
     
    +
    ManagedServersUpStep - Class in oracle.kubernetes.operator.steps
    +
     
    +
    ManagedServersUpStep(Step) - Constructor for class oracle.kubernetes.operator.steps.ManagedServersUpStep
    +
     
    +
    ManagedServerUpAfterStep - Class in oracle.kubernetes.operator.steps
    +
     
    +
    ManagedServerUpAfterStep(Step) - Constructor for class oracle.kubernetes.operator.steps.ManagedServerUpAfterStep
    +
     
    +
    ManagedServerUpIteratorStep - Class in oracle.kubernetes.operator.steps
    +
     
    +
    ManagedServerUpIteratorStep(Collection<DomainPresenceInfo.ServerStartupInfo>, Step) - Constructor for class oracle.kubernetes.operator.steps.ManagedServerUpIteratorStep
    +
     
    +
    matchesResourceVersion(V1ObjectMeta, String) - Static method in class oracle.kubernetes.operator.helpers.VersionHelper
    +
    +
    Determines whether a resource matches a version
    +
    MATCHING_DOMAIN_NOT_FOUND - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
     
    maxRetryCount - Variable in class oracle.kubernetes.operator.helpers.CallBuilder
    @@ -2531,8 +2438,16 @@

    M

    N

    +
    name - Variable in class oracle.kubernetes.operator.calls.RequestParams
    +
     
    +
    namespace - oracle.kubernetes.operator.helpers.AuthorizationProxy.Scope
    +
     
    +
    namespace - Variable in class oracle.kubernetes.operator.calls.RequestParams
    +
     
    NAMESPACE_IS_DEFAULT - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
     
    +
    NAMESPACES - oracle.kubernetes.operator.helpers.AuthorizationProxy.Resource
    +
     
    NETWORK_ACCESS_POINT - Static variable in interface oracle.kubernetes.operator.ProcessingConstants
     
    NetworkAccessPoint - Class in oracle.kubernetes.operator.wlsconfig
    @@ -2541,6 +2456,8 @@

    N

    NetworkAccessPoint(String, String, Integer, Integer) - Constructor for class oracle.kubernetes.operator.wlsconfig.NetworkAccessPoint
     
    +
    NETWORKPOLICIES - oracle.kubernetes.operator.helpers.AuthorizationProxy.Resource
    +
     
    next() - Method in class oracle.kubernetes.operator.builders.WatchImpl
     
    NextAction - Class in oracle.kubernetes.operator.work
    @@ -2618,6 +2535,8 @@

    O

     
    OPERATOR_STARTED - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
     
    +
    OPERATOR_V1 - Static variable in interface oracle.kubernetes.operator.VersionConstants
    +
     
    OperatorLiveness - Class in oracle.kubernetes.operator
    This thread maintains the "liveness" indicator so that Kubernetes knows the Operator is still alive.
    @@ -2638,6 +2557,8 @@

    O

    Builders for complex objects that interact with Kubernetes.
    +
    oracle.kubernetes.operator.calls - package oracle.kubernetes.operator.calls
    +
     
    oracle.kubernetes.operator.helpers - package oracle.kubernetes.operator.helpers
    Helper classes that provide easy access to various Kubernetes features.
    @@ -2666,6 +2587,8 @@

    O

    REST resources.
    +
    oracle.kubernetes.operator.steps - package oracle.kubernetes.operator.steps
    +
     
    oracle.kubernetes.operator.utils - package oracle.kubernetes.operator.utils
     
    oracle.kubernetes.operator.watcher - package oracle.kubernetes.operator.watcher
    @@ -2680,12 +2603,6 @@

    O

    User-level thread infrastructure for the Operator.
    -
    oracle.kubernetes.weblogic.domain.v1 - package oracle.kubernetes.weblogic.domain.v1
    -
    -
    Model classes that are part of the WebLogic Custom Resource extensions to the Kubernetes API.
    -
    -
    oracle.kubernetes.weblogic.domain.v1.api - package oracle.kubernetes.weblogic.domain.v1.api
    -
     
    orphanDependents - Variable in class oracle.kubernetes.operator.helpers.CallBuilder
     
    owner - Variable in class oracle.kubernetes.operator.work.Fiber
    @@ -2696,62 +2613,16 @@

    O

    P

    +
    packet - Variable in class oracle.kubernetes.operator.work.Step.StepAndPacket
    +
     
    Packet - Class in oracle.kubernetes.operator.work
    Context of a single processing flow.
    Packet() - Constructor for class oracle.kubernetes.operator.work.Packet
     
    -
    packet - Variable in class oracle.kubernetes.operator.work.Step.StepAndPacket
    +
    patch - oracle.kubernetes.operator.helpers.AuthorizationProxy.Operation
     
    -
    patchWebLogicOracleV1NamespacedDomain(String, String, Patch, String) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    partially update the specified Domain
    -
    -
    patchWebLogicOracleV1NamespacedDomainAsync(String, String, Patch, String, ApiCallback<Domain>) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    (asynchronously) partially update the specified Domain
    -
    -
    patchWebLogicOracleV1NamespacedDomainCall(String, String, Patch, String, ProgressResponseBody.ProgressListener, ProgressRequestBody.ProgressRequestListener) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    Build call for patchWebLogicOracleV1NamespacedDomain
    -
    -
    patchWebLogicOracleV1NamespacedDomainScale(String, String, Patch, String) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    partially update scale of the specified Domain
    -
    -
    patchWebLogicOracleV1NamespacedDomainScaleAsync(String, String, Patch, String, ApiCallback<V1Scale>) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    (asynchronously) partially update scale of the specified Domain
    -
    -
    patchWebLogicOracleV1NamespacedDomainScaleCall(String, String, Patch, String, ProgressResponseBody.ProgressListener, ProgressRequestBody.ProgressRequestListener) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    Build call for patchWebLogicOracleV1NamespacedDomainScale
    -
    -
    patchWebLogicOracleV1NamespacedDomainScaleWithHttpInfo(String, String, Patch, String) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    partially update scale of the specified Domain
    -
    -
    patchWebLogicOracleV1NamespacedDomainStatus(String, String, Patch, String) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    partially update status of the specified Domain
    -
    -
    patchWebLogicOracleV1NamespacedDomainStatusAsync(String, String, Patch, String, ApiCallback<Domain>) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    (asynchronously) partially update status of the specified Domain
    -
    -
    patchWebLogicOracleV1NamespacedDomainStatusCall(String, String, Patch, String, ProgressResponseBody.ProgressListener, ProgressRequestBody.ProgressRequestListener) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    Build call for patchWebLogicOracleV1NamespacedDomainStatus
    -
    -
    patchWebLogicOracleV1NamespacedDomainStatusWithHttpInfo(String, String, Patch, String) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    partially update status of the specified Domain
    -
    -
    patchWebLogicOracleV1NamespacedDomainWithHttpInfo(String, String, Patch, String) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    partially update the specified Domain
    -
    performK8sVersionCheck() - Method in class oracle.kubernetes.operator.helpers.HealthCheckHelper
    Verify the k8s version.
    @@ -2760,10 +2631,14 @@

    P

    Execute health checks that are not security related.
    -
    performSecurityChecks(String) - Method in class oracle.kubernetes.operator.helpers.HealthCheckHelper
    +
    performSecurityChecks(HealthCheckHelper.KubernetesVersion) - Method in class oracle.kubernetes.operator.helpers.HealthCheckHelper
    Verify Access.
    +
    PERSISTENTVOLUMECLAIMS - oracle.kubernetes.operator.helpers.AuthorizationProxy.Resource
    +
     
    +
    PERSISTENTVOLUMES - oracle.kubernetes.operator.helpers.AuthorizationProxy.Resource
    +
     
    POD_DELETED - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
     
    POD_IS_FAILED - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
    @@ -2776,6 +2651,14 @@

    P

     
    PodHelper.ManagedPodStep - Class in oracle.kubernetes.operator.helpers
     
    +
    PODPRESETS - oracle.kubernetes.operator.helpers.AuthorizationProxy.Resource
    +
     
    +
    PODS - oracle.kubernetes.operator.helpers.AuthorizationProxy.Resource
    +
     
    +
    PODSECURITYPOLICIES - oracle.kubernetes.operator.helpers.AuthorizationProxy.Resource
    +
     
    +
    PODTEMPLATES - oracle.kubernetes.operator.helpers.AuthorizationProxy.Resource
    +
     
    PodTuning(int, int, int, int, int, int) - Constructor for class oracle.kubernetes.operator.TuningParameters.PodTuning
     
    PodWatcher - Class in oracle.kubernetes.operator
    @@ -2808,14 +2691,16 @@

    P

    propagationPolicy - Variable in class oracle.kubernetes.operator.helpers.CallBuilder
     
    +
    proxy - oracle.kubernetes.operator.helpers.AuthorizationProxy.Operation
    +
     
    +
    put(String, Object) - Method in class oracle.kubernetes.operator.work.Packet
    +
     
    put(String, String) - Method in class oracle.kubernetes.operator.helpers.ConfigMapConsumer
     
    put(K, V) - Method in class oracle.kubernetes.operator.utils.ConcurrentWeakHashMap
    Maps the specified key to the specified value in this table.
    -
    put(String, Object) - Method in class oracle.kubernetes.operator.work.Packet
    -
     
    putAll(Map<? extends String, ? extends String>) - Method in class oracle.kubernetes.operator.helpers.ConfigMapConsumer
     
    putAll(Map<? extends K, ? extends V>) - Method in class oracle.kubernetes.operator.utils.ConcurrentWeakHashMap
    @@ -2841,10 +2726,6 @@

    R

    Returns from admin server selected server configurations of all WLS servers configured in the domain.
    -
    readConfigMap(String, String) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    -
    -
    Read config map
    -
    readConfigMapAsync(String, String, ResponseStep<V1ConfigMap>) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    Asynchronous step for reading config map
    @@ -2853,22 +2734,14 @@

    R

    Creates asynchronous Step to read configuration from an admin server
    +
    readCustomResourceDefinition(ApiClient, String, String, Boolean, Boolean) - Method in class oracle.kubernetes.operator.helpers.CallBuilder.SynchronousCallFactoryImpl
    +
     
    +
    readCustomResourceDefinition(ApiClient, String, String, Boolean, Boolean) - Method in interface oracle.kubernetes.operator.helpers.SynchronousCallFactory
    +
     
    readCustomResourceDefinition(String) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    Read custom resource definition
    -
    readCustomResourceDefinitionAsync(String, ResponseStep<V1beta1CustomResourceDefinition>) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    -
    -
    Asynchronous step for reading custom resource definition
    -
    -
    readEvent(String, String) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    -
    -
    Read event
    -
    -
    readEventAsync(String, String, ResponseStep<V1Event>) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    -
    -
    Asynchronous step for reading event
    -
    readHealthStep(Step) - Static method in class oracle.kubernetes.operator.wlsconfig.WlsRetriever
    Creates asynchronous Step to read health from a server instance.
    @@ -2893,14 +2766,6 @@

    R

    Read namespace
    -
    readNamespaceAsync(String, ResponseStep<V1Namespace>) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    -
    -
    Asynchronous step for reading namespace
    -
    -
    readPod(String, String) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    -
    -
    Read pod
    -
    readPodAsync(String, String, ResponseStep<V1Pod>) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    Asynchronous step for reading pod
    @@ -2927,54 +2792,6 @@

    R

    Read Kubernetes version code
    -
    readWebLogicOracleV1NamespacedDomain(String, String, String, Boolean, Boolean) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    read the specified Domain
    -
    -
    readWebLogicOracleV1NamespacedDomainAsync(String, String, String, Boolean, Boolean, ApiCallback<Domain>) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    (asynchronously) read the specified Domain
    -
    -
    readWebLogicOracleV1NamespacedDomainCall(String, String, String, Boolean, Boolean, ProgressResponseBody.ProgressListener, ProgressRequestBody.ProgressRequestListener) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    Build call for readWebLogicOracleV1NamespacedDomain
    -
    -
    readWebLogicOracleV1NamespacedDomainScale(String, String, String) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    read scale of the specified Domain
    -
    -
    readWebLogicOracleV1NamespacedDomainScaleAsync(String, String, String, ApiCallback<V1Scale>) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    (asynchronously) read scale of the specified Domain
    -
    -
    readWebLogicOracleV1NamespacedDomainScaleCall(String, String, String, ProgressResponseBody.ProgressListener, ProgressRequestBody.ProgressRequestListener) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    Build call for readWebLogicOracleV1NamespacedDomainScale
    -
    -
    readWebLogicOracleV1NamespacedDomainScaleWithHttpInfo(String, String, String) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    read scale of the specified Domain
    -
    -
    readWebLogicOracleV1NamespacedDomainStatus(String, String, String) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    read status of the specified Domain
    -
    -
    readWebLogicOracleV1NamespacedDomainStatusAsync(String, String, String, ApiCallback<Domain>) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    (asynchronously) read status of the specified Domain
    -
    -
    readWebLogicOracleV1NamespacedDomainStatusCall(String, String, String, ProgressResponseBody.ProgressListener, ProgressRequestBody.ProgressRequestListener) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    Build call for readWebLogicOracleV1NamespacedDomainStatus
    -
    -
    readWebLogicOracleV1NamespacedDomainStatusWithHttpInfo(String, String, String) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    read status of the specified Domain
    -
    -
    readWebLogicOracleV1NamespacedDomainWithHttpInfo(String, String, String, Boolean, Boolean) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    read the specified Domain
    -
    receivedResponse(Watch.Response<V1Pod>) - Method in class oracle.kubernetes.operator.PodWatcher
     
    receivedResponse(Watch.Response<T>) - Method in interface oracle.kubernetes.operator.watcher.WatchListener
    @@ -2985,6 +2802,8 @@

    R

    Returns an object back to the pool.
    +
    redirect - oracle.kubernetes.operator.helpers.AuthorizationProxy.Operation
    +
     
    remove(Object) - Method in class oracle.kubernetes.operator.helpers.ConfigMapConsumer
     
    remove(Object) - Method in class oracle.kubernetes.operator.utils.ConcurrentWeakHashMap
    @@ -2994,12 +2813,10 @@

    R

    remove(Object, Object) - Method in class oracle.kubernetes.operator.utils.ConcurrentWeakHashMap
    REMOVING_INGRESS - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
     
    -
    replace(K, V, V) - Method in class oracle.kubernetes.operator.utils.ConcurrentWeakHashMap
    +
    replace - oracle.kubernetes.operator.helpers.AuthorizationProxy.Operation
    +
     
    replace(K, V) - Method in class oracle.kubernetes.operator.utils.ConcurrentWeakHashMap
    -
    replaceConfigMap(String, String, V1ConfigMap) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    -
    -
    Replace config map
    -
    +
    replace(K, V, V) - Method in class oracle.kubernetes.operator.utils.ConcurrentWeakHashMap
    replaceConfigMapAsync(String, String, V1ConfigMap, ResponseStep<V1ConfigMap>) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    Asynchronous step for replacing config map
    @@ -3012,86 +2829,10 @@

    R

    Asynchronous step for replacing domain
    -
    replaceDomainStatus(String, String, Domain) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    -
    -
    Replace domain status
    -
    -
    replaceDomainStatusAsync(String, String, Domain, ResponseStep<Domain>) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    -
    -
    Asynchronous step for replacing domain status
    -
    -
    replaceIngress(String, String, V1beta1Ingress) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    -
    -
    Replace ingress
    -
    replaceIngressAsync(String, String, V1beta1Ingress, ResponseStep<V1beta1Ingress>) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    Asynchronous step for replacing ingress
    -
    replacePod(String, String, V1Pod) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    -
    -
    Replace pod
    -
    -
    replacePodAsync(String, String, V1Pod, ResponseStep<V1Pod>) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    -
    -
    Asynchronous step for replacing pod
    -
    -
    replaceService(String, String, V1Service) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    -
    -
    Replace service
    -
    -
    replaceServiceAsync(String, String, V1Service, ResponseStep<V1Service>) - Method in class oracle.kubernetes.operator.helpers.CallBuilder
    -
    -
    Asynchronous step for replacing service
    -
    -
    replaceWebLogicOracleV1NamespacedDomain(String, String, Domain, String) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    replace the specified Domain
    -
    -
    replaceWebLogicOracleV1NamespacedDomainAsync(String, String, Domain, String, ApiCallback<Domain>) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    (asynchronously) replace the specified Domain
    -
    -
    replaceWebLogicOracleV1NamespacedDomainCall(String, String, Domain, String, ProgressResponseBody.ProgressListener, ProgressRequestBody.ProgressRequestListener) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    Build call for replaceWebLogicOracleV1NamespacedDomain
    -
    -
    replaceWebLogicOracleV1NamespacedDomainScale(String, String, V1Scale, String) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    replace scale of the specified Domain
    -
    -
    replaceWebLogicOracleV1NamespacedDomainScaleAsync(String, String, V1Scale, String, ApiCallback<V1Scale>) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    (asynchronously) replace scale of the specified Domain
    -
    -
    replaceWebLogicOracleV1NamespacedDomainScaleCall(String, String, V1Scale, String, ProgressResponseBody.ProgressListener, ProgressRequestBody.ProgressRequestListener) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    Build call for replaceWebLogicOracleV1NamespacedDomainScale
    -
    -
    replaceWebLogicOracleV1NamespacedDomainScaleWithHttpInfo(String, String, V1Scale, String) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    replace scale of the specified Domain
    -
    -
    replaceWebLogicOracleV1NamespacedDomainStatus(String, String, Domain, String) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    replace status of the specified Domain
    -
    -
    replaceWebLogicOracleV1NamespacedDomainStatusAsync(String, String, Domain, String, ApiCallback<Domain>) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    (asynchronously) replace status of the specified Domain
    -
    -
    replaceWebLogicOracleV1NamespacedDomainStatusCall(String, String, Domain, String, ProgressResponseBody.ProgressListener, ProgressRequestBody.ProgressRequestListener) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    Build call for replaceWebLogicOracleV1NamespacedDomainStatus
    -
    -
    replaceWebLogicOracleV1NamespacedDomainStatusWithHttpInfo(String, String, Domain, String) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    replace status of the specified Domain
    -
    -
    replaceWebLogicOracleV1NamespacedDomainWithHttpInfo(String, String, Domain, String) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    replace the specified Domain
    -
    REPLICA_MORE_THAN_WLS_SERVERS - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
     
    REQUEST_DEBUG_LOGGING_FILTER_PRIORITY - Static variable in class oracle.kubernetes.operator.rest.FilterPriorities
    @@ -3106,15 +2847,23 @@

    R

    Construct a RequestDebugLoggingFilter
    -
    reset() - Method in interface oracle.kubernetes.operator.helpers.CallBuilder.RetryStrategy
    +
    RequestParams - Class in oracle.kubernetes.operator.calls
    +
     
    +
    RequestParams(String, String, String, Object) - Constructor for class oracle.kubernetes.operator.calls.RequestParams
    +
     
    +
    reset() - Method in interface oracle.kubernetes.operator.calls.RetryStrategy
    -
    Called when retry count, or other statistics, should be reset, such as when partial list +
    Called when retry count, or other statistics, should be reset, such as when partial list was returned and new request for next portion of list (continue) is invoked.
    RESOURCE_BUNDLE_NOT_FOUND - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
     
    +
    RESOURCE_VERSION_LABEL - Static variable in interface oracle.kubernetes.operator.LabelConstants
    +
     
    resourceVersion - Variable in class oracle.kubernetes.operator.helpers.CallBuilder
     
    +
    RESPONSE_COMPONENT_NAME - Static variable in class oracle.kubernetes.operator.calls.AsyncRequestStep
    +
     
    RESPONSE_DEBUG_LOGGING_FILTER_PRIORITY - Static variable in class oracle.kubernetes.operator.rest.FilterPriorities
    The response debug logging filter's priority.
    @@ -3127,6 +2876,8 @@

    R

    Construct a ResponseDebugLoggingFilter
    +
    responseHeaders - Variable in class oracle.kubernetes.operator.calls.CallResponse
    +
     
    ResponseStep<T> - Class in oracle.kubernetes.operator.helpers
    Step to receive response of Kubernetes API server call.
    @@ -3144,12 +2895,8 @@

    R

    This constant is used internally to pass the RestConfig instance from the RestServer to the filters and resources so that they can access it.
    -
    RESTART_ADMIN_COMPLETE - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
    -
     
    RESTART_ADMIN_STARTING - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
     
    -
    RESTART_SERVERS_COMPLETE - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
    -
     
    RESTART_SERVERS_STARTING - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
     
    RestBackend - Interface in oracle.kubernetes.operator.rest.backend
    @@ -3187,6 +2934,8 @@

    R

    Constructs the WebLogic Operator REST server.
    +
    result - Variable in class oracle.kubernetes.operator.calls.CallResponse
    +
     
    Result - Class in oracle.kubernetes.operator.http
    Holder of response received from REST requests invoked using methods in HttpClient class
    @@ -3206,8 +2955,10 @@

    R

     
    RETRIEVING_SECRET - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
     
    -
    ROLLING_CLUSTERS_COMPLETE - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
    -
     
    +
    RetryStrategy - Interface in oracle.kubernetes.operator.calls
    +
    +
    Failed or timed-out call retry strategy
    +
    ROLLING_CLUSTERS_STARTING - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
     
    ROLLING_SERVERS - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
    @@ -3264,6 +3015,8 @@

    S

     
    SCRIPT_CONFIG_MAP - Static variable in interface oracle.kubernetes.operator.ProcessingConstants
     
    +
    SCRIPT_LOADED - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
    +
     
    ScriptConfigMapStep(String, String, Step) - Constructor for class oracle.kubernetes.operator.helpers.ConfigMapHelper.ScriptConfigMapStep
     
    SECRET_DATA_KEY - Static variable in class oracle.kubernetes.operator.helpers.SecretHelper
    @@ -3282,6 +3035,12 @@

    S

    SecretHelper.SecretType - Enum in oracle.kubernetes.operator.helpers
     
    +
    SECRETS - oracle.kubernetes.operator.helpers.AuthorizationProxy.Resource
    +
     
    +
    SELFSUBJECTACCESSREVIEWS - oracle.kubernetes.operator.helpers.AuthorizationProxy.Resource
    +
     
    +
    SELFSUBJECTRULESREVIEWS - oracle.kubernetes.operator.helpers.AuthorizationProxy.Resource
    +
     
    SERVER_HEALTH_MAP - Static variable in interface oracle.kubernetes.operator.ProcessingConstants
     
    SERVER_NAME - Static variable in interface oracle.kubernetes.operator.ProcessingConstants
    @@ -3294,11 +3053,17 @@

    S

     
    serverConfig - Variable in class oracle.kubernetes.operator.helpers.DomainPresenceInfo.ServerStartupInfo
     
    -
    ServerHealth - Class in oracle.kubernetes.weblogic.domain.v1
    -
    -
    ServerHealth describes the current status and health of a specific WebLogic server.
    -
    -
    ServerHealth() - Constructor for class oracle.kubernetes.weblogic.domain.v1.ServerHealth
    +
    ServerDownFinalizeStep - Class in oracle.kubernetes.operator.steps
    +
     
    +
    ServerDownFinalizeStep(String, Step) - Constructor for class oracle.kubernetes.operator.steps.ServerDownFinalizeStep
    +
     
    +
    ServerDownIteratorStep - Class in oracle.kubernetes.operator.steps
    +
     
    +
    ServerDownIteratorStep(Collection<Map.Entry<String, ServerKubernetesObjects>>, Step) - Constructor for class oracle.kubernetes.operator.steps.ServerDownIteratorStep
    +
     
    +
    ServerDownStep - Class in oracle.kubernetes.operator.steps
    +
     
    +
    ServerDownStep(String, ServerKubernetesObjects, Step) - Constructor for class oracle.kubernetes.operator.steps.ServerDownStep
     
    ServerKubernetesObjects - Class in oracle.kubernetes.operator.helpers
    @@ -3316,22 +3081,10 @@

    S

     
    serverStartup - Variable in class oracle.kubernetes.operator.helpers.DomainPresenceInfo.ServerStartupInfo
     
    -
    ServerStartup - Class in oracle.kubernetes.weblogic.domain.v1
    -
    -
    ServerStarup describes the desired startup state and passed environment variables for a specific managed server.
    -
    -
    ServerStartup() - Constructor for class oracle.kubernetes.weblogic.domain.v1.ServerStartup
    -
     
    ServerStartupInfo(WlsServerConfig, WlsClusterConfig, List<V1EnvVar>, ServerStartup) - Constructor for class oracle.kubernetes.operator.helpers.DomainPresenceInfo.ServerStartupInfo
    Create server startup info
    -
    ServerStatus - Class in oracle.kubernetes.weblogic.domain.v1
    -
    -
    ServerStatus describes the current status of a specific WebLogic server.
    -
    -
    ServerStatus() - Constructor for class oracle.kubernetes.weblogic.domain.v1.ServerStatus
    -
     
    ServerStatusReader - Class in oracle.kubernetes.operator
    Creates an asynchronous step to read the WebLogic server state from a particular pod
    @@ -3342,36 +3095,12 @@

    S

     
    ServiceHelper - Class in oracle.kubernetes.operator.helpers
     
    +
    SERVICES - oracle.kubernetes.operator.helpers.AuthorizationProxy.Resource
    +
     
    ServiceWatcher - Class in oracle.kubernetes.operator
    This class handles Service watching.
    -
    setActivationTime(DateTime) - Method in class oracle.kubernetes.weblogic.domain.v1.ServerHealth
    -
    -
    RFC 3339 date and time at which the server started.
    -
    -
    setAdminSecret(V1SecretReference) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainSpec
    -
    -
    Reference to secret containing domain administrator username and password.
    -
    -
    setApiClient(ApiClient) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
     
    -
    setApiVersion(String) - Method in class oracle.kubernetes.weblogic.domain.v1.Domain
    -
    -
    APIVersion defines the versioned schema of this representation of an object.
    -
    -
    setApiVersion(String) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainList
    -
    -
    APIVersion defines the versioned schema of this representation of an object.
    -
    -
    setAsName(String) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainSpec
    -
    -
    Admin server name.
    -
    -
    setAsPort(Integer) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainSpec
    -
    -
    Administration server port.
    -
    setClaims(V1PersistentVolumeClaimList) - Method in class oracle.kubernetes.operator.helpers.DomainPresenceInfo
    Sets claims
    @@ -3380,37 +3109,13 @@

    S

    Set the cluster's name.
    -
    setClusterName(String) - Method in class oracle.kubernetes.weblogic.domain.v1.ClusterStartup
    -
    -
    Name of specific cluster to start.
    -
    -
    setClusterName(String) - Method in class oracle.kubernetes.weblogic.domain.v1.ServerStatus
    -
    -
    WebLogic cluster name, if the server is part of a cluster
    -
    -
    setClusterStartup(List<ClusterStartup>) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainSpec
    -
    -
    List of server startup details for selected clusters
    -
    setCompletionCallback(Fiber.CompletionCallback) - Method in class oracle.kubernetes.operator.work.Fiber
    Updates completion callback associated with this Fiber
    -
    setConditions(List<DomainCondition>) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainStatus
    -
    -
    Current service state of domain.
    -
    setContextClassLoader(ClassLoader) - Method in class oracle.kubernetes.operator.work.Fiber
    -
    Sets the context ClassLoader of this fiber.
    -
    -
    setDesiredState(String) - Method in class oracle.kubernetes.weblogic.domain.v1.ClusterStartup
    -
    -
    Desired startup state for any managed server started in this cluster.
    -
    -
    setDesiredState(String) - Method in class oracle.kubernetes.weblogic.domain.v1.ServerStartup
    -
    -
    Desired startup state.
    +
    Sets the context ClassLoader of this fiber.
    setDetail(String) - Method in class oracle.kubernetes.operator.rest.model.ErrorModel
    @@ -3420,51 +3125,14 @@

    S

    Sets the domain.
    -
    setDomainName(String) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainSpec
    -
    -
    Domain name - (Required)
    -
    setDomainUID(String) - Method in class oracle.kubernetes.operator.rest.model.DomainModel
    Set the unique identifier that has been assigned to this WebLogic domain.
    -
    setDomainUID(String) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainSpec
    -
    -
    Domain unique identifier.
    -
    -
    setEnv(List<V1EnvVar>) - Method in class oracle.kubernetes.weblogic.domain.v1.ClusterStartup
    -
    -
    Environment variables to pass while starting managed servers in this cluster.
    -
    -
    setEnv(List<V1EnvVar>) - Method in class oracle.kubernetes.weblogic.domain.v1.ServerStartup
    -
    -
    Environment variables to pass while starting this managed server.
    -
    -
    setExportT3Channels(List<String>) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainSpec
    -
    -
    List of specific T3 channels to export.
    -
    -
    setHealth(ServerHealth) - Method in class oracle.kubernetes.weblogic.domain.v1.ServerStatus
    -
    -
    ServerHealth describes the current status and health of a specific WebLogic server.
    -
    -
    setHealth(String) - Method in class oracle.kubernetes.weblogic.domain.v1.SubsystemHealth
    -
    -
    Server health of this WebLogic server.
    -
    setHref(String) - Method in class oracle.kubernetes.operator.rest.model.LinkModel
    Set the link's hypertext reference.
    -
    setImage(String) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainSpec
    -
    -
    WebLogic Docker image.
    -
    -
    setImagePullPolicy(String) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainSpec
    -
    -
    Image pull policy.
    -
    setInstance(ContainerResolver) - Static method in class oracle.kubernetes.operator.work.ContainerResolver
    Sets the custom container resolver which can be used to get client's @@ -3474,32 +3142,12 @@

    S

    Set the items in the collection.
    -
    setItems(List<Domain>) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainList
    -
    -
    List of domains.
    -
    setJSON(JSON) - Static method in class oracle.kubernetes.operator.logging.LoggingFactory
     
    -
    setKind(String) - Method in class oracle.kubernetes.weblogic.domain.v1.Domain
    -
    -
    Kind is a string value representing the REST resource this object represents.
    -
    -
    setKind(String) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainList
    -
    -
    Kind is a string value representing the REST resource this object represents.
    -
    -
    setLastProbeTime(DateTime) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainCondition
    -
    -
    Last time we probed the condition.
    -
    setLastScanTime(DateTime) - Method in class oracle.kubernetes.operator.helpers.DomainPresenceInfo
    Sets last scan time
    -
    setLastTransitionTime(DateTime) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainCondition
    -
    -
    Last time the condition transitioned from one status to another.
    -
    setLatest(boolean) - Method in class oracle.kubernetes.operator.rest.model.VersionModel
    Set whether or not this is the latest version of the WebLogic operator REST api.
    @@ -3520,59 +3168,17 @@

    S

    Set the desired number of managed servers in the WebLogic cluster.
    -
    setMessage(String) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainCondition
    -
    -
    Human-readable message indicating details about last transition.
    -
    -
    setMessage(String) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainStatus
    -
    -
    A human readable message indicating details about why the domain is in this condition.
    -
    -
    setMetadata(V1ObjectMeta) - Method in class oracle.kubernetes.weblogic.domain.v1.Domain
    -
    -
    Standard object's metadata.
    -
    -
    setMetadata(V1ListMeta) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainList
    -
    -
    Standard list metadata.
    -
    setNext(Step) - Method in class oracle.kubernetes.operator.work.NextAction
    Sets the next step
    -
    setNodeName(String) - Method in class oracle.kubernetes.weblogic.domain.v1.ServerStatus
    -
    -
    Name of node that is hosting the Pod containing this WebLogic server.
    -
    -
    setNodePort(Integer) - Method in class oracle.kubernetes.weblogic.domain.v1.ServerStartup
    -
    -
    Managed server NodePort port.
    -
    -
    setOverallHealth(String) - Method in class oracle.kubernetes.weblogic.domain.v1.ServerHealth
    -
    -
    Server health of this WebLogic server.
    -
    -
    setReason(String) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainCondition
    -
    -
    Unique, one-word, CamelCase reason for the condition's last transition.
    -
    -
    setReason(String) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainStatus
    -
    -
    A brief CamelCase message indicating details about why the domain is in this state.
    -
    +
    setPrevious(Step) - Method in class oracle.kubernetes.operator.helpers.ResponseStep
    +
     
    setRel(String) - Method in class oracle.kubernetes.operator.rest.model.LinkModel
    Set the link's relationship.
    -
    setReplicas(Integer) - Method in class oracle.kubernetes.weblogic.domain.v1.ClusterStartup
    -
    -
    Replicas is the desired number of managed servers running for this cluster.
    -
    -
    setReplicas(Integer) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainSpec
    -
    -
    Replicas is the desired number of managed servers running in each WebLogic cluster that is not configured under clusterStartup.
    -
    -
    setRetryStep(Step) - Method in interface oracle.kubernetes.operator.helpers.CallBuilder.RetryStrategy
    +
    setRetryStep(Step) - Method in interface oracle.kubernetes.operator.calls.RetryStrategy
    Initialization that provides reference to step that should be invoked on a retry attempt
    @@ -3580,66 +3186,14 @@

    S

    Sets scan
    -
    setServerName(String) - Method in class oracle.kubernetes.weblogic.domain.v1.ServerStartup
    -
    -
    Managed server name of instance to start.
    -
    -
    setServerName(String) - Method in class oracle.kubernetes.weblogic.domain.v1.ServerStatus
    -
    -
    WebLogic server name.
    -
    -
    setServers(List<ServerStatus>) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainStatus
    -
    -
    Status of WebLogic servers in this domain.
    -
    -
    setServerStartup(List<ServerStartup>) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainSpec
    -
    -
    List of server startup details for selected servers.
    -
    setServerStartupInfo(Collection<DomainPresenceInfo.ServerStartupInfo>) - Method in class oracle.kubernetes.operator.helpers.DomainPresenceInfo
    Sets server startup info
    -
    setSpec(DomainSpec) - Method in class oracle.kubernetes.weblogic.domain.v1.Domain
    -
    -
    DomainSpec is a description of a domain.
    -
    -
    setStartTime(DateTime) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainStatus
    -
    -
    RFC 3339 date and time at which the operator started the domain.
    -
    -
    setStartupControl(String) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainSpec
    -
    -
    Controls which managed servers will be started.
    -
    -
    setState(String) - Method in class oracle.kubernetes.weblogic.domain.v1.ServerStatus
    -
    -
    Current state of this WebLogic server.
    -
    setStatus(int) - Method in class oracle.kubernetes.operator.rest.model.ErrorModel
    Set the error's HTTP status code.
    -
    setStatus(DomainStatus) - Method in class oracle.kubernetes.weblogic.domain.v1.Domain
    -
    -
    DomainStatus represents information about the status of a domain.
    -
    -
    setStatus(String) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainCondition
    -
    -
    Status is the status of the condition.
    -
    -
    setSubsystemName(String) - Method in class oracle.kubernetes.weblogic.domain.v1.SubsystemHealth
    -
    -
    Name of subsystem providing symptom information.
    -
    -
    setSubsystems(List<SubsystemHealth>) - Method in class oracle.kubernetes.weblogic.domain.v1.ServerHealth
    -
    -
    Status of unhealthy subsystems, if any.
    -
    -
    setSymptoms(List<String>) - Method in class oracle.kubernetes.weblogic.domain.v1.SubsystemHealth
    -
    -
    Symptoms provided by the reporting subsystem.
    -
    setTitle(String) - Method in class oracle.kubernetes.operator.rest.model.ErrorModel
    Set the error's title.
    @@ -3652,14 +3206,14 @@

    S

    Set the error's type.
    -
    setType(String) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainCondition
    -
    -
    Type is the type of the condition.
    -
    setVersion(String) - Method in class oracle.kubernetes.operator.rest.model.VersionModel
    Set the name of this version of the WebLogic operator REST api.
    +
    setWlsDomainConfig(WlsDomainConfig) - Method in class oracle.kubernetes.operator.wlsconfig.WlsClusterConfig
    +
    +
    Associate this cluster to the WlsDomainConfig object for the WLS domain that this cluster belongs to
    +
    severe(String) - Method in class oracle.kubernetes.operator.logging.LoggingFacade
    Logs a message at the SEVERE level.
    @@ -3717,8 +3271,12 @@

    S

     
    STATUS_UNCHANGED - Static variable in interface oracle.kubernetes.operator.ProcessingConstants
     
    +
    statusCode - Variable in class oracle.kubernetes.operator.calls.CallResponse
    +
     
    statusUpdateTimeoutSeconds - Variable in class oracle.kubernetes.operator.TuningParameters.MainTuning
     
    +
    step - Variable in class oracle.kubernetes.operator.work.Step.StepAndPacket
    +
     
    Step - Class in oracle.kubernetes.operator.work
    Individual step in a processing flow
    @@ -3727,8 +3285,6 @@

    S

    Create a step with the indicated next step.
    -
    step - Variable in class oracle.kubernetes.operator.work.Step.StepAndPacket
    -
     
    Step.MultiThrowable - Exception in oracle.kubernetes.operator.work
    Multi-exception
    @@ -3741,13 +3297,11 @@

    S

    Stops WebLogic operator's REST api.
    +
    STORAGECLASSES - oracle.kubernetes.operator.helpers.AuthorizationProxy.Resource
    +
     
    SUBJECT_ACCESS_REVIEW - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
     
    -
    SubsystemHealth - Class in oracle.kubernetes.weblogic.domain.v1
    -
    -
    SubsystemHealth describes the current health of a specific subsystem.
    -
    -
    SubsystemHealth() - Constructor for class oracle.kubernetes.weblogic.domain.v1.SubsystemHealth
    +
    SUBJECTACCESSREVIEWS - oracle.kubernetes.operator.helpers.AuthorizationProxy.Resource
     
    suspend(Consumer<Fiber>) - Method in class oracle.kubernetes.operator.work.NextAction
    @@ -3757,17 +3311,12 @@

    S

    Indicates that the fiber should be suspended.
    +
    SUSPEND - oracle.kubernetes.operator.work.NextAction.Kind
    +
     
    SUSPENDING_STATE - Static variable in interface oracle.kubernetes.operator.WebLogicConstants
     
    SVC_ACCOUNT_IS_DEFAULT - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
     
    -
    SwaggerBuildHelper - Class in oracle.kubernetes.operator
    -
    -
    A helper class that is used during the build process to update the Swagger HTML file with the - latest version of the REST API documentation.
    -
    -
    SwaggerBuildHelper() - Constructor for class oracle.kubernetes.operator.SwaggerBuildHelper
    -
     
    SwaggerResource - Class in oracle.kubernetes.operator.rest.resource
    SwaggerResource is a jaxrs resource that implements the REST api for the @@ -3777,6 +3326,10 @@

    S

    Construct a SwaggerResource.
    +
    SynchronousCallFactory - Interface in oracle.kubernetes.operator.helpers
    +
     
    +
    SynchronousCallFactoryImpl() - Constructor for class oracle.kubernetes.operator.helpers.CallBuilder.SynchronousCallFactoryImpl
    +
     
    @@ -3797,10 +3350,12 @@

    T

    ThreadLocalContainerResolver - Class in oracle.kubernetes.operator.work
    -
    ContainerResolver based on ThreadLocal.
    +
    ContainerResolver based on ThreadLocal.
    ThreadLocalContainerResolver() - Constructor for class oracle.kubernetes.operator.work.ThreadLocalContainerResolver
     
    +
    THROW - oracle.kubernetes.operator.work.NextAction.Kind
    +
     
    throwing(Throwable) - Method in class oracle.kubernetes.operator.logging.LoggingFacade
    Logs that an exception will be thrown.
    @@ -3811,6 +3366,8 @@

    T

    Converts value to nearest DNS-1123 legal name, which can be used as a Kubernetes identifier
    +
    TOKENREVIEWS - oracle.kubernetes.operator.helpers.AuthorizationProxy.Resource
    +
     
    toResponse(Exception) - Method in class oracle.kubernetes.operator.rest.ExceptionMapper
    toString() - Method in class oracle.kubernetes.operator.http.Result
     
    @@ -3821,6 +3378,12 @@

    T

     
    toString() - Method in class oracle.kubernetes.operator.wlsconfig.WlsDomainConfig
     
    +
    toString() - Method in class oracle.kubernetes.operator.wlsconfig.WlsDynamicServerConfig
    +
     
    +
    toString() - Method in class oracle.kubernetes.operator.wlsconfig.WlsDynamicServersConfig
    +
     
    +
    toString() - Method in class oracle.kubernetes.operator.wlsconfig.WlsMachineConfig
    +
     
    toString() - Method in class oracle.kubernetes.operator.wlsconfig.WlsServerConfig
     
    toString() - Method in class oracle.kubernetes.operator.work.Fiber
    @@ -3829,28 +3392,6 @@

    T

    Dumps the contents to assist debugging.
    -
    toString() - Method in class oracle.kubernetes.weblogic.domain.v1.ClusterStartup
    -
     
    -
    toString() - Method in class oracle.kubernetes.weblogic.domain.v1.Domain
    -
     
    -
    toString() - Method in class oracle.kubernetes.weblogic.domain.v1.DomainCondition
    -
     
    -
    toString() - Method in class oracle.kubernetes.weblogic.domain.v1.DomainList
    -
     
    -
    toString() - Method in class oracle.kubernetes.weblogic.domain.v1.DomainSpec
    -
     
    -
    toString() - Method in class oracle.kubernetes.weblogic.domain.v1.DomainStatus
    -
     
    -
    toString() - Method in class oracle.kubernetes.weblogic.domain.v1.ServerHealth
    -
     
    -
    toString() - Method in class oracle.kubernetes.weblogic.domain.v1.ServerStartup
    -
     
    -
    toString() - Method in class oracle.kubernetes.weblogic.domain.v1.ServerStatus
    -
     
    -
    toString() - Method in class oracle.kubernetes.weblogic.domain.v1.SubsystemHealth
    -
     
    -
    TRACE - Static variable in class oracle.kubernetes.operator.logging.LoggingFacade
    -
     
    trace(String) - Method in class oracle.kubernetes.operator.logging.LoggingFacade
    Logs a trace message with the ID FMW-TRACE at the FINER level
    @@ -3859,6 +3400,10 @@

    T

    Logs a trace message with the ID FMW-TRACE at the FINER level
    +
    TRACE - Static variable in class oracle.kubernetes.operator.logging.LoggingFacade
    +
     
    +
    TRAEFIK_LOAD_BALANCER_V1 - Static variable in interface oracle.kubernetes.operator.VersionConstants
    +
     
    TUNING_PARAMETERS - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
     
    TuningParameters - Interface in oracle.kubernetes.operator
    @@ -3883,11 +3428,7 @@

    U

     
    UNKNOWN_STATE - Static variable in interface oracle.kubernetes.operator.WebLogicConstants
     
    -
    updateDomainSpecAsNeeded(DomainSpec) - Method in class oracle.kubernetes.operator.wlsconfig.WlsDomainConfig
    -
    -
    Update the provided k8s domain spec to be consistent with the configuration of the WLS domain.
    -
    -
    updateTuningParameters() - Static method in class oracle.kubernetes.operator.TuningParametersImpl
    +
    update - oracle.kubernetes.operator.helpers.AuthorizationProxy.Operation
     
    @@ -3895,12 +3436,21 @@

    U

    V

    -
    validateClusterStartup(ClusterStartup) - Method in class oracle.kubernetes.operator.wlsconfig.WlsClusterConfig
    +
    validate(DomainSpec) - Method in class oracle.kubernetes.operator.wlsconfig.WlsDomainConfig
    +
     
    +
    validate(DomainSpec, List<ConfigUpdate>) - Method in class oracle.kubernetes.operator.wlsconfig.WlsDomainConfig
    +
    +
    Checks the provided k8s domain spec to see if it is consistent with the configuration of the WLS domain.
    +
    +
    validateClusterStartup(ClusterStartup, List<ConfigUpdate>) - Method in class oracle.kubernetes.operator.wlsconfig.WlsClusterConfig
    Validate the clusterStartup configured should be consistent with this configured WLS cluster.
    -
    validateReplicas(Integer, String) - Method in class oracle.kubernetes.operator.wlsconfig.WlsClusterConfig
    -
     
    +
    validateReplicas(Integer, String, List<ConfigUpdate>) - Method in class oracle.kubernetes.operator.wlsconfig.WlsClusterConfig
    +
    +
    Validate the configured replicas value in the kubernetes WebLogic domain spec against the + configured size of this cluster.
    +
    valueOf(String) - Static method in enum oracle.kubernetes.operator.helpers.AuthorizationProxy.Operation
    Returns the enum constant of this type with the specified name.
    @@ -3945,7 +3495,7 @@

    V

    values() - Method in class oracle.kubernetes.operator.utils.ConcurrentWeakHashMap
    -
    Returns a Collection view of the values contained in this map.
    +
    Returns a Collection view of the values contained in this map.
    values() - Static method in enum oracle.kubernetes.operator.work.NextAction.Kind
    @@ -3958,6 +3508,14 @@

    V

     
    VERIFY_K8S_MIN_VERSION - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
     
    +
    VersionConstants - Interface in oracle.kubernetes.operator
    +
     
    +
    VersionHelper - Class in oracle.kubernetes.operator.helpers
    +
    +
    Helper methods for managing versions.
    +
    +
    VersionHelper() - Constructor for class oracle.kubernetes.operator.helpers.VersionHelper
    +
     
    VersionModel - Class in oracle.kubernetes.operator.rest.model
    VersionModel describes a version of the WebLogic operator REST api.
    @@ -3993,6 +3551,8 @@

    V

    VersionUtils contains utilities for managing the versions of the WebLogic operator REST api.
    +
    VOYAGER_LOAD_BALANCER_V1 - Static variable in interface oracle.kubernetes.operator.VersionConstants
    +
     
    @@ -4017,6 +3577,8 @@

    W

    Logs a message which accompanies a Throwable at the WARNING level.
    +
    watch - oracle.kubernetes.operator.helpers.AuthorizationProxy.Operation
    +
     
    watch - Variable in class oracle.kubernetes.operator.helpers.CallBuilder
     
    WATCH_DOMAIN - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
    @@ -4035,7 +3597,7 @@

    W

     
    WatchI<T> - Interface in oracle.kubernetes.operator.builders
    -
    An interface that allows test-stubbing of the Kubernetes Watch class.
    +
    An iterator over watch responses from the server.
    WatchImpl<T> - Class in oracle.kubernetes.operator.builders
    @@ -4048,63 +3610,13 @@

    W

    This interface is used for the final destination to deliver watch events
    -
    WatchTuning(int) - Constructor for class oracle.kubernetes.operator.TuningParameters.WatchTuning
    -
     
    -
    watchWebLogicOracleV1DomainListForAllNamespaces(String, String, Boolean, String, Integer, String, String, Integer, Boolean) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    watch individual changes to a list of Domain
    -
    -
    watchWebLogicOracleV1DomainListForAllNamespacesAsync(String, String, Boolean, String, Integer, String, String, Integer, Boolean, ApiCallback<V1WatchEvent>) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    (asynchronously) watch individual changes to a list of Domain
    -
    -
    watchWebLogicOracleV1DomainListForAllNamespacesCall(String, String, Boolean, String, Integer, String, String, Integer, Boolean, ProgressResponseBody.ProgressListener, ProgressRequestBody.ProgressRequestListener) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    Build call for watchWebLogicOracleV1DomainListForAllNamespaces
    -
    -
    watchWebLogicOracleV1DomainListForAllNamespacesWithHttpInfo(String, String, Boolean, String, Integer, String, String, Integer, Boolean) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    watch individual changes to a list of Domain
    -
    -
    watchWebLogicOracleV1NamespacedDomain(String, String, String, String, Boolean, String, Integer, String, String, Integer, Boolean) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    watch changes to an object of kind Domain
    -
    -
    watchWebLogicOracleV1NamespacedDomainAsync(String, String, String, String, Boolean, String, Integer, String, String, Integer, Boolean, ApiCallback<V1WatchEvent>) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    (asynchronously) watch changes to an object of kind Domain
    -
    -
    watchWebLogicOracleV1NamespacedDomainCall(String, String, String, String, Boolean, String, Integer, String, String, Integer, Boolean, ProgressResponseBody.ProgressListener, ProgressRequestBody.ProgressRequestListener) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    Build call for watchWebLogicOracleV1NamespacedDomain
    -
    -
    watchWebLogicOracleV1NamespacedDomainList(String, String, String, Boolean, String, Integer, String, String, Integer, Boolean) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    watch individual changes to a list of Domain
    -
    -
    watchWebLogicOracleV1NamespacedDomainListAsync(String, String, String, Boolean, String, Integer, String, String, Integer, Boolean, ApiCallback<V1WatchEvent>) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    (asynchronously) watch individual changes to a list of Domain
    -
    -
    watchWebLogicOracleV1NamespacedDomainListCall(String, String, String, Boolean, String, Integer, String, String, Integer, Boolean, ProgressResponseBody.ProgressListener, ProgressRequestBody.ProgressRequestListener) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    Build call for watchWebLogicOracleV1NamespacedDomainList
    -
    -
    watchWebLogicOracleV1NamespacedDomainListWithHttpInfo(String, String, String, Boolean, String, Integer, String, String, Integer, Boolean) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    watch individual changes to a list of Domain
    -
    -
    watchWebLogicOracleV1NamespacedDomainWithHttpInfo(String, String, String, String, Boolean, String, Integer, String, String, Integer, Boolean) - Method in class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    -
    -
    watch changes to an object of kind Domain
    -
    -
    WEBLOGIC_DOMAIN - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
    +
    WatchPodReadyAdminStep - Class in oracle.kubernetes.operator.steps
     
    -
    WeblogicApi - Class in oracle.kubernetes.weblogic.domain.v1.api
    +
    WatchPodReadyAdminStep(Map<String, PodWatcher>, Step) - Constructor for class oracle.kubernetes.operator.steps.WatchPodReadyAdminStep
     
    -
    WeblogicApi() - Constructor for class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    +
    WatchTuning(int) - Constructor for class oracle.kubernetes.operator.TuningParameters.WatchTuning
     
    -
    WeblogicApi(ApiClient) - Constructor for class oracle.kubernetes.weblogic.domain.v1.api.WeblogicApi
    +
    WEBLOGIC_DOMAIN - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
     
    WebLogicConstants - Interface in oracle.kubernetes.operator
     
    @@ -4112,233 +3624,30 @@

    W

    Consumer for lambda-based builder pattern
    -
    withActivationTime(DateTime) - Method in class oracle.kubernetes.weblogic.domain.v1.ServerHealth
    -
    -
    RFC 3339 date and time at which the server started.
    -
    -
    withAdminSecret(V1SecretReference) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainSpec
    -
    -
    Reference to secret containing domain administrator username and password.
    -
    -
    withApiVersion(String) - Method in class oracle.kubernetes.weblogic.domain.v1.Domain
    -
    -
    APIVersion defines the versioned schema of this representation of an object.
    -
    -
    withApiVersion(String) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainList
    -
    -
    APIVersion defines the versioned schema of this representation of an object.
    -
    -
    withAsName(String) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainSpec
    -
    -
    Admin server name.
    -
    -
    withAsPort(Integer) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainSpec
    -
    -
    Administration server port.
    -
    -
    withClusterName(String) - Method in class oracle.kubernetes.weblogic.domain.v1.ClusterStartup
    -
    -
    Name of specific cluster to start.
    -
    -
    withClusterName(String) - Method in class oracle.kubernetes.weblogic.domain.v1.ServerStatus
    -
    -
    WebLogic cluster name, if the server is part of a cluster
    -
    -
    withClusterStartup(List<ClusterStartup>) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainSpec
    -
    -
    List of server startup details for selected clusters
    -
    -
    withConditions(List<DomainCondition>) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainStatus
    -
    -
    Current service state of domain.
    -
    -
    withDesiredState(String) - Method in class oracle.kubernetes.weblogic.domain.v1.ClusterStartup
    -
    -
    Desired startup state for any managed server started in this cluster.
    -
    -
    withDesiredState(String) - Method in class oracle.kubernetes.weblogic.domain.v1.ServerStartup
    -
    -
    Desired startup state.
    -
    -
    withDomainName(String) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainSpec
    -
    -
    Domain name - (Required)
    -
    -
    withDomainUID(String) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainSpec
    -
    -
    Domain unique identifier.
    -
    -
    withEnv(List<V1EnvVar>) - Method in class oracle.kubernetes.weblogic.domain.v1.ClusterStartup
    -
    -
    Environment variables to pass while starting managed servers in this cluster.
    -
    -
    withEnv(List<V1EnvVar>) - Method in class oracle.kubernetes.weblogic.domain.v1.ServerStartup
    -
    -
    Environment variables to pass while starting this managed server.
    -
    -
    withExportT3Channels(List<String>) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainSpec
    -
    -
    List of specific T3 channels to export.
    -
    withFieldSelector(String) - Method in class oracle.kubernetes.operator.builders.WatchBuilder
    Sets a value for the fieldSelector parameter for the call that will set up this watch.
    -
    withHealth(ServerHealth) - Method in class oracle.kubernetes.weblogic.domain.v1.ServerStatus
    -
    -
    ServerHealth describes the current status and health of a specific WebLogic server.
    -
    -
    withHealth(String) - Method in class oracle.kubernetes.weblogic.domain.v1.SubsystemHealth
    -
    -
    Server health of this WebLogic server.
    -
    -
    withImage(String) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainSpec
    -
    -
    WebLogic Docker image.
    -
    -
    withImagePullPolicy(String) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainSpec
    -
    -
    Image pull policy.
    -
    withIncludeUninitialized(Boolean) - Method in class oracle.kubernetes.operator.builders.WatchBuilder
     
    -
    withItems(List<Domain>) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainList
    -
    -
    List of domains.
    -
    -
    withKind(String) - Method in class oracle.kubernetes.weblogic.domain.v1.Domain
    -
    -
    Kind is a string value representing the REST resource this object represents.
    -
    -
    withKind(String) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainList
    -
    -
    Kind is a string value representing the REST resource this object represents.
    -
    withLabelSelector(String) - Method in class oracle.kubernetes.operator.builders.WatchBuilder
     
    withLabelSelectors(String...) - Method in class oracle.kubernetes.operator.builders.WatchBuilder
     
    -
    withLastProbeTime(DateTime) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainCondition
    -
    -
    Last time we probed the condition.
    -
    -
    withLastTransitionTime(DateTime) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainCondition
    -
    -
    Last time the condition transitioned from one status to another.
    -
    withLimit(Integer) - Method in class oracle.kubernetes.operator.builders.WatchBuilder
     
    -
    withMessage(String) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainCondition
    -
    -
    Human-readable message indicating details about last transition.
    -
    -
    withMessage(String) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainStatus
    -
    -
    A human readable message indicating details about why the domain is in this condition.
    -
    -
    withMetadata(V1ObjectMeta) - Method in class oracle.kubernetes.weblogic.domain.v1.Domain
    -
    -
    Standard object's metadata.
    -
    -
    withMetadata(V1ListMeta) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainList
    -
    -
    Standard list metadata.
    -
    -
    withNodeName(String) - Method in class oracle.kubernetes.weblogic.domain.v1.ServerStatus
    -
    -
    Name of node that is hosting the Pod containing this WebLogic server.
    -
    -
    withNodePort(Integer) - Method in class oracle.kubernetes.weblogic.domain.v1.ServerStartup
    -
    -
    Managed server NodePort port.
    -
    -
    withOverallHealth(String) - Method in class oracle.kubernetes.weblogic.domain.v1.ServerHealth
    -
    -
    Server health of this WebLogic server.
    -
    withPrettyPrinting() - Method in class oracle.kubernetes.operator.builders.WatchBuilder
     
    withProgressListener(ProgressResponseBody.ProgressListener) - Method in class oracle.kubernetes.operator.builders.WatchBuilder
     
    withProgressRequestListener(ProgressRequestBody.ProgressRequestListener) - Method in class oracle.kubernetes.operator.builders.WatchBuilder
     
    -
    withReason(String) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainCondition
    -
    -
    Unique, one-word, CamelCase reason for the condition's last transition.
    -
    -
    withReason(String) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainStatus
    -
    -
    A brief CamelCase message indicating details about why the domain is in this state.
    -
    -
    withReplicas(Integer) - Method in class oracle.kubernetes.weblogic.domain.v1.ClusterStartup
    -
    -
    Replicas is the desired number of managed servers running for this cluster.
    -
    -
    withReplicas(Integer) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainSpec
    -
    -
    Replicas is the desired number of managed servers running in each WebLogic cluster that is not configured under clusterStartup.
    -
    withResourceVersion(String) - Method in class oracle.kubernetes.operator.builders.WatchBuilder
     
    -
    withServerName(String) - Method in class oracle.kubernetes.weblogic.domain.v1.ServerStartup
    -
    -
    Managed server name of instance to start.
    -
    -
    withServerName(String) - Method in class oracle.kubernetes.weblogic.domain.v1.ServerStatus
    -
    -
    WebLogic server name.
    -
    -
    withServers(List<ServerStatus>) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainStatus
    -
    -
    Status of WebLogic servers in this domain.
    -
    -
    withServerStartup(List<ServerStartup>) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainSpec
    -
    -
    List of server startup details for selected servers.
    -
    -
    withSpec(DomainSpec) - Method in class oracle.kubernetes.weblogic.domain.v1.Domain
    -
    -
    DomainSpec is a description of a domain.
    -
    -
    withStartTime(DateTime) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainStatus
    -
    -
    RFC 3339 date and time at which the operator started the domain.
    -
    -
    withStartupControl(String) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainSpec
    -
    -
    Controls which managed servers will be started.
    -
    -
    withState(String) - Method in class oracle.kubernetes.weblogic.domain.v1.ServerStatus
    -
    -
    Current state of this WebLogic server.
    -
    -
    withStatus(DomainStatus) - Method in class oracle.kubernetes.weblogic.domain.v1.Domain
    -
    -
    DomainStatus represents information about the status of a domain.
    -
    -
    withStatus(String) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainCondition
    -
    -
    Status is the status of the condition.
    -
    -
    withSubsystemName(String) - Method in class oracle.kubernetes.weblogic.domain.v1.SubsystemHealth
    -
    -
    Name of subsystem providing symptom information.
    -
    -
    withSubsystems(List<SubsystemHealth>) - Method in class oracle.kubernetes.weblogic.domain.v1.ServerHealth
    -
    -
    Status of unhealthy subsystems, if any.
    -
    -
    withSymptoms(List<String>) - Method in class oracle.kubernetes.weblogic.domain.v1.SubsystemHealth
    -
    -
    Symptoms provided by the reporting subsystem.
    -
    withTimeoutSeconds(Integer) - Method in class oracle.kubernetes.operator.builders.WatchBuilder
     
    -
    withType(String) - Method in class oracle.kubernetes.weblogic.domain.v1.DomainCondition
    -
    -
    Type is the type of the condition.
    -
    +
    WLS_CLUSTER_SIZE_UPDATED - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
    +
     
    WLS_CONFIGURATION_READ - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
     
    WLS_CONFIGURATION_READ_FAILED - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
    @@ -4351,17 +3660,54 @@

    W

     
    WLS_HEALTH_READ_FAILED - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
     
    +
    WLS_SERVER_TEMPLATE_NOT_FOUND - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
    +
     
    +
    WLS_UPDATE_CLUSTER_SIZE_FAILED - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
    +
     
    +
    WLS_UPDATE_CLUSTER_SIZE_INVALID_CLUSTER - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
    +
     
    +
    WLS_UPDATE_CLUSTER_SIZE_STARTING - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
    +
     
    +
    WLS_UPDATE_CLUSTER_SIZE_TIMED_OUT - Static variable in class oracle.kubernetes.operator.logging.MessageKeys
    +
     
    WlsClusterConfig - Class in oracle.kubernetes.operator.wlsconfig
    Contains configuration of a WLS cluster
    WlsClusterConfig(String) - Constructor for class oracle.kubernetes.operator.wlsconfig.WlsClusterConfig
    -
     
    +
    +
    Constructor for a static cluster when Json result is not available
    +
    +
    WlsClusterConfig(String, WlsDynamicServersConfig) - Constructor for class oracle.kubernetes.operator.wlsconfig.WlsClusterConfig
    +
    +
    Constructor that can also be used for a dynamic cluster + *
    +
    WlsDomainConfig - Class in oracle.kubernetes.operator.wlsconfig
    -
    Contains configuration for a WebLogid Domain
    +
    Contains a snapshot of configuration for a WebLogic Domain
    +
    +
    WlsDomainConfig(String) - Constructor for class oracle.kubernetes.operator.wlsconfig.WlsDomainConfig
    +
    +
    Constructor when no JSON response is available
    +
    +
    WlsDynamicServerConfig - Class in oracle.kubernetes.operator.wlsconfig
    +
    +
    Contains configuration of a WLS server that belongs to a dynamic cluster
    +
    +
    WlsDynamicServersConfig - Class in oracle.kubernetes.operator.wlsconfig
    +
    +
    Contains values from a WLS dynamic servers configuration, which configures a WLS dynamic cluster
    +
    +
    WlsDynamicServersConfig(Integer, Integer, String, boolean, String, WlsServerConfig, List<WlsServerConfig>) - Constructor for class oracle.kubernetes.operator.wlsconfig.WlsDynamicServersConfig
    +
    +
    Constructor
    -
    WlsDomainConfig() - Constructor for class oracle.kubernetes.operator.wlsconfig.WlsDomainConfig
    +
    WlsMachineConfig - Class in oracle.kubernetes.operator.wlsconfig
    +
    +
    Contains values from a WLS machine configuration
    +
    +
    WlsMachineConfig(String, Integer, String, String) - Constructor for class oracle.kubernetes.operator.wlsconfig.WlsMachineConfig
     
    WlsRetriever - Class in oracle.kubernetes.operator.wlsconfig
    @@ -4375,8 +3721,10 @@

    W

    Contains configuration of a WebLogic server
    -
    WlsServerConfig(String, Integer, String, Map<String, Object>) - Constructor for class oracle.kubernetes.operator.wlsconfig.WlsServerConfig
    -
     
    +
    WlsServerConfig(String, Integer, String, Integer, boolean, String, List<NetworkAccessPoint>) - Constructor for class oracle.kubernetes.operator.wlsconfig.WlsServerConfig
    +
    +
    Construct a WlsServerConfig object using values provided
    +
    wrappedExecutorService(String, Container) - Static method in class oracle.kubernetes.operator.work.Engine
     
    @@ -4423,6 +3771,9 @@

    W

    } //--> +
    diff --git a/docs/apidocs/index.html b/docs/apidocs/index.html index 215263c671b..0b67e21ffaa 100644 --- a/docs/apidocs/index.html +++ b/docs/apidocs/index.html @@ -1,10 +1,11 @@ - + - + +operator-runtime 1.0 API -weblogic-kubernetes-operator 0.2 API + - - - - - - - -<noscript> -<div>JavaScript is disabled on your browser.</div> -</noscript> -<h2>Frame Alert</h2> -<p>This document is designed to be viewed using the frames feature. If you see this message, you are using a non-frame-capable web client. Link to <a href="overview-summary.html">Non-frame version</a>.</p> - - + + + +
    +
    +
    + +
    +
    + +
    +
    +
    + +
    +
    + diff --git a/docs/apidocs/jquery/external/jquery/jquery.js b/docs/apidocs/jquery/external/jquery/jquery.js new file mode 100644 index 00000000000..c5c648255c1 --- /dev/null +++ b/docs/apidocs/jquery/external/jquery/jquery.js @@ -0,0 +1,9789 @@ +/*! + * jQuery JavaScript Library v1.10.2 + * http://jquery.com/ + * + * Includes Sizzle.js + * http://sizzlejs.com/ + * + * Copyright 2005, 2013 jQuery Foundation, Inc. and other contributors + * Released under the MIT license + * http://jquery.org/license + * + * Date: 2013-07-03T13:48Z + */ +(function( window, undefined ) { + +// Can't do this because several apps including ASP.NET trace +// the stack via arguments.caller.callee and Firefox dies if +// you try to trace through "use strict" call chains. (#13335) +// Support: Firefox 18+ +//"use strict"; +var + // The deferred used on DOM ready + readyList, + + // A central reference to the root jQuery(document) + rootjQuery, + + // Support: IE<10 + // For `typeof xmlNode.method` instead of `xmlNode.method !== undefined` + core_strundefined = typeof undefined, + + // Use the correct document accordingly with window argument (sandbox) + location = window.location, + document = window.document, + docElem = document.documentElement, + + // Map over jQuery in case of overwrite + _jQuery = window.jQuery, + + // Map over the $ in case of overwrite + _$ = window.$, + + // [[Class]] -> type pairs + class2type = {}, + + // List of deleted data cache ids, so we can reuse them + core_deletedIds = [], + + core_version = "1.10.2", + + // Save a reference to some core methods + core_concat = core_deletedIds.concat, + core_push = core_deletedIds.push, + core_slice = core_deletedIds.slice, + core_indexOf = core_deletedIds.indexOf, + core_toString = class2type.toString, + core_hasOwn = class2type.hasOwnProperty, + core_trim = core_version.trim, + + // Define a local copy of jQuery + jQuery = function( selector, context ) { + // The jQuery object is actually just the init constructor 'enhanced' + return new jQuery.fn.init( selector, context, rootjQuery ); + }, + + // Used for matching numbers + core_pnum = /[+-]?(?:\d*\.|)\d+(?:[eE][+-]?\d+|)/.source, + + // Used for splitting on whitespace + core_rnotwhite = /\S+/g, + + // Make sure we trim BOM and NBSP (here's looking at you, Safari 5.0 and IE) + rtrim = /^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g, + + // A simple way to check for HTML strings + // Prioritize #id over to avoid XSS via location.hash (#9521) + // Strict HTML recognition (#11290: must start with <) + rquickExpr = /^(?:\s*(<[\w\W]+>)[^>]*|#([\w-]*))$/, + + // Match a standalone tag + rsingleTag = /^<(\w+)\s*\/?>(?:<\/\1>|)$/, + + // JSON RegExp + rvalidchars = /^[\],:{}\s]*$/, + rvalidbraces = /(?:^|:|,)(?:\s*\[)+/g, + rvalidescape = /\\(?:["\\\/bfnrt]|u[\da-fA-F]{4})/g, + rvalidtokens = /"[^"\\\r\n]*"|true|false|null|-?(?:\d+\.|)\d+(?:[eE][+-]?\d+|)/g, + + // Matches dashed string for camelizing + rmsPrefix = /^-ms-/, + rdashAlpha = /-([\da-z])/gi, + + // Used by jQuery.camelCase as callback to replace() + fcamelCase = function( all, letter ) { + return letter.toUpperCase(); + }, + + // The ready event handler + completed = function( event ) { + + // readyState === "complete" is good enough for us to call the dom ready in oldIE + if ( document.addEventListener || event.type === "load" || document.readyState === "complete" ) { + detach(); + jQuery.ready(); + } + }, + // Clean-up method for dom ready events + detach = function() { + if ( document.addEventListener ) { + document.removeEventListener( "DOMContentLoaded", completed, false ); + window.removeEventListener( "load", completed, false ); + + } else { + document.detachEvent( "onreadystatechange", completed ); + window.detachEvent( "onload", completed ); + } + }; + +jQuery.fn = jQuery.prototype = { + // The current version of jQuery being used + jquery: core_version, + + constructor: jQuery, + init: function( selector, context, rootjQuery ) { + var match, elem; + + // HANDLE: $(""), $(null), $(undefined), $(false) + if ( !selector ) { + return this; + } + + // Handle HTML strings + if ( typeof selector === "string" ) { + if ( selector.charAt(0) === "<" && selector.charAt( selector.length - 1 ) === ">" && selector.length >= 3 ) { + // Assume that strings that start and end with <> are HTML and skip the regex check + match = [ null, selector, null ]; + + } else { + match = rquickExpr.exec( selector ); + } + + // Match html or make sure no context is specified for #id + if ( match && (match[1] || !context) ) { + + // HANDLE: $(html) -> $(array) + if ( match[1] ) { + context = context instanceof jQuery ? context[0] : context; + + // scripts is true for back-compat + jQuery.merge( this, jQuery.parseHTML( + match[1], + context && context.nodeType ? context.ownerDocument || context : document, + true + ) ); + + // HANDLE: $(html, props) + if ( rsingleTag.test( match[1] ) && jQuery.isPlainObject( context ) ) { + for ( match in context ) { + // Properties of context are called as methods if possible + if ( jQuery.isFunction( this[ match ] ) ) { + this[ match ]( context[ match ] ); + + // ...and otherwise set as attributes + } else { + this.attr( match, context[ match ] ); + } + } + } + + return this; + + // HANDLE: $(#id) + } else { + elem = document.getElementById( match[2] ); + + // Check parentNode to catch when Blackberry 4.6 returns + // nodes that are no longer in the document #6963 + if ( elem && elem.parentNode ) { + // Handle the case where IE and Opera return items + // by name instead of ID + if ( elem.id !== match[2] ) { + return rootjQuery.find( selector ); + } + + // Otherwise, we inject the element directly into the jQuery object + this.length = 1; + this[0] = elem; + } + + this.context = document; + this.selector = selector; + return this; + } + + // HANDLE: $(expr, $(...)) + } else if ( !context || context.jquery ) { + return ( context || rootjQuery ).find( selector ); + + // HANDLE: $(expr, context) + // (which is just equivalent to: $(context).find(expr) + } else { + return this.constructor( context ).find( selector ); + } + + // HANDLE: $(DOMElement) + } else if ( selector.nodeType ) { + this.context = this[0] = selector; + this.length = 1; + return this; + + // HANDLE: $(function) + // Shortcut for document ready + } else if ( jQuery.isFunction( selector ) ) { + return rootjQuery.ready( selector ); + } + + if ( selector.selector !== undefined ) { + this.selector = selector.selector; + this.context = selector.context; + } + + return jQuery.makeArray( selector, this ); + }, + + // Start with an empty selector + selector: "", + + // The default length of a jQuery object is 0 + length: 0, + + toArray: function() { + return core_slice.call( this ); + }, + + // Get the Nth element in the matched element set OR + // Get the whole matched element set as a clean array + get: function( num ) { + return num == null ? + + // Return a 'clean' array + this.toArray() : + + // Return just the object + ( num < 0 ? this[ this.length + num ] : this[ num ] ); + }, + + // Take an array of elements and push it onto the stack + // (returning the new matched element set) + pushStack: function( elems ) { + + // Build a new jQuery matched element set + var ret = jQuery.merge( this.constructor(), elems ); + + // Add the old object onto the stack (as a reference) + ret.prevObject = this; + ret.context = this.context; + + // Return the newly-formed element set + return ret; + }, + + // Execute a callback for every element in the matched set. + // (You can seed the arguments with an array of args, but this is + // only used internally.) + each: function( callback, args ) { + return jQuery.each( this, callback, args ); + }, + + ready: function( fn ) { + // Add the callback + jQuery.ready.promise().done( fn ); + + return this; + }, + + slice: function() { + return this.pushStack( core_slice.apply( this, arguments ) ); + }, + + first: function() { + return this.eq( 0 ); + }, + + last: function() { + return this.eq( -1 ); + }, + + eq: function( i ) { + var len = this.length, + j = +i + ( i < 0 ? len : 0 ); + return this.pushStack( j >= 0 && j < len ? [ this[j] ] : [] ); + }, + + map: function( callback ) { + return this.pushStack( jQuery.map(this, function( elem, i ) { + return callback.call( elem, i, elem ); + })); + }, + + end: function() { + return this.prevObject || this.constructor(null); + }, + + // For internal use only. + // Behaves like an Array's method, not like a jQuery method. + push: core_push, + sort: [].sort, + splice: [].splice +}; + +// Give the init function the jQuery prototype for later instantiation +jQuery.fn.init.prototype = jQuery.fn; + +jQuery.extend = jQuery.fn.extend = function() { + var src, copyIsArray, copy, name, options, clone, + target = arguments[0] || {}, + i = 1, + length = arguments.length, + deep = false; + + // Handle a deep copy situation + if ( typeof target === "boolean" ) { + deep = target; + target = arguments[1] || {}; + // skip the boolean and the target + i = 2; + } + + // Handle case when target is a string or something (possible in deep copy) + if ( typeof target !== "object" && !jQuery.isFunction(target) ) { + target = {}; + } + + // extend jQuery itself if only one argument is passed + if ( length === i ) { + target = this; + --i; + } + + for ( ; i < length; i++ ) { + // Only deal with non-null/undefined values + if ( (options = arguments[ i ]) != null ) { + // Extend the base object + for ( name in options ) { + src = target[ name ]; + copy = options[ name ]; + + // Prevent never-ending loop + if ( target === copy ) { + continue; + } + + // Recurse if we're merging plain objects or arrays + if ( deep && copy && ( jQuery.isPlainObject(copy) || (copyIsArray = jQuery.isArray(copy)) ) ) { + if ( copyIsArray ) { + copyIsArray = false; + clone = src && jQuery.isArray(src) ? src : []; + + } else { + clone = src && jQuery.isPlainObject(src) ? src : {}; + } + + // Never move original objects, clone them + target[ name ] = jQuery.extend( deep, clone, copy ); + + // Don't bring in undefined values + } else if ( copy !== undefined ) { + target[ name ] = copy; + } + } + } + } + + // Return the modified object + return target; +}; + +jQuery.extend({ + // Unique for each copy of jQuery on the page + // Non-digits removed to match rinlinejQuery + expando: "jQuery" + ( core_version + Math.random() ).replace( /\D/g, "" ), + + noConflict: function( deep ) { + if ( window.$ === jQuery ) { + window.$ = _$; + } + + if ( deep && window.jQuery === jQuery ) { + window.jQuery = _jQuery; + } + + return jQuery; + }, + + // Is the DOM ready to be used? Set to true once it occurs. + isReady: false, + + // A counter to track how many items to wait for before + // the ready event fires. See #6781 + readyWait: 1, + + // Hold (or release) the ready event + holdReady: function( hold ) { + if ( hold ) { + jQuery.readyWait++; + } else { + jQuery.ready( true ); + } + }, + + // Handle when the DOM is ready + ready: function( wait ) { + + // Abort if there are pending holds or we're already ready + if ( wait === true ? --jQuery.readyWait : jQuery.isReady ) { + return; + } + + // Make sure body exists, at least, in case IE gets a little overzealous (ticket #5443). + if ( !document.body ) { + return setTimeout( jQuery.ready ); + } + + // Remember that the DOM is ready + jQuery.isReady = true; + + // If a normal DOM Ready event fired, decrement, and wait if need be + if ( wait !== true && --jQuery.readyWait > 0 ) { + return; + } + + // If there are functions bound, to execute + readyList.resolveWith( document, [ jQuery ] ); + + // Trigger any bound ready events + if ( jQuery.fn.trigger ) { + jQuery( document ).trigger("ready").off("ready"); + } + }, + + // See test/unit/core.js for details concerning isFunction. + // Since version 1.3, DOM methods and functions like alert + // aren't supported. They return false on IE (#2968). + isFunction: function( obj ) { + return jQuery.type(obj) === "function"; + }, + + isArray: Array.isArray || function( obj ) { + return jQuery.type(obj) === "array"; + }, + + isWindow: function( obj ) { + /* jshint eqeqeq: false */ + return obj != null && obj == obj.window; + }, + + isNumeric: function( obj ) { + return !isNaN( parseFloat(obj) ) && isFinite( obj ); + }, + + type: function( obj ) { + if ( obj == null ) { + return String( obj ); + } + return typeof obj === "object" || typeof obj === "function" ? + class2type[ core_toString.call(obj) ] || "object" : + typeof obj; + }, + + isPlainObject: function( obj ) { + var key; + + // Must be an Object. + // Because of IE, we also have to check the presence of the constructor property. + // Make sure that DOM nodes and window objects don't pass through, as well + if ( !obj || jQuery.type(obj) !== "object" || obj.nodeType || jQuery.isWindow( obj ) ) { + return false; + } + + try { + // Not own constructor property must be Object + if ( obj.constructor && + !core_hasOwn.call(obj, "constructor") && + !core_hasOwn.call(obj.constructor.prototype, "isPrototypeOf") ) { + return false; + } + } catch ( e ) { + // IE8,9 Will throw exceptions on certain host objects #9897 + return false; + } + + // Support: IE<9 + // Handle iteration over inherited properties before own properties. + if ( jQuery.support.ownLast ) { + for ( key in obj ) { + return core_hasOwn.call( obj, key ); + } + } + + // Own properties are enumerated firstly, so to speed up, + // if last one is own, then all properties are own. + for ( key in obj ) {} + + return key === undefined || core_hasOwn.call( obj, key ); + }, + + isEmptyObject: function( obj ) { + var name; + for ( name in obj ) { + return false; + } + return true; + }, + + error: function( msg ) { + throw new Error( msg ); + }, + + // data: string of html + // context (optional): If specified, the fragment will be created in this context, defaults to document + // keepScripts (optional): If true, will include scripts passed in the html string + parseHTML: function( data, context, keepScripts ) { + if ( !data || typeof data !== "string" ) { + return null; + } + if ( typeof context === "boolean" ) { + keepScripts = context; + context = false; + } + context = context || document; + + var parsed = rsingleTag.exec( data ), + scripts = !keepScripts && []; + + // Single tag + if ( parsed ) { + return [ context.createElement( parsed[1] ) ]; + } + + parsed = jQuery.buildFragment( [ data ], context, scripts ); + if ( scripts ) { + jQuery( scripts ).remove(); + } + return jQuery.merge( [], parsed.childNodes ); + }, + + parseJSON: function( data ) { + // Attempt to parse using the native JSON parser first + if ( window.JSON && window.JSON.parse ) { + return window.JSON.parse( data ); + } + + if ( data === null ) { + return data; + } + + if ( typeof data === "string" ) { + + // Make sure leading/trailing whitespace is removed (IE can't handle it) + data = jQuery.trim( data ); + + if ( data ) { + // Make sure the incoming data is actual JSON + // Logic borrowed from http://json.org/json2.js + if ( rvalidchars.test( data.replace( rvalidescape, "@" ) + .replace( rvalidtokens, "]" ) + .replace( rvalidbraces, "")) ) { + + return ( new Function( "return " + data ) )(); + } + } + } + + jQuery.error( "Invalid JSON: " + data ); + }, + + // Cross-browser xml parsing + parseXML: function( data ) { + var xml, tmp; + if ( !data || typeof data !== "string" ) { + return null; + } + try { + if ( window.DOMParser ) { // Standard + tmp = new DOMParser(); + xml = tmp.parseFromString( data , "text/xml" ); + } else { // IE + xml = new ActiveXObject( "Microsoft.XMLDOM" ); + xml.async = "false"; + xml.loadXML( data ); + } + } catch( e ) { + xml = undefined; + } + if ( !xml || !xml.documentElement || xml.getElementsByTagName( "parsererror" ).length ) { + jQuery.error( "Invalid XML: " + data ); + } + return xml; + }, + + noop: function() {}, + + // Evaluates a script in a global context + // Workarounds based on findings by Jim Driscoll + // http://weblogs.java.net/blog/driscoll/archive/2009/09/08/eval-javascript-global-context + globalEval: function( data ) { + if ( data && jQuery.trim( data ) ) { + // We use execScript on Internet Explorer + // We use an anonymous function so that context is window + // rather than jQuery in Firefox + ( window.execScript || function( data ) { + window[ "eval" ].call( window, data ); + } )( data ); + } + }, + + // Convert dashed to camelCase; used by the css and data modules + // Microsoft forgot to hump their vendor prefix (#9572) + camelCase: function( string ) { + return string.replace( rmsPrefix, "ms-" ).replace( rdashAlpha, fcamelCase ); + }, + + nodeName: function( elem, name ) { + return elem.nodeName && elem.nodeName.toLowerCase() === name.toLowerCase(); + }, + + // args is for internal usage only + each: function( obj, callback, args ) { + var value, + i = 0, + length = obj.length, + isArray = isArraylike( obj ); + + if ( args ) { + if ( isArray ) { + for ( ; i < length; i++ ) { + value = callback.apply( obj[ i ], args ); + + if ( value === false ) { + break; + } + } + } else { + for ( i in obj ) { + value = callback.apply( obj[ i ], args ); + + if ( value === false ) { + break; + } + } + } + + // A special, fast, case for the most common use of each + } else { + if ( isArray ) { + for ( ; i < length; i++ ) { + value = callback.call( obj[ i ], i, obj[ i ] ); + + if ( value === false ) { + break; + } + } + } else { + for ( i in obj ) { + value = callback.call( obj[ i ], i, obj[ i ] ); + + if ( value === false ) { + break; + } + } + } + } + + return obj; + }, + + // Use native String.trim function wherever possible + trim: core_trim && !core_trim.call("\uFEFF\xA0") ? + function( text ) { + return text == null ? + "" : + core_trim.call( text ); + } : + + // Otherwise use our own trimming functionality + function( text ) { + return text == null ? + "" : + ( text + "" ).replace( rtrim, "" ); + }, + + // results is for internal usage only + makeArray: function( arr, results ) { + var ret = results || []; + + if ( arr != null ) { + if ( isArraylike( Object(arr) ) ) { + jQuery.merge( ret, + typeof arr === "string" ? + [ arr ] : arr + ); + } else { + core_push.call( ret, arr ); + } + } + + return ret; + }, + + inArray: function( elem, arr, i ) { + var len; + + if ( arr ) { + if ( core_indexOf ) { + return core_indexOf.call( arr, elem, i ); + } + + len = arr.length; + i = i ? i < 0 ? Math.max( 0, len + i ) : i : 0; + + for ( ; i < len; i++ ) { + // Skip accessing in sparse arrays + if ( i in arr && arr[ i ] === elem ) { + return i; + } + } + } + + return -1; + }, + + merge: function( first, second ) { + var l = second.length, + i = first.length, + j = 0; + + if ( typeof l === "number" ) { + for ( ; j < l; j++ ) { + first[ i++ ] = second[ j ]; + } + } else { + while ( second[j] !== undefined ) { + first[ i++ ] = second[ j++ ]; + } + } + + first.length = i; + + return first; + }, + + grep: function( elems, callback, inv ) { + var retVal, + ret = [], + i = 0, + length = elems.length; + inv = !!inv; + + // Go through the array, only saving the items + // that pass the validator function + for ( ; i < length; i++ ) { + retVal = !!callback( elems[ i ], i ); + if ( inv !== retVal ) { + ret.push( elems[ i ] ); + } + } + + return ret; + }, + + // arg is for internal usage only + map: function( elems, callback, arg ) { + var value, + i = 0, + length = elems.length, + isArray = isArraylike( elems ), + ret = []; + + // Go through the array, translating each of the items to their + if ( isArray ) { + for ( ; i < length; i++ ) { + value = callback( elems[ i ], i, arg ); + + if ( value != null ) { + ret[ ret.length ] = value; + } + } + + // Go through every key on the object, + } else { + for ( i in elems ) { + value = callback( elems[ i ], i, arg ); + + if ( value != null ) { + ret[ ret.length ] = value; + } + } + } + + // Flatten any nested arrays + return core_concat.apply( [], ret ); + }, + + // A global GUID counter for objects + guid: 1, + + // Bind a function to a context, optionally partially applying any + // arguments. + proxy: function( fn, context ) { + var args, proxy, tmp; + + if ( typeof context === "string" ) { + tmp = fn[ context ]; + context = fn; + fn = tmp; + } + + // Quick check to determine if target is callable, in the spec + // this throws a TypeError, but we will just return undefined. + if ( !jQuery.isFunction( fn ) ) { + return undefined; + } + + // Simulated bind + args = core_slice.call( arguments, 2 ); + proxy = function() { + return fn.apply( context || this, args.concat( core_slice.call( arguments ) ) ); + }; + + // Set the guid of unique handler to the same of original handler, so it can be removed + proxy.guid = fn.guid = fn.guid || jQuery.guid++; + + return proxy; + }, + + // Multifunctional method to get and set values of a collection + // The value/s can optionally be executed if it's a function + access: function( elems, fn, key, value, chainable, emptyGet, raw ) { + var i = 0, + length = elems.length, + bulk = key == null; + + // Sets many values + if ( jQuery.type( key ) === "object" ) { + chainable = true; + for ( i in key ) { + jQuery.access( elems, fn, i, key[i], true, emptyGet, raw ); + } + + // Sets one value + } else if ( value !== undefined ) { + chainable = true; + + if ( !jQuery.isFunction( value ) ) { + raw = true; + } + + if ( bulk ) { + // Bulk operations run against the entire set + if ( raw ) { + fn.call( elems, value ); + fn = null; + + // ...except when executing function values + } else { + bulk = fn; + fn = function( elem, key, value ) { + return bulk.call( jQuery( elem ), value ); + }; + } + } + + if ( fn ) { + for ( ; i < length; i++ ) { + fn( elems[i], key, raw ? value : value.call( elems[i], i, fn( elems[i], key ) ) ); + } + } + } + + return chainable ? + elems : + + // Gets + bulk ? + fn.call( elems ) : + length ? fn( elems[0], key ) : emptyGet; + }, + + now: function() { + return ( new Date() ).getTime(); + }, + + // A method for quickly swapping in/out CSS properties to get correct calculations. + // Note: this method belongs to the css module but it's needed here for the support module. + // If support gets modularized, this method should be moved back to the css module. + swap: function( elem, options, callback, args ) { + var ret, name, + old = {}; + + // Remember the old values, and insert the new ones + for ( name in options ) { + old[ name ] = elem.style[ name ]; + elem.style[ name ] = options[ name ]; + } + + ret = callback.apply( elem, args || [] ); + + // Revert the old values + for ( name in options ) { + elem.style[ name ] = old[ name ]; + } + + return ret; + } +}); + +jQuery.ready.promise = function( obj ) { + if ( !readyList ) { + + readyList = jQuery.Deferred(); + + // Catch cases where $(document).ready() is called after the browser event has already occurred. + // we once tried to use readyState "interactive" here, but it caused issues like the one + // discovered by ChrisS here: http://bugs.jquery.com/ticket/12282#comment:15 + if ( document.readyState === "complete" ) { + // Handle it asynchronously to allow scripts the opportunity to delay ready + setTimeout( jQuery.ready ); + + // Standards-based browsers support DOMContentLoaded + } else if ( document.addEventListener ) { + // Use the handy event callback + document.addEventListener( "DOMContentLoaded", completed, false ); + + // A fallback to window.onload, that will always work + window.addEventListener( "load", completed, false ); + + // If IE event model is used + } else { + // Ensure firing before onload, maybe late but safe also for iframes + document.attachEvent( "onreadystatechange", completed ); + + // A fallback to window.onload, that will always work + window.attachEvent( "onload", completed ); + + // If IE and not a frame + // continually check to see if the document is ready + var top = false; + + try { + top = window.frameElement == null && document.documentElement; + } catch(e) {} + + if ( top && top.doScroll ) { + (function doScrollCheck() { + if ( !jQuery.isReady ) { + + try { + // Use the trick by Diego Perini + // http://javascript.nwbox.com/IEContentLoaded/ + top.doScroll("left"); + } catch(e) { + return setTimeout( doScrollCheck, 50 ); + } + + // detach all dom ready events + detach(); + + // and execute any waiting functions + jQuery.ready(); + } + })(); + } + } + } + return readyList.promise( obj ); +}; + +// Populate the class2type map +jQuery.each("Boolean Number String Function Array Date RegExp Object Error".split(" "), function(i, name) { + class2type[ "[object " + name + "]" ] = name.toLowerCase(); +}); + +function isArraylike( obj ) { + var length = obj.length, + type = jQuery.type( obj ); + + if ( jQuery.isWindow( obj ) ) { + return false; + } + + if ( obj.nodeType === 1 && length ) { + return true; + } + + return type === "array" || type !== "function" && + ( length === 0 || + typeof length === "number" && length > 0 && ( length - 1 ) in obj ); +} + +// All jQuery objects should point back to these +rootjQuery = jQuery(document); +/*! + * Sizzle CSS Selector Engine v1.10.2 + * http://sizzlejs.com/ + * + * Copyright 2013 jQuery Foundation, Inc. and other contributors + * Released under the MIT license + * http://jquery.org/license + * + * Date: 2013-07-03 + */ +(function( window, undefined ) { + +var i, + support, + cachedruns, + Expr, + getText, + isXML, + compile, + outermostContext, + sortInput, + + // Local document vars + setDocument, + document, + docElem, + documentIsHTML, + rbuggyQSA, + rbuggyMatches, + matches, + contains, + + // Instance-specific data + expando = "sizzle" + -(new Date()), + preferredDoc = window.document, + dirruns = 0, + done = 0, + classCache = createCache(), + tokenCache = createCache(), + compilerCache = createCache(), + hasDuplicate = false, + sortOrder = function( a, b ) { + if ( a === b ) { + hasDuplicate = true; + return 0; + } + return 0; + }, + + // General-purpose constants + strundefined = typeof undefined, + MAX_NEGATIVE = 1 << 31, + + // Instance methods + hasOwn = ({}).hasOwnProperty, + arr = [], + pop = arr.pop, + push_native = arr.push, + push = arr.push, + slice = arr.slice, + // Use a stripped-down indexOf if we can't use a native one + indexOf = arr.indexOf || function( elem ) { + var i = 0, + len = this.length; + for ( ; i < len; i++ ) { + if ( this[i] === elem ) { + return i; + } + } + return -1; + }, + + booleans = "checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped", + + // Regular expressions + + // Whitespace characters http://www.w3.org/TR/css3-selectors/#whitespace + whitespace = "[\\x20\\t\\r\\n\\f]", + // http://www.w3.org/TR/css3-syntax/#characters + characterEncoding = "(?:\\\\.|[\\w-]|[^\\x00-\\xa0])+", + + // Loosely modeled on CSS identifier characters + // An unquoted value should be a CSS identifier http://www.w3.org/TR/css3-selectors/#attribute-selectors + // Proper syntax: http://www.w3.org/TR/CSS21/syndata.html#value-def-identifier + identifier = characterEncoding.replace( "w", "w#" ), + + // Acceptable operators http://www.w3.org/TR/selectors/#attribute-selectors + attributes = "\\[" + whitespace + "*(" + characterEncoding + ")" + whitespace + + "*(?:([*^$|!~]?=)" + whitespace + "*(?:(['\"])((?:\\\\.|[^\\\\])*?)\\3|(" + identifier + ")|)|)" + whitespace + "*\\]", + + // Prefer arguments quoted, + // then not containing pseudos/brackets, + // then attribute selectors/non-parenthetical expressions, + // then anything else + // These preferences are here to reduce the number of selectors + // needing tokenize in the PSEUDO preFilter + pseudos = ":(" + characterEncoding + ")(?:\\(((['\"])((?:\\\\.|[^\\\\])*?)\\3|((?:\\\\.|[^\\\\()[\\]]|" + attributes.replace( 3, 8 ) + ")*)|.*)\\)|)", + + // Leading and non-escaped trailing whitespace, capturing some non-whitespace characters preceding the latter + rtrim = new RegExp( "^" + whitespace + "+|((?:^|[^\\\\])(?:\\\\.)*)" + whitespace + "+$", "g" ), + + rcomma = new RegExp( "^" + whitespace + "*," + whitespace + "*" ), + rcombinators = new RegExp( "^" + whitespace + "*([>+~]|" + whitespace + ")" + whitespace + "*" ), + + rsibling = new RegExp( whitespace + "*[+~]" ), + rattributeQuotes = new RegExp( "=" + whitespace + "*([^\\]'\"]*)" + whitespace + "*\\]", "g" ), + + rpseudo = new RegExp( pseudos ), + ridentifier = new RegExp( "^" + identifier + "$" ), + + matchExpr = { + "ID": new RegExp( "^#(" + characterEncoding + ")" ), + "CLASS": new RegExp( "^\\.(" + characterEncoding + ")" ), + "TAG": new RegExp( "^(" + characterEncoding.replace( "w", "w*" ) + ")" ), + "ATTR": new RegExp( "^" + attributes ), + "PSEUDO": new RegExp( "^" + pseudos ), + "CHILD": new RegExp( "^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\(" + whitespace + + "*(even|odd|(([+-]|)(\\d*)n|)" + whitespace + "*(?:([+-]|)" + whitespace + + "*(\\d+)|))" + whitespace + "*\\)|)", "i" ), + "bool": new RegExp( "^(?:" + booleans + ")$", "i" ), + // For use in libraries implementing .is() + // We use this for POS matching in `select` + "needsContext": new RegExp( "^" + whitespace + "*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\(" + + whitespace + "*((?:-\\d)?\\d*)" + whitespace + "*\\)|)(?=[^-]|$)", "i" ) + }, + + rnative = /^[^{]+\{\s*\[native \w/, + + // Easily-parseable/retrievable ID or TAG or CLASS selectors + rquickExpr = /^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/, + + rinputs = /^(?:input|select|textarea|button)$/i, + rheader = /^h\d$/i, + + rescape = /'|\\/g, + + // CSS escapes http://www.w3.org/TR/CSS21/syndata.html#escaped-characters + runescape = new RegExp( "\\\\([\\da-f]{1,6}" + whitespace + "?|(" + whitespace + ")|.)", "ig" ), + funescape = function( _, escaped, escapedWhitespace ) { + var high = "0x" + escaped - 0x10000; + // NaN means non-codepoint + // Support: Firefox + // Workaround erroneous numeric interpretation of +"0x" + return high !== high || escapedWhitespace ? + escaped : + // BMP codepoint + high < 0 ? + String.fromCharCode( high + 0x10000 ) : + // Supplemental Plane codepoint (surrogate pair) + String.fromCharCode( high >> 10 | 0xD800, high & 0x3FF | 0xDC00 ); + }; + +// Optimize for push.apply( _, NodeList ) +try { + push.apply( + (arr = slice.call( preferredDoc.childNodes )), + preferredDoc.childNodes + ); + // Support: Android<4.0 + // Detect silently failing push.apply + arr[ preferredDoc.childNodes.length ].nodeType; +} catch ( e ) { + push = { apply: arr.length ? + + // Leverage slice if possible + function( target, els ) { + push_native.apply( target, slice.call(els) ); + } : + + // Support: IE<9 + // Otherwise append directly + function( target, els ) { + var j = target.length, + i = 0; + // Can't trust NodeList.length + while ( (target[j++] = els[i++]) ) {} + target.length = j - 1; + } + }; +} + +function Sizzle( selector, context, results, seed ) { + var match, elem, m, nodeType, + // QSA vars + i, groups, old, nid, newContext, newSelector; + + if ( ( context ? context.ownerDocument || context : preferredDoc ) !== document ) { + setDocument( context ); + } + + context = context || document; + results = results || []; + + if ( !selector || typeof selector !== "string" ) { + return results; + } + + if ( (nodeType = context.nodeType) !== 1 && nodeType !== 9 ) { + return []; + } + + if ( documentIsHTML && !seed ) { + + // Shortcuts + if ( (match = rquickExpr.exec( selector )) ) { + // Speed-up: Sizzle("#ID") + if ( (m = match[1]) ) { + if ( nodeType === 9 ) { + elem = context.getElementById( m ); + // Check parentNode to catch when Blackberry 4.6 returns + // nodes that are no longer in the document #6963 + if ( elem && elem.parentNode ) { + // Handle the case where IE, Opera, and Webkit return items + // by name instead of ID + if ( elem.id === m ) { + results.push( elem ); + return results; + } + } else { + return results; + } + } else { + // Context is not a document + if ( context.ownerDocument && (elem = context.ownerDocument.getElementById( m )) && + contains( context, elem ) && elem.id === m ) { + results.push( elem ); + return results; + } + } + + // Speed-up: Sizzle("TAG") + } else if ( match[2] ) { + push.apply( results, context.getElementsByTagName( selector ) ); + return results; + + // Speed-up: Sizzle(".CLASS") + } else if ( (m = match[3]) && support.getElementsByClassName && context.getElementsByClassName ) { + push.apply( results, context.getElementsByClassName( m ) ); + return results; + } + } + + // QSA path + if ( support.qsa && (!rbuggyQSA || !rbuggyQSA.test( selector )) ) { + nid = old = expando; + newContext = context; + newSelector = nodeType === 9 && selector; + + // qSA works strangely on Element-rooted queries + // We can work around this by specifying an extra ID on the root + // and working up from there (Thanks to Andrew Dupont for the technique) + // IE 8 doesn't work on object elements + if ( nodeType === 1 && context.nodeName.toLowerCase() !== "object" ) { + groups = tokenize( selector ); + + if ( (old = context.getAttribute("id")) ) { + nid = old.replace( rescape, "\\$&" ); + } else { + context.setAttribute( "id", nid ); + } + nid = "[id='" + nid + "'] "; + + i = groups.length; + while ( i-- ) { + groups[i] = nid + toSelector( groups[i] ); + } + newContext = rsibling.test( selector ) && context.parentNode || context; + newSelector = groups.join(","); + } + + if ( newSelector ) { + try { + push.apply( results, + newContext.querySelectorAll( newSelector ) + ); + return results; + } catch(qsaError) { + } finally { + if ( !old ) { + context.removeAttribute("id"); + } + } + } + } + } + + // All others + return select( selector.replace( rtrim, "$1" ), context, results, seed ); +} + +/** + * Create key-value caches of limited size + * @returns {Function(string, Object)} Returns the Object data after storing it on itself with + * property name the (space-suffixed) string and (if the cache is larger than Expr.cacheLength) + * deleting the oldest entry + */ +function createCache() { + var keys = []; + + function cache( key, value ) { + // Use (key + " ") to avoid collision with native prototype properties (see Issue #157) + if ( keys.push( key += " " ) > Expr.cacheLength ) { + // Only keep the most recent entries + delete cache[ keys.shift() ]; + } + return (cache[ key ] = value); + } + return cache; +} + +/** + * Mark a function for special use by Sizzle + * @param {Function} fn The function to mark + */ +function markFunction( fn ) { + fn[ expando ] = true; + return fn; +} + +/** + * Support testing using an element + * @param {Function} fn Passed the created div and expects a boolean result + */ +function assert( fn ) { + var div = document.createElement("div"); + + try { + return !!fn( div ); + } catch (e) { + return false; + } finally { + // Remove from its parent by default + if ( div.parentNode ) { + div.parentNode.removeChild( div ); + } + // release memory in IE + div = null; + } +} + +/** + * Adds the same handler for all of the specified attrs + * @param {String} attrs Pipe-separated list of attributes + * @param {Function} handler The method that will be applied + */ +function addHandle( attrs, handler ) { + var arr = attrs.split("|"), + i = attrs.length; + + while ( i-- ) { + Expr.attrHandle[ arr[i] ] = handler; + } +} + +/** + * Checks document order of two siblings + * @param {Element} a + * @param {Element} b + * @returns {Number} Returns less than 0 if a precedes b, greater than 0 if a follows b + */ +function siblingCheck( a, b ) { + var cur = b && a, + diff = cur && a.nodeType === 1 && b.nodeType === 1 && + ( ~b.sourceIndex || MAX_NEGATIVE ) - + ( ~a.sourceIndex || MAX_NEGATIVE ); + + // Use IE sourceIndex if available on both nodes + if ( diff ) { + return diff; + } + + // Check if b follows a + if ( cur ) { + while ( (cur = cur.nextSibling) ) { + if ( cur === b ) { + return -1; + } + } + } + + return a ? 1 : -1; +} + +/** + * Returns a function to use in pseudos for input types + * @param {String} type + */ +function createInputPseudo( type ) { + return function( elem ) { + var name = elem.nodeName.toLowerCase(); + return name === "input" && elem.type === type; + }; +} + +/** + * Returns a function to use in pseudos for buttons + * @param {String} type + */ +function createButtonPseudo( type ) { + return function( elem ) { + var name = elem.nodeName.toLowerCase(); + return (name === "input" || name === "button") && elem.type === type; + }; +} + +/** + * Returns a function to use in pseudos for positionals + * @param {Function} fn + */ +function createPositionalPseudo( fn ) { + return markFunction(function( argument ) { + argument = +argument; + return markFunction(function( seed, matches ) { + var j, + matchIndexes = fn( [], seed.length, argument ), + i = matchIndexes.length; + + // Match elements found at the specified indexes + while ( i-- ) { + if ( seed[ (j = matchIndexes[i]) ] ) { + seed[j] = !(matches[j] = seed[j]); + } + } + }); + }); +} + +/** + * Detect xml + * @param {Element|Object} elem An element or a document + */ +isXML = Sizzle.isXML = function( elem ) { + // documentElement is verified for cases where it doesn't yet exist + // (such as loading iframes in IE - #4833) + var documentElement = elem && (elem.ownerDocument || elem).documentElement; + return documentElement ? documentElement.nodeName !== "HTML" : false; +}; + +// Expose support vars for convenience +support = Sizzle.support = {}; + +/** + * Sets document-related variables once based on the current document + * @param {Element|Object} [doc] An element or document object to use to set the document + * @returns {Object} Returns the current document + */ +setDocument = Sizzle.setDocument = function( node ) { + var doc = node ? node.ownerDocument || node : preferredDoc, + parent = doc.defaultView; + + // If no document and documentElement is available, return + if ( doc === document || doc.nodeType !== 9 || !doc.documentElement ) { + return document; + } + + // Set our document + document = doc; + docElem = doc.documentElement; + + // Support tests + documentIsHTML = !isXML( doc ); + + // Support: IE>8 + // If iframe document is assigned to "document" variable and if iframe has been reloaded, + // IE will throw "permission denied" error when accessing "document" variable, see jQuery #13936 + // IE6-8 do not support the defaultView property so parent will be undefined + if ( parent && parent.attachEvent && parent !== parent.top ) { + parent.attachEvent( "onbeforeunload", function() { + setDocument(); + }); + } + + /* Attributes + ---------------------------------------------------------------------- */ + + // Support: IE<8 + // Verify that getAttribute really returns attributes and not properties (excepting IE8 booleans) + support.attributes = assert(function( div ) { + div.className = "i"; + return !div.getAttribute("className"); + }); + + /* getElement(s)By* + ---------------------------------------------------------------------- */ + + // Check if getElementsByTagName("*") returns only elements + support.getElementsByTagName = assert(function( div ) { + div.appendChild( doc.createComment("") ); + return !div.getElementsByTagName("*").length; + }); + + // Check if getElementsByClassName can be trusted + support.getElementsByClassName = assert(function( div ) { + div.innerHTML = "
    "; + + // Support: Safari<4 + // Catch class over-caching + div.firstChild.className = "i"; + // Support: Opera<10 + // Catch gEBCN failure to find non-leading classes + return div.getElementsByClassName("i").length === 2; + }); + + // Support: IE<10 + // Check if getElementById returns elements by name + // The broken getElementById methods don't pick up programatically-set names, + // so use a roundabout getElementsByName test + support.getById = assert(function( div ) { + docElem.appendChild( div ).id = expando; + return !doc.getElementsByName || !doc.getElementsByName( expando ).length; + }); + + // ID find and filter + if ( support.getById ) { + Expr.find["ID"] = function( id, context ) { + if ( typeof context.getElementById !== strundefined && documentIsHTML ) { + var m = context.getElementById( id ); + // Check parentNode to catch when Blackberry 4.6 returns + // nodes that are no longer in the document #6963 + return m && m.parentNode ? [m] : []; + } + }; + Expr.filter["ID"] = function( id ) { + var attrId = id.replace( runescape, funescape ); + return function( elem ) { + return elem.getAttribute("id") === attrId; + }; + }; + } else { + // Support: IE6/7 + // getElementById is not reliable as a find shortcut + delete Expr.find["ID"]; + + Expr.filter["ID"] = function( id ) { + var attrId = id.replace( runescape, funescape ); + return function( elem ) { + var node = typeof elem.getAttributeNode !== strundefined && elem.getAttributeNode("id"); + return node && node.value === attrId; + }; + }; + } + + // Tag + Expr.find["TAG"] = support.getElementsByTagName ? + function( tag, context ) { + if ( typeof context.getElementsByTagName !== strundefined ) { + return context.getElementsByTagName( tag ); + } + } : + function( tag, context ) { + var elem, + tmp = [], + i = 0, + results = context.getElementsByTagName( tag ); + + // Filter out possible comments + if ( tag === "*" ) { + while ( (elem = results[i++]) ) { + if ( elem.nodeType === 1 ) { + tmp.push( elem ); + } + } + + return tmp; + } + return results; + }; + + // Class + Expr.find["CLASS"] = support.getElementsByClassName && function( className, context ) { + if ( typeof context.getElementsByClassName !== strundefined && documentIsHTML ) { + return context.getElementsByClassName( className ); + } + }; + + /* QSA/matchesSelector + ---------------------------------------------------------------------- */ + + // QSA and matchesSelector support + + // matchesSelector(:active) reports false when true (IE9/Opera 11.5) + rbuggyMatches = []; + + // qSa(:focus) reports false when true (Chrome 21) + // We allow this because of a bug in IE8/9 that throws an error + // whenever `document.activeElement` is accessed on an iframe + // So, we allow :focus to pass through QSA all the time to avoid the IE error + // See http://bugs.jquery.com/ticket/13378 + rbuggyQSA = []; + + if ( (support.qsa = rnative.test( doc.querySelectorAll )) ) { + // Build QSA regex + // Regex strategy adopted from Diego Perini + assert(function( div ) { + // Select is set to empty string on purpose + // This is to test IE's treatment of not explicitly + // setting a boolean content attribute, + // since its presence should be enough + // http://bugs.jquery.com/ticket/12359 + div.innerHTML = ""; + + // Support: IE8 + // Boolean attributes and "value" are not treated correctly + if ( !div.querySelectorAll("[selected]").length ) { + rbuggyQSA.push( "\\[" + whitespace + "*(?:value|" + booleans + ")" ); + } + + // Webkit/Opera - :checked should return selected option elements + // http://www.w3.org/TR/2011/REC-css3-selectors-20110929/#checked + // IE8 throws error here and will not see later tests + if ( !div.querySelectorAll(":checked").length ) { + rbuggyQSA.push(":checked"); + } + }); + + assert(function( div ) { + + // Support: Opera 10-12/IE8 + // ^= $= *= and empty values + // Should not select anything + // Support: Windows 8 Native Apps + // The type attribute is restricted during .innerHTML assignment + var input = doc.createElement("input"); + input.setAttribute( "type", "hidden" ); + div.appendChild( input ).setAttribute( "t", "" ); + + if ( div.querySelectorAll("[t^='']").length ) { + rbuggyQSA.push( "[*^$]=" + whitespace + "*(?:''|\"\")" ); + } + + // FF 3.5 - :enabled/:disabled and hidden elements (hidden elements are still enabled) + // IE8 throws error here and will not see later tests + if ( !div.querySelectorAll(":enabled").length ) { + rbuggyQSA.push( ":enabled", ":disabled" ); + } + + // Opera 10-11 does not throw on post-comma invalid pseudos + div.querySelectorAll("*,:x"); + rbuggyQSA.push(",.*:"); + }); + } + + if ( (support.matchesSelector = rnative.test( (matches = docElem.webkitMatchesSelector || + docElem.mozMatchesSelector || + docElem.oMatchesSelector || + docElem.msMatchesSelector) )) ) { + + assert(function( div ) { + // Check to see if it's possible to do matchesSelector + // on a disconnected node (IE 9) + support.disconnectedMatch = matches.call( div, "div" ); + + // This should fail with an exception + // Gecko does not error, returns false instead + matches.call( div, "[s!='']:x" ); + rbuggyMatches.push( "!=", pseudos ); + }); + } + + rbuggyQSA = rbuggyQSA.length && new RegExp( rbuggyQSA.join("|") ); + rbuggyMatches = rbuggyMatches.length && new RegExp( rbuggyMatches.join("|") ); + + /* Contains + ---------------------------------------------------------------------- */ + + // Element contains another + // Purposefully does not implement inclusive descendent + // As in, an element does not contain itself + contains = rnative.test( docElem.contains ) || docElem.compareDocumentPosition ? + function( a, b ) { + var adown = a.nodeType === 9 ? a.documentElement : a, + bup = b && b.parentNode; + return a === bup || !!( bup && bup.nodeType === 1 && ( + adown.contains ? + adown.contains( bup ) : + a.compareDocumentPosition && a.compareDocumentPosition( bup ) & 16 + )); + } : + function( a, b ) { + if ( b ) { + while ( (b = b.parentNode) ) { + if ( b === a ) { + return true; + } + } + } + return false; + }; + + /* Sorting + ---------------------------------------------------------------------- */ + + // Document order sorting + sortOrder = docElem.compareDocumentPosition ? + function( a, b ) { + + // Flag for duplicate removal + if ( a === b ) { + hasDuplicate = true; + return 0; + } + + var compare = b.compareDocumentPosition && a.compareDocumentPosition && a.compareDocumentPosition( b ); + + if ( compare ) { + // Disconnected nodes + if ( compare & 1 || + (!support.sortDetached && b.compareDocumentPosition( a ) === compare) ) { + + // Choose the first element that is related to our preferred document + if ( a === doc || contains(preferredDoc, a) ) { + return -1; + } + if ( b === doc || contains(preferredDoc, b) ) { + return 1; + } + + // Maintain original order + return sortInput ? + ( indexOf.call( sortInput, a ) - indexOf.call( sortInput, b ) ) : + 0; + } + + return compare & 4 ? -1 : 1; + } + + // Not directly comparable, sort on existence of method + return a.compareDocumentPosition ? -1 : 1; + } : + function( a, b ) { + var cur, + i = 0, + aup = a.parentNode, + bup = b.parentNode, + ap = [ a ], + bp = [ b ]; + + // Exit early if the nodes are identical + if ( a === b ) { + hasDuplicate = true; + return 0; + + // Parentless nodes are either documents or disconnected + } else if ( !aup || !bup ) { + return a === doc ? -1 : + b === doc ? 1 : + aup ? -1 : + bup ? 1 : + sortInput ? + ( indexOf.call( sortInput, a ) - indexOf.call( sortInput, b ) ) : + 0; + + // If the nodes are siblings, we can do a quick check + } else if ( aup === bup ) { + return siblingCheck( a, b ); + } + + // Otherwise we need full lists of their ancestors for comparison + cur = a; + while ( (cur = cur.parentNode) ) { + ap.unshift( cur ); + } + cur = b; + while ( (cur = cur.parentNode) ) { + bp.unshift( cur ); + } + + // Walk down the tree looking for a discrepancy + while ( ap[i] === bp[i] ) { + i++; + } + + return i ? + // Do a sibling check if the nodes have a common ancestor + siblingCheck( ap[i], bp[i] ) : + + // Otherwise nodes in our document sort first + ap[i] === preferredDoc ? -1 : + bp[i] === preferredDoc ? 1 : + 0; + }; + + return doc; +}; + +Sizzle.matches = function( expr, elements ) { + return Sizzle( expr, null, null, elements ); +}; + +Sizzle.matchesSelector = function( elem, expr ) { + // Set document vars if needed + if ( ( elem.ownerDocument || elem ) !== document ) { + setDocument( elem ); + } + + // Make sure that attribute selectors are quoted + expr = expr.replace( rattributeQuotes, "='$1']" ); + + if ( support.matchesSelector && documentIsHTML && + ( !rbuggyMatches || !rbuggyMatches.test( expr ) ) && + ( !rbuggyQSA || !rbuggyQSA.test( expr ) ) ) { + + try { + var ret = matches.call( elem, expr ); + + // IE 9's matchesSelector returns false on disconnected nodes + if ( ret || support.disconnectedMatch || + // As well, disconnected nodes are said to be in a document + // fragment in IE 9 + elem.document && elem.document.nodeType !== 11 ) { + return ret; + } + } catch(e) {} + } + + return Sizzle( expr, document, null, [elem] ).length > 0; +}; + +Sizzle.contains = function( context, elem ) { + // Set document vars if needed + if ( ( context.ownerDocument || context ) !== document ) { + setDocument( context ); + } + return contains( context, elem ); +}; + +Sizzle.attr = function( elem, name ) { + // Set document vars if needed + if ( ( elem.ownerDocument || elem ) !== document ) { + setDocument( elem ); + } + + var fn = Expr.attrHandle[ name.toLowerCase() ], + // Don't get fooled by Object.prototype properties (jQuery #13807) + val = fn && hasOwn.call( Expr.attrHandle, name.toLowerCase() ) ? + fn( elem, name, !documentIsHTML ) : + undefined; + + return val === undefined ? + support.attributes || !documentIsHTML ? + elem.getAttribute( name ) : + (val = elem.getAttributeNode(name)) && val.specified ? + val.value : + null : + val; +}; + +Sizzle.error = function( msg ) { + throw new Error( "Syntax error, unrecognized expression: " + msg ); +}; + +/** + * Document sorting and removing duplicates + * @param {ArrayLike} results + */ +Sizzle.uniqueSort = function( results ) { + var elem, + duplicates = [], + j = 0, + i = 0; + + // Unless we *know* we can detect duplicates, assume their presence + hasDuplicate = !support.detectDuplicates; + sortInput = !support.sortStable && results.slice( 0 ); + results.sort( sortOrder ); + + if ( hasDuplicate ) { + while ( (elem = results[i++]) ) { + if ( elem === results[ i ] ) { + j = duplicates.push( i ); + } + } + while ( j-- ) { + results.splice( duplicates[ j ], 1 ); + } + } + + return results; +}; + +/** + * Utility function for retrieving the text value of an array of DOM nodes + * @param {Array|Element} elem + */ +getText = Sizzle.getText = function( elem ) { + var node, + ret = "", + i = 0, + nodeType = elem.nodeType; + + if ( !nodeType ) { + // If no nodeType, this is expected to be an array + for ( ; (node = elem[i]); i++ ) { + // Do not traverse comment nodes + ret += getText( node ); + } + } else if ( nodeType === 1 || nodeType === 9 || nodeType === 11 ) { + // Use textContent for elements + // innerText usage removed for consistency of new lines (see #11153) + if ( typeof elem.textContent === "string" ) { + return elem.textContent; + } else { + // Traverse its children + for ( elem = elem.firstChild; elem; elem = elem.nextSibling ) { + ret += getText( elem ); + } + } + } else if ( nodeType === 3 || nodeType === 4 ) { + return elem.nodeValue; + } + // Do not include comment or processing instruction nodes + + return ret; +}; + +Expr = Sizzle.selectors = { + + // Can be adjusted by the user + cacheLength: 50, + + createPseudo: markFunction, + + match: matchExpr, + + attrHandle: {}, + + find: {}, + + relative: { + ">": { dir: "parentNode", first: true }, + " ": { dir: "parentNode" }, + "+": { dir: "previousSibling", first: true }, + "~": { dir: "previousSibling" } + }, + + preFilter: { + "ATTR": function( match ) { + match[1] = match[1].replace( runescape, funescape ); + + // Move the given value to match[3] whether quoted or unquoted + match[3] = ( match[4] || match[5] || "" ).replace( runescape, funescape ); + + if ( match[2] === "~=" ) { + match[3] = " " + match[3] + " "; + } + + return match.slice( 0, 4 ); + }, + + "CHILD": function( match ) { + /* matches from matchExpr["CHILD"] + 1 type (only|nth|...) + 2 what (child|of-type) + 3 argument (even|odd|\d*|\d*n([+-]\d+)?|...) + 4 xn-component of xn+y argument ([+-]?\d*n|) + 5 sign of xn-component + 6 x of xn-component + 7 sign of y-component + 8 y of y-component + */ + match[1] = match[1].toLowerCase(); + + if ( match[1].slice( 0, 3 ) === "nth" ) { + // nth-* requires argument + if ( !match[3] ) { + Sizzle.error( match[0] ); + } + + // numeric x and y parameters for Expr.filter.CHILD + // remember that false/true cast respectively to 0/1 + match[4] = +( match[4] ? match[5] + (match[6] || 1) : 2 * ( match[3] === "even" || match[3] === "odd" ) ); + match[5] = +( ( match[7] + match[8] ) || match[3] === "odd" ); + + // other types prohibit arguments + } else if ( match[3] ) { + Sizzle.error( match[0] ); + } + + return match; + }, + + "PSEUDO": function( match ) { + var excess, + unquoted = !match[5] && match[2]; + + if ( matchExpr["CHILD"].test( match[0] ) ) { + return null; + } + + // Accept quoted arguments as-is + if ( match[3] && match[4] !== undefined ) { + match[2] = match[4]; + + // Strip excess characters from unquoted arguments + } else if ( unquoted && rpseudo.test( unquoted ) && + // Get excess from tokenize (recursively) + (excess = tokenize( unquoted, true )) && + // advance to the next closing parenthesis + (excess = unquoted.indexOf( ")", unquoted.length - excess ) - unquoted.length) ) { + + // excess is a negative index + match[0] = match[0].slice( 0, excess ); + match[2] = unquoted.slice( 0, excess ); + } + + // Return only captures needed by the pseudo filter method (type and argument) + return match.slice( 0, 3 ); + } + }, + + filter: { + + "TAG": function( nodeNameSelector ) { + var nodeName = nodeNameSelector.replace( runescape, funescape ).toLowerCase(); + return nodeNameSelector === "*" ? + function() { return true; } : + function( elem ) { + return elem.nodeName && elem.nodeName.toLowerCase() === nodeName; + }; + }, + + "CLASS": function( className ) { + var pattern = classCache[ className + " " ]; + + return pattern || + (pattern = new RegExp( "(^|" + whitespace + ")" + className + "(" + whitespace + "|$)" )) && + classCache( className, function( elem ) { + return pattern.test( typeof elem.className === "string" && elem.className || typeof elem.getAttribute !== strundefined && elem.getAttribute("class") || "" ); + }); + }, + + "ATTR": function( name, operator, check ) { + return function( elem ) { + var result = Sizzle.attr( elem, name ); + + if ( result == null ) { + return operator === "!="; + } + if ( !operator ) { + return true; + } + + result += ""; + + return operator === "=" ? result === check : + operator === "!=" ? result !== check : + operator === "^=" ? check && result.indexOf( check ) === 0 : + operator === "*=" ? check && result.indexOf( check ) > -1 : + operator === "$=" ? check && result.slice( -check.length ) === check : + operator === "~=" ? ( " " + result + " " ).indexOf( check ) > -1 : + operator === "|=" ? result === check || result.slice( 0, check.length + 1 ) === check + "-" : + false; + }; + }, + + "CHILD": function( type, what, argument, first, last ) { + var simple = type.slice( 0, 3 ) !== "nth", + forward = type.slice( -4 ) !== "last", + ofType = what === "of-type"; + + return first === 1 && last === 0 ? + + // Shortcut for :nth-*(n) + function( elem ) { + return !!elem.parentNode; + } : + + function( elem, context, xml ) { + var cache, outerCache, node, diff, nodeIndex, start, + dir = simple !== forward ? "nextSibling" : "previousSibling", + parent = elem.parentNode, + name = ofType && elem.nodeName.toLowerCase(), + useCache = !xml && !ofType; + + if ( parent ) { + + // :(first|last|only)-(child|of-type) + if ( simple ) { + while ( dir ) { + node = elem; + while ( (node = node[ dir ]) ) { + if ( ofType ? node.nodeName.toLowerCase() === name : node.nodeType === 1 ) { + return false; + } + } + // Reverse direction for :only-* (if we haven't yet done so) + start = dir = type === "only" && !start && "nextSibling"; + } + return true; + } + + start = [ forward ? parent.firstChild : parent.lastChild ]; + + // non-xml :nth-child(...) stores cache data on `parent` + if ( forward && useCache ) { + // Seek `elem` from a previously-cached index + outerCache = parent[ expando ] || (parent[ expando ] = {}); + cache = outerCache[ type ] || []; + nodeIndex = cache[0] === dirruns && cache[1]; + diff = cache[0] === dirruns && cache[2]; + node = nodeIndex && parent.childNodes[ nodeIndex ]; + + while ( (node = ++nodeIndex && node && node[ dir ] || + + // Fallback to seeking `elem` from the start + (diff = nodeIndex = 0) || start.pop()) ) { + + // When found, cache indexes on `parent` and break + if ( node.nodeType === 1 && ++diff && node === elem ) { + outerCache[ type ] = [ dirruns, nodeIndex, diff ]; + break; + } + } + + // Use previously-cached element index if available + } else if ( useCache && (cache = (elem[ expando ] || (elem[ expando ] = {}))[ type ]) && cache[0] === dirruns ) { + diff = cache[1]; + + // xml :nth-child(...) or :nth-last-child(...) or :nth(-last)?-of-type(...) + } else { + // Use the same loop as above to seek `elem` from the start + while ( (node = ++nodeIndex && node && node[ dir ] || + (diff = nodeIndex = 0) || start.pop()) ) { + + if ( ( ofType ? node.nodeName.toLowerCase() === name : node.nodeType === 1 ) && ++diff ) { + // Cache the index of each encountered element + if ( useCache ) { + (node[ expando ] || (node[ expando ] = {}))[ type ] = [ dirruns, diff ]; + } + + if ( node === elem ) { + break; + } + } + } + } + + // Incorporate the offset, then check against cycle size + diff -= last; + return diff === first || ( diff % first === 0 && diff / first >= 0 ); + } + }; + }, + + "PSEUDO": function( pseudo, argument ) { + // pseudo-class names are case-insensitive + // http://www.w3.org/TR/selectors/#pseudo-classes + // Prioritize by case sensitivity in case custom pseudos are added with uppercase letters + // Remember that setFilters inherits from pseudos + var args, + fn = Expr.pseudos[ pseudo ] || Expr.setFilters[ pseudo.toLowerCase() ] || + Sizzle.error( "unsupported pseudo: " + pseudo ); + + // The user may use createPseudo to indicate that + // arguments are needed to create the filter function + // just as Sizzle does + if ( fn[ expando ] ) { + return fn( argument ); + } + + // But maintain support for old signatures + if ( fn.length > 1 ) { + args = [ pseudo, pseudo, "", argument ]; + return Expr.setFilters.hasOwnProperty( pseudo.toLowerCase() ) ? + markFunction(function( seed, matches ) { + var idx, + matched = fn( seed, argument ), + i = matched.length; + while ( i-- ) { + idx = indexOf.call( seed, matched[i] ); + seed[ idx ] = !( matches[ idx ] = matched[i] ); + } + }) : + function( elem ) { + return fn( elem, 0, args ); + }; + } + + return fn; + } + }, + + pseudos: { + // Potentially complex pseudos + "not": markFunction(function( selector ) { + // Trim the selector passed to compile + // to avoid treating leading and trailing + // spaces as combinators + var input = [], + results = [], + matcher = compile( selector.replace( rtrim, "$1" ) ); + + return matcher[ expando ] ? + markFunction(function( seed, matches, context, xml ) { + var elem, + unmatched = matcher( seed, null, xml, [] ), + i = seed.length; + + // Match elements unmatched by `matcher` + while ( i-- ) { + if ( (elem = unmatched[i]) ) { + seed[i] = !(matches[i] = elem); + } + } + }) : + function( elem, context, xml ) { + input[0] = elem; + matcher( input, null, xml, results ); + return !results.pop(); + }; + }), + + "has": markFunction(function( selector ) { + return function( elem ) { + return Sizzle( selector, elem ).length > 0; + }; + }), + + "contains": markFunction(function( text ) { + return function( elem ) { + return ( elem.textContent || elem.innerText || getText( elem ) ).indexOf( text ) > -1; + }; + }), + + // "Whether an element is represented by a :lang() selector + // is based solely on the element's language value + // being equal to the identifier C, + // or beginning with the identifier C immediately followed by "-". + // The matching of C against the element's language value is performed case-insensitively. + // The identifier C does not have to be a valid language name." + // http://www.w3.org/TR/selectors/#lang-pseudo + "lang": markFunction( function( lang ) { + // lang value must be a valid identifier + if ( !ridentifier.test(lang || "") ) { + Sizzle.error( "unsupported lang: " + lang ); + } + lang = lang.replace( runescape, funescape ).toLowerCase(); + return function( elem ) { + var elemLang; + do { + if ( (elemLang = documentIsHTML ? + elem.lang : + elem.getAttribute("xml:lang") || elem.getAttribute("lang")) ) { + + elemLang = elemLang.toLowerCase(); + return elemLang === lang || elemLang.indexOf( lang + "-" ) === 0; + } + } while ( (elem = elem.parentNode) && elem.nodeType === 1 ); + return false; + }; + }), + + // Miscellaneous + "target": function( elem ) { + var hash = window.location && window.location.hash; + return hash && hash.slice( 1 ) === elem.id; + }, + + "root": function( elem ) { + return elem === docElem; + }, + + "focus": function( elem ) { + return elem === document.activeElement && (!document.hasFocus || document.hasFocus()) && !!(elem.type || elem.href || ~elem.tabIndex); + }, + + // Boolean properties + "enabled": function( elem ) { + return elem.disabled === false; + }, + + "disabled": function( elem ) { + return elem.disabled === true; + }, + + "checked": function( elem ) { + // In CSS3, :checked should return both checked and selected elements + // http://www.w3.org/TR/2011/REC-css3-selectors-20110929/#checked + var nodeName = elem.nodeName.toLowerCase(); + return (nodeName === "input" && !!elem.checked) || (nodeName === "option" && !!elem.selected); + }, + + "selected": function( elem ) { + // Accessing this property makes selected-by-default + // options in Safari work properly + if ( elem.parentNode ) { + elem.parentNode.selectedIndex; + } + + return elem.selected === true; + }, + + // Contents + "empty": function( elem ) { + // http://www.w3.org/TR/selectors/#empty-pseudo + // :empty is only affected by element nodes and content nodes(including text(3), cdata(4)), + // not comment, processing instructions, or others + // Thanks to Diego Perini for the nodeName shortcut + // Greater than "@" means alpha characters (specifically not starting with "#" or "?") + for ( elem = elem.firstChild; elem; elem = elem.nextSibling ) { + if ( elem.nodeName > "@" || elem.nodeType === 3 || elem.nodeType === 4 ) { + return false; + } + } + return true; + }, + + "parent": function( elem ) { + return !Expr.pseudos["empty"]( elem ); + }, + + // Element/input types + "header": function( elem ) { + return rheader.test( elem.nodeName ); + }, + + "input": function( elem ) { + return rinputs.test( elem.nodeName ); + }, + + "button": function( elem ) { + var name = elem.nodeName.toLowerCase(); + return name === "input" && elem.type === "button" || name === "button"; + }, + + "text": function( elem ) { + var attr; + // IE6 and 7 will map elem.type to 'text' for new HTML5 types (search, etc) + // use getAttribute instead to test this case + return elem.nodeName.toLowerCase() === "input" && + elem.type === "text" && + ( (attr = elem.getAttribute("type")) == null || attr.toLowerCase() === elem.type ); + }, + + // Position-in-collection + "first": createPositionalPseudo(function() { + return [ 0 ]; + }), + + "last": createPositionalPseudo(function( matchIndexes, length ) { + return [ length - 1 ]; + }), + + "eq": createPositionalPseudo(function( matchIndexes, length, argument ) { + return [ argument < 0 ? argument + length : argument ]; + }), + + "even": createPositionalPseudo(function( matchIndexes, length ) { + var i = 0; + for ( ; i < length; i += 2 ) { + matchIndexes.push( i ); + } + return matchIndexes; + }), + + "odd": createPositionalPseudo(function( matchIndexes, length ) { + var i = 1; + for ( ; i < length; i += 2 ) { + matchIndexes.push( i ); + } + return matchIndexes; + }), + + "lt": createPositionalPseudo(function( matchIndexes, length, argument ) { + var i = argument < 0 ? argument + length : argument; + for ( ; --i >= 0; ) { + matchIndexes.push( i ); + } + return matchIndexes; + }), + + "gt": createPositionalPseudo(function( matchIndexes, length, argument ) { + var i = argument < 0 ? argument + length : argument; + for ( ; ++i < length; ) { + matchIndexes.push( i ); + } + return matchIndexes; + }) + } +}; + +Expr.pseudos["nth"] = Expr.pseudos["eq"]; + +// Add button/input type pseudos +for ( i in { radio: true, checkbox: true, file: true, password: true, image: true } ) { + Expr.pseudos[ i ] = createInputPseudo( i ); +} +for ( i in { submit: true, reset: true } ) { + Expr.pseudos[ i ] = createButtonPseudo( i ); +} + +// Easy API for creating new setFilters +function setFilters() {} +setFilters.prototype = Expr.filters = Expr.pseudos; +Expr.setFilters = new setFilters(); + +function tokenize( selector, parseOnly ) { + var matched, match, tokens, type, + soFar, groups, preFilters, + cached = tokenCache[ selector + " " ]; + + if ( cached ) { + return parseOnly ? 0 : cached.slice( 0 ); + } + + soFar = selector; + groups = []; + preFilters = Expr.preFilter; + + while ( soFar ) { + + // Comma and first run + if ( !matched || (match = rcomma.exec( soFar )) ) { + if ( match ) { + // Don't consume trailing commas as valid + soFar = soFar.slice( match[0].length ) || soFar; + } + groups.push( tokens = [] ); + } + + matched = false; + + // Combinators + if ( (match = rcombinators.exec( soFar )) ) { + matched = match.shift(); + tokens.push({ + value: matched, + // Cast descendant combinators to space + type: match[0].replace( rtrim, " " ) + }); + soFar = soFar.slice( matched.length ); + } + + // Filters + for ( type in Expr.filter ) { + if ( (match = matchExpr[ type ].exec( soFar )) && (!preFilters[ type ] || + (match = preFilters[ type ]( match ))) ) { + matched = match.shift(); + tokens.push({ + value: matched, + type: type, + matches: match + }); + soFar = soFar.slice( matched.length ); + } + } + + if ( !matched ) { + break; + } + } + + // Return the length of the invalid excess + // if we're just parsing + // Otherwise, throw an error or return tokens + return parseOnly ? + soFar.length : + soFar ? + Sizzle.error( selector ) : + // Cache the tokens + tokenCache( selector, groups ).slice( 0 ); +} + +function toSelector( tokens ) { + var i = 0, + len = tokens.length, + selector = ""; + for ( ; i < len; i++ ) { + selector += tokens[i].value; + } + return selector; +} + +function addCombinator( matcher, combinator, base ) { + var dir = combinator.dir, + checkNonElements = base && dir === "parentNode", + doneName = done++; + + return combinator.first ? + // Check against closest ancestor/preceding element + function( elem, context, xml ) { + while ( (elem = elem[ dir ]) ) { + if ( elem.nodeType === 1 || checkNonElements ) { + return matcher( elem, context, xml ); + } + } + } : + + // Check against all ancestor/preceding elements + function( elem, context, xml ) { + var data, cache, outerCache, + dirkey = dirruns + " " + doneName; + + // We can't set arbitrary data on XML nodes, so they don't benefit from dir caching + if ( xml ) { + while ( (elem = elem[ dir ]) ) { + if ( elem.nodeType === 1 || checkNonElements ) { + if ( matcher( elem, context, xml ) ) { + return true; + } + } + } + } else { + while ( (elem = elem[ dir ]) ) { + if ( elem.nodeType === 1 || checkNonElements ) { + outerCache = elem[ expando ] || (elem[ expando ] = {}); + if ( (cache = outerCache[ dir ]) && cache[0] === dirkey ) { + if ( (data = cache[1]) === true || data === cachedruns ) { + return data === true; + } + } else { + cache = outerCache[ dir ] = [ dirkey ]; + cache[1] = matcher( elem, context, xml ) || cachedruns; + if ( cache[1] === true ) { + return true; + } + } + } + } + } + }; +} + +function elementMatcher( matchers ) { + return matchers.length > 1 ? + function( elem, context, xml ) { + var i = matchers.length; + while ( i-- ) { + if ( !matchers[i]( elem, context, xml ) ) { + return false; + } + } + return true; + } : + matchers[0]; +} + +function condense( unmatched, map, filter, context, xml ) { + var elem, + newUnmatched = [], + i = 0, + len = unmatched.length, + mapped = map != null; + + for ( ; i < len; i++ ) { + if ( (elem = unmatched[i]) ) { + if ( !filter || filter( elem, context, xml ) ) { + newUnmatched.push( elem ); + if ( mapped ) { + map.push( i ); + } + } + } + } + + return newUnmatched; +} + +function setMatcher( preFilter, selector, matcher, postFilter, postFinder, postSelector ) { + if ( postFilter && !postFilter[ expando ] ) { + postFilter = setMatcher( postFilter ); + } + if ( postFinder && !postFinder[ expando ] ) { + postFinder = setMatcher( postFinder, postSelector ); + } + return markFunction(function( seed, results, context, xml ) { + var temp, i, elem, + preMap = [], + postMap = [], + preexisting = results.length, + + // Get initial elements from seed or context + elems = seed || multipleContexts( selector || "*", context.nodeType ? [ context ] : context, [] ), + + // Prefilter to get matcher input, preserving a map for seed-results synchronization + matcherIn = preFilter && ( seed || !selector ) ? + condense( elems, preMap, preFilter, context, xml ) : + elems, + + matcherOut = matcher ? + // If we have a postFinder, or filtered seed, or non-seed postFilter or preexisting results, + postFinder || ( seed ? preFilter : preexisting || postFilter ) ? + + // ...intermediate processing is necessary + [] : + + // ...otherwise use results directly + results : + matcherIn; + + // Find primary matches + if ( matcher ) { + matcher( matcherIn, matcherOut, context, xml ); + } + + // Apply postFilter + if ( postFilter ) { + temp = condense( matcherOut, postMap ); + postFilter( temp, [], context, xml ); + + // Un-match failing elements by moving them back to matcherIn + i = temp.length; + while ( i-- ) { + if ( (elem = temp[i]) ) { + matcherOut[ postMap[i] ] = !(matcherIn[ postMap[i] ] = elem); + } + } + } + + if ( seed ) { + if ( postFinder || preFilter ) { + if ( postFinder ) { + // Get the final matcherOut by condensing this intermediate into postFinder contexts + temp = []; + i = matcherOut.length; + while ( i-- ) { + if ( (elem = matcherOut[i]) ) { + // Restore matcherIn since elem is not yet a final match + temp.push( (matcherIn[i] = elem) ); + } + } + postFinder( null, (matcherOut = []), temp, xml ); + } + + // Move matched elements from seed to results to keep them synchronized + i = matcherOut.length; + while ( i-- ) { + if ( (elem = matcherOut[i]) && + (temp = postFinder ? indexOf.call( seed, elem ) : preMap[i]) > -1 ) { + + seed[temp] = !(results[temp] = elem); + } + } + } + + // Add elements to results, through postFinder if defined + } else { + matcherOut = condense( + matcherOut === results ? + matcherOut.splice( preexisting, matcherOut.length ) : + matcherOut + ); + if ( postFinder ) { + postFinder( null, results, matcherOut, xml ); + } else { + push.apply( results, matcherOut ); + } + } + }); +} + +function matcherFromTokens( tokens ) { + var checkContext, matcher, j, + len = tokens.length, + leadingRelative = Expr.relative[ tokens[0].type ], + implicitRelative = leadingRelative || Expr.relative[" "], + i = leadingRelative ? 1 : 0, + + // The foundational matcher ensures that elements are reachable from top-level context(s) + matchContext = addCombinator( function( elem ) { + return elem === checkContext; + }, implicitRelative, true ), + matchAnyContext = addCombinator( function( elem ) { + return indexOf.call( checkContext, elem ) > -1; + }, implicitRelative, true ), + matchers = [ function( elem, context, xml ) { + return ( !leadingRelative && ( xml || context !== outermostContext ) ) || ( + (checkContext = context).nodeType ? + matchContext( elem, context, xml ) : + matchAnyContext( elem, context, xml ) ); + } ]; + + for ( ; i < len; i++ ) { + if ( (matcher = Expr.relative[ tokens[i].type ]) ) { + matchers = [ addCombinator(elementMatcher( matchers ), matcher) ]; + } else { + matcher = Expr.filter[ tokens[i].type ].apply( null, tokens[i].matches ); + + // Return special upon seeing a positional matcher + if ( matcher[ expando ] ) { + // Find the next relative operator (if any) for proper handling + j = ++i; + for ( ; j < len; j++ ) { + if ( Expr.relative[ tokens[j].type ] ) { + break; + } + } + return setMatcher( + i > 1 && elementMatcher( matchers ), + i > 1 && toSelector( + // If the preceding token was a descendant combinator, insert an implicit any-element `*` + tokens.slice( 0, i - 1 ).concat({ value: tokens[ i - 2 ].type === " " ? "*" : "" }) + ).replace( rtrim, "$1" ), + matcher, + i < j && matcherFromTokens( tokens.slice( i, j ) ), + j < len && matcherFromTokens( (tokens = tokens.slice( j )) ), + j < len && toSelector( tokens ) + ); + } + matchers.push( matcher ); + } + } + + return elementMatcher( matchers ); +} + +function matcherFromGroupMatchers( elementMatchers, setMatchers ) { + // A counter to specify which element is currently being matched + var matcherCachedRuns = 0, + bySet = setMatchers.length > 0, + byElement = elementMatchers.length > 0, + superMatcher = function( seed, context, xml, results, expandContext ) { + var elem, j, matcher, + setMatched = [], + matchedCount = 0, + i = "0", + unmatched = seed && [], + outermost = expandContext != null, + contextBackup = outermostContext, + // We must always have either seed elements or context + elems = seed || byElement && Expr.find["TAG"]( "*", expandContext && context.parentNode || context ), + // Use integer dirruns iff this is the outermost matcher + dirrunsUnique = (dirruns += contextBackup == null ? 1 : Math.random() || 0.1); + + if ( outermost ) { + outermostContext = context !== document && context; + cachedruns = matcherCachedRuns; + } + + // Add elements passing elementMatchers directly to results + // Keep `i` a string if there are no elements so `matchedCount` will be "00" below + for ( ; (elem = elems[i]) != null; i++ ) { + if ( byElement && elem ) { + j = 0; + while ( (matcher = elementMatchers[j++]) ) { + if ( matcher( elem, context, xml ) ) { + results.push( elem ); + break; + } + } + if ( outermost ) { + dirruns = dirrunsUnique; + cachedruns = ++matcherCachedRuns; + } + } + + // Track unmatched elements for set filters + if ( bySet ) { + // They will have gone through all possible matchers + if ( (elem = !matcher && elem) ) { + matchedCount--; + } + + // Lengthen the array for every element, matched or not + if ( seed ) { + unmatched.push( elem ); + } + } + } + + // Apply set filters to unmatched elements + matchedCount += i; + if ( bySet && i !== matchedCount ) { + j = 0; + while ( (matcher = setMatchers[j++]) ) { + matcher( unmatched, setMatched, context, xml ); + } + + if ( seed ) { + // Reintegrate element matches to eliminate the need for sorting + if ( matchedCount > 0 ) { + while ( i-- ) { + if ( !(unmatched[i] || setMatched[i]) ) { + setMatched[i] = pop.call( results ); + } + } + } + + // Discard index placeholder values to get only actual matches + setMatched = condense( setMatched ); + } + + // Add matches to results + push.apply( results, setMatched ); + + // Seedless set matches succeeding multiple successful matchers stipulate sorting + if ( outermost && !seed && setMatched.length > 0 && + ( matchedCount + setMatchers.length ) > 1 ) { + + Sizzle.uniqueSort( results ); + } + } + + // Override manipulation of globals by nested matchers + if ( outermost ) { + dirruns = dirrunsUnique; + outermostContext = contextBackup; + } + + return unmatched; + }; + + return bySet ? + markFunction( superMatcher ) : + superMatcher; +} + +compile = Sizzle.compile = function( selector, group /* Internal Use Only */ ) { + var i, + setMatchers = [], + elementMatchers = [], + cached = compilerCache[ selector + " " ]; + + if ( !cached ) { + // Generate a function of recursive functions that can be used to check each element + if ( !group ) { + group = tokenize( selector ); + } + i = group.length; + while ( i-- ) { + cached = matcherFromTokens( group[i] ); + if ( cached[ expando ] ) { + setMatchers.push( cached ); + } else { + elementMatchers.push( cached ); + } + } + + // Cache the compiled function + cached = compilerCache( selector, matcherFromGroupMatchers( elementMatchers, setMatchers ) ); + } + return cached; +}; + +function multipleContexts( selector, contexts, results ) { + var i = 0, + len = contexts.length; + for ( ; i < len; i++ ) { + Sizzle( selector, contexts[i], results ); + } + return results; +} + +function select( selector, context, results, seed ) { + var i, tokens, token, type, find, + match = tokenize( selector ); + + if ( !seed ) { + // Try to minimize operations if there is only one group + if ( match.length === 1 ) { + + // Take a shortcut and set the context if the root selector is an ID + tokens = match[0] = match[0].slice( 0 ); + if ( tokens.length > 2 && (token = tokens[0]).type === "ID" && + support.getById && context.nodeType === 9 && documentIsHTML && + Expr.relative[ tokens[1].type ] ) { + + context = ( Expr.find["ID"]( token.matches[0].replace(runescape, funescape), context ) || [] )[0]; + if ( !context ) { + return results; + } + selector = selector.slice( tokens.shift().value.length ); + } + + // Fetch a seed set for right-to-left matching + i = matchExpr["needsContext"].test( selector ) ? 0 : tokens.length; + while ( i-- ) { + token = tokens[i]; + + // Abort if we hit a combinator + if ( Expr.relative[ (type = token.type) ] ) { + break; + } + if ( (find = Expr.find[ type ]) ) { + // Search, expanding context for leading sibling combinators + if ( (seed = find( + token.matches[0].replace( runescape, funescape ), + rsibling.test( tokens[0].type ) && context.parentNode || context + )) ) { + + // If seed is empty or no tokens remain, we can return early + tokens.splice( i, 1 ); + selector = seed.length && toSelector( tokens ); + if ( !selector ) { + push.apply( results, seed ); + return results; + } + + break; + } + } + } + } + } + + // Compile and execute a filtering function + // Provide `match` to avoid retokenization if we modified the selector above + compile( selector, match )( + seed, + context, + !documentIsHTML, + results, + rsibling.test( selector ) + ); + return results; +} + +// One-time assignments + +// Sort stability +support.sortStable = expando.split("").sort( sortOrder ).join("") === expando; + +// Support: Chrome<14 +// Always assume duplicates if they aren't passed to the comparison function +support.detectDuplicates = hasDuplicate; + +// Initialize against the default document +setDocument(); + +// Support: Webkit<537.32 - Safari 6.0.3/Chrome 25 (fixed in Chrome 27) +// Detached nodes confoundingly follow *each other* +support.sortDetached = assert(function( div1 ) { + // Should return 1, but returns 4 (following) + return div1.compareDocumentPosition( document.createElement("div") ) & 1; +}); + +// Support: IE<8 +// Prevent attribute/property "interpolation" +// http://msdn.microsoft.com/en-us/library/ms536429%28VS.85%29.aspx +if ( !assert(function( div ) { + div.innerHTML = "    "; + return div.firstChild.getAttribute("href") === "#" ; +}) ) { + addHandle( "type|href|height|width", function( elem, name, isXML ) { + if ( !isXML ) { + return elem.getAttribute( name, name.toLowerCase() === "type" ? 1 : 2 ); + } + }); +} + +// Support: IE<9 +// Use defaultValue in place of getAttribute("value") +if ( !support.attributes || !assert(function( div ) { + div.innerHTML = ""; + div.firstChild.setAttribute( "value", "" ); + return div.firstChild.getAttribute( "value" ) === ""; +}) ) { + addHandle( "value", function( elem, name, isXML ) { + if ( !isXML && elem.nodeName.toLowerCase() === "input" ) { + return elem.defaultValue; + } + }); +} + +// Support: IE<9 +// Use getAttributeNode to fetch booleans when getAttribute lies +if ( !assert(function( div ) { + return div.getAttribute("disabled") == null; +}) ) { + addHandle( booleans, function( elem, name, isXML ) { + var val; + if ( !isXML ) { + return (val = elem.getAttributeNode( name )) && val.specified ? + val.value : + elem[ name ] === true ? name.toLowerCase() : null; + } + }); +} + +jQuery.find = Sizzle; +jQuery.expr = Sizzle.selectors; +jQuery.expr[":"] = jQuery.expr.pseudos; +jQuery.unique = Sizzle.uniqueSort; +jQuery.text = Sizzle.getText; +jQuery.isXMLDoc = Sizzle.isXML; +jQuery.contains = Sizzle.contains; + + +})( window ); +// String to Object options format cache +var optionsCache = {}; + +// Convert String-formatted options into Object-formatted ones and store in cache +function createOptions( options ) { + var object = optionsCache[ options ] = {}; + jQuery.each( options.match( core_rnotwhite ) || [], function( _, flag ) { + object[ flag ] = true; + }); + return object; +} + +/* + * Create a callback list using the following parameters: + * + * options: an optional list of space-separated options that will change how + * the callback list behaves or a more traditional option object + * + * By default a callback list will act like an event callback list and can be + * "fired" multiple times. + * + * Possible options: + * + * once: will ensure the callback list can only be fired once (like a Deferred) + * + * memory: will keep track of previous values and will call any callback added + * after the list has been fired right away with the latest "memorized" + * values (like a Deferred) + * + * unique: will ensure a callback can only be added once (no duplicate in the list) + * + * stopOnFalse: interrupt callings when a callback returns false + * + */ +jQuery.Callbacks = function( options ) { + + // Convert options from String-formatted to Object-formatted if needed + // (we check in cache first) + options = typeof options === "string" ? + ( optionsCache[ options ] || createOptions( options ) ) : + jQuery.extend( {}, options ); + + var // Flag to know if list is currently firing + firing, + // Last fire value (for non-forgettable lists) + memory, + // Flag to know if list was already fired + fired, + // End of the loop when firing + firingLength, + // Index of currently firing callback (modified by remove if needed) + firingIndex, + // First callback to fire (used internally by add and fireWith) + firingStart, + // Actual callback list + list = [], + // Stack of fire calls for repeatable lists + stack = !options.once && [], + // Fire callbacks + fire = function( data ) { + memory = options.memory && data; + fired = true; + firingIndex = firingStart || 0; + firingStart = 0; + firingLength = list.length; + firing = true; + for ( ; list && firingIndex < firingLength; firingIndex++ ) { + if ( list[ firingIndex ].apply( data[ 0 ], data[ 1 ] ) === false && options.stopOnFalse ) { + memory = false; // To prevent further calls using add + break; + } + } + firing = false; + if ( list ) { + if ( stack ) { + if ( stack.length ) { + fire( stack.shift() ); + } + } else if ( memory ) { + list = []; + } else { + self.disable(); + } + } + }, + // Actual Callbacks object + self = { + // Add a callback or a collection of callbacks to the list + add: function() { + if ( list ) { + // First, we save the current length + var start = list.length; + (function add( args ) { + jQuery.each( args, function( _, arg ) { + var type = jQuery.type( arg ); + if ( type === "function" ) { + if ( !options.unique || !self.has( arg ) ) { + list.push( arg ); + } + } else if ( arg && arg.length && type !== "string" ) { + // Inspect recursively + add( arg ); + } + }); + })( arguments ); + // Do we need to add the callbacks to the + // current firing batch? + if ( firing ) { + firingLength = list.length; + // With memory, if we're not firing then + // we should call right away + } else if ( memory ) { + firingStart = start; + fire( memory ); + } + } + return this; + }, + // Remove a callback from the list + remove: function() { + if ( list ) { + jQuery.each( arguments, function( _, arg ) { + var index; + while( ( index = jQuery.inArray( arg, list, index ) ) > -1 ) { + list.splice( index, 1 ); + // Handle firing indexes + if ( firing ) { + if ( index <= firingLength ) { + firingLength--; + } + if ( index <= firingIndex ) { + firingIndex--; + } + } + } + }); + } + return this; + }, + // Check if a given callback is in the list. + // If no argument is given, return whether or not list has callbacks attached. + has: function( fn ) { + return fn ? jQuery.inArray( fn, list ) > -1 : !!( list && list.length ); + }, + // Remove all callbacks from the list + empty: function() { + list = []; + firingLength = 0; + return this; + }, + // Have the list do nothing anymore + disable: function() { + list = stack = memory = undefined; + return this; + }, + // Is it disabled? + disabled: function() { + return !list; + }, + // Lock the list in its current state + lock: function() { + stack = undefined; + if ( !memory ) { + self.disable(); + } + return this; + }, + // Is it locked? + locked: function() { + return !stack; + }, + // Call all callbacks with the given context and arguments + fireWith: function( context, args ) { + if ( list && ( !fired || stack ) ) { + args = args || []; + args = [ context, args.slice ? args.slice() : args ]; + if ( firing ) { + stack.push( args ); + } else { + fire( args ); + } + } + return this; + }, + // Call all the callbacks with the given arguments + fire: function() { + self.fireWith( this, arguments ); + return this; + }, + // To know if the callbacks have already been called at least once + fired: function() { + return !!fired; + } + }; + + return self; +}; +jQuery.extend({ + + Deferred: function( func ) { + var tuples = [ + // action, add listener, listener list, final state + [ "resolve", "done", jQuery.Callbacks("once memory"), "resolved" ], + [ "reject", "fail", jQuery.Callbacks("once memory"), "rejected" ], + [ "notify", "progress", jQuery.Callbacks("memory") ] + ], + state = "pending", + promise = { + state: function() { + return state; + }, + always: function() { + deferred.done( arguments ).fail( arguments ); + return this; + }, + then: function( /* fnDone, fnFail, fnProgress */ ) { + var fns = arguments; + return jQuery.Deferred(function( newDefer ) { + jQuery.each( tuples, function( i, tuple ) { + var action = tuple[ 0 ], + fn = jQuery.isFunction( fns[ i ] ) && fns[ i ]; + // deferred[ done | fail | progress ] for forwarding actions to newDefer + deferred[ tuple[1] ](function() { + var returned = fn && fn.apply( this, arguments ); + if ( returned && jQuery.isFunction( returned.promise ) ) { + returned.promise() + .done( newDefer.resolve ) + .fail( newDefer.reject ) + .progress( newDefer.notify ); + } else { + newDefer[ action + "With" ]( this === promise ? newDefer.promise() : this, fn ? [ returned ] : arguments ); + } + }); + }); + fns = null; + }).promise(); + }, + // Get a promise for this deferred + // If obj is provided, the promise aspect is added to the object + promise: function( obj ) { + return obj != null ? jQuery.extend( obj, promise ) : promise; + } + }, + deferred = {}; + + // Keep pipe for back-compat + promise.pipe = promise.then; + + // Add list-specific methods + jQuery.each( tuples, function( i, tuple ) { + var list = tuple[ 2 ], + stateString = tuple[ 3 ]; + + // promise[ done | fail | progress ] = list.add + promise[ tuple[1] ] = list.add; + + // Handle state + if ( stateString ) { + list.add(function() { + // state = [ resolved | rejected ] + state = stateString; + + // [ reject_list | resolve_list ].disable; progress_list.lock + }, tuples[ i ^ 1 ][ 2 ].disable, tuples[ 2 ][ 2 ].lock ); + } + + // deferred[ resolve | reject | notify ] + deferred[ tuple[0] ] = function() { + deferred[ tuple[0] + "With" ]( this === deferred ? promise : this, arguments ); + return this; + }; + deferred[ tuple[0] + "With" ] = list.fireWith; + }); + + // Make the deferred a promise + promise.promise( deferred ); + + // Call given func if any + if ( func ) { + func.call( deferred, deferred ); + } + + // All done! + return deferred; + }, + + // Deferred helper + when: function( subordinate /* , ..., subordinateN */ ) { + var i = 0, + resolveValues = core_slice.call( arguments ), + length = resolveValues.length, + + // the count of uncompleted subordinates + remaining = length !== 1 || ( subordinate && jQuery.isFunction( subordinate.promise ) ) ? length : 0, + + // the master Deferred. If resolveValues consist of only a single Deferred, just use that. + deferred = remaining === 1 ? subordinate : jQuery.Deferred(), + + // Update function for both resolve and progress values + updateFunc = function( i, contexts, values ) { + return function( value ) { + contexts[ i ] = this; + values[ i ] = arguments.length > 1 ? core_slice.call( arguments ) : value; + if( values === progressValues ) { + deferred.notifyWith( contexts, values ); + } else if ( !( --remaining ) ) { + deferred.resolveWith( contexts, values ); + } + }; + }, + + progressValues, progressContexts, resolveContexts; + + // add listeners to Deferred subordinates; treat others as resolved + if ( length > 1 ) { + progressValues = new Array( length ); + progressContexts = new Array( length ); + resolveContexts = new Array( length ); + for ( ; i < length; i++ ) { + if ( resolveValues[ i ] && jQuery.isFunction( resolveValues[ i ].promise ) ) { + resolveValues[ i ].promise() + .done( updateFunc( i, resolveContexts, resolveValues ) ) + .fail( deferred.reject ) + .progress( updateFunc( i, progressContexts, progressValues ) ); + } else { + --remaining; + } + } + } + + // if we're not waiting on anything, resolve the master + if ( !remaining ) { + deferred.resolveWith( resolveContexts, resolveValues ); + } + + return deferred.promise(); + } +}); +jQuery.support = (function( support ) { + + var all, a, input, select, fragment, opt, eventName, isSupported, i, + div = document.createElement("div"); + + // Setup + div.setAttribute( "className", "t" ); + div.innerHTML = "
    oracle.kubernetes.operator.wlsconfig.WlsRetriever 
    Modifier and TypeConstant FieldConstant Field Value
    -public static final StringKEYKEY "wlsDomainConfig"
    a"; + + // Finish early in limited (non-browser) environments + all = div.getElementsByTagName("*") || []; + a = div.getElementsByTagName("a")[ 0 ]; + if ( !a || !a.style || !all.length ) { + return support; + } + + // First batch of tests + select = document.createElement("select"); + opt = select.appendChild( document.createElement("option") ); + input = div.getElementsByTagName("input")[ 0 ]; + + a.style.cssText = "top:1px;float:left;opacity:.5"; + + // Test setAttribute on camelCase class. If it works, we need attrFixes when doing get/setAttribute (ie6/7) + support.getSetAttribute = div.className !== "t"; + + // IE strips leading whitespace when .innerHTML is used + support.leadingWhitespace = div.firstChild.nodeType === 3; + + // Make sure that tbody elements aren't automatically inserted + // IE will insert them into empty tables + support.tbody = !div.getElementsByTagName("tbody").length; + + // Make sure that link elements get serialized correctly by innerHTML + // This requires a wrapper element in IE + support.htmlSerialize = !!div.getElementsByTagName("link").length; + + // Get the style information from getAttribute + // (IE uses .cssText instead) + support.style = /top/.test( a.getAttribute("style") ); + + // Make sure that URLs aren't manipulated + // (IE normalizes it by default) + support.hrefNormalized = a.getAttribute("href") === "/a"; + + // Make sure that element opacity exists + // (IE uses filter instead) + // Use a regex to work around a WebKit issue. See #5145 + support.opacity = /^0.5/.test( a.style.opacity ); + + // Verify style float existence + // (IE uses styleFloat instead of cssFloat) + support.cssFloat = !!a.style.cssFloat; + + // Check the default checkbox/radio value ("" on WebKit; "on" elsewhere) + support.checkOn = !!input.value; + + // Make sure that a selected-by-default option has a working selected property. + // (WebKit defaults to false instead of true, IE too, if it's in an optgroup) + support.optSelected = opt.selected; + + // Tests for enctype support on a form (#6743) + support.enctype = !!document.createElement("form").enctype; + + // Makes sure cloning an html5 element does not cause problems + // Where outerHTML is undefined, this still works + support.html5Clone = document.createElement("nav").cloneNode( true ).outerHTML !== "<:nav>"; + + // Will be defined later + support.inlineBlockNeedsLayout = false; + support.shrinkWrapBlocks = false; + support.pixelPosition = false; + support.deleteExpando = true; + support.noCloneEvent = true; + support.reliableMarginRight = true; + support.boxSizingReliable = true; + + // Make sure checked status is properly cloned + input.checked = true; + support.noCloneChecked = input.cloneNode( true ).checked; + + // Make sure that the options inside disabled selects aren't marked as disabled + // (WebKit marks them as disabled) + select.disabled = true; + support.optDisabled = !opt.disabled; + + // Support: IE<9 + try { + delete div.test; + } catch( e ) { + support.deleteExpando = false; + } + + // Check if we can trust getAttribute("value") + input = document.createElement("input"); + input.setAttribute( "value", "" ); + support.input = input.getAttribute( "value" ) === ""; + + // Check if an input maintains its value after becoming a radio + input.value = "t"; + input.setAttribute( "type", "radio" ); + support.radioValue = input.value === "t"; + + // #11217 - WebKit loses check when the name is after the checked attribute + input.setAttribute( "checked", "t" ); + input.setAttribute( "name", "t" ); + + fragment = document.createDocumentFragment(); + fragment.appendChild( input ); + + // Check if a disconnected checkbox will retain its checked + // value of true after appended to the DOM (IE6/7) + support.appendChecked = input.checked; + + // WebKit doesn't clone checked state correctly in fragments + support.checkClone = fragment.cloneNode( true ).cloneNode( true ).lastChild.checked; + + // Support: IE<9 + // Opera does not clone events (and typeof div.attachEvent === undefined). + // IE9-10 clones events bound via attachEvent, but they don't trigger with .click() + if ( div.attachEvent ) { + div.attachEvent( "onclick", function() { + support.noCloneEvent = false; + }); + + div.cloneNode( true ).click(); + } + + // Support: IE<9 (lack submit/change bubble), Firefox 17+ (lack focusin event) + // Beware of CSP restrictions (https://developer.mozilla.org/en/Security/CSP) + for ( i in { submit: true, change: true, focusin: true }) { + div.setAttribute( eventName = "on" + i, "t" ); + + support[ i + "Bubbles" ] = eventName in window || div.attributes[ eventName ].expando === false; + } + + div.style.backgroundClip = "content-box"; + div.cloneNode( true ).style.backgroundClip = ""; + support.clearCloneStyle = div.style.backgroundClip === "content-box"; + + // Support: IE<9 + // Iteration over object's inherited properties before its own. + for ( i in jQuery( support ) ) { + break; + } + support.ownLast = i !== "0"; + + // Run tests that need a body at doc ready + jQuery(function() { + var container, marginDiv, tds, + divReset = "padding:0;margin:0;border:0;display:block;box-sizing:content-box;-moz-box-sizing:content-box;-webkit-box-sizing:content-box;", + body = document.getElementsByTagName("body")[0]; + + if ( !body ) { + // Return for frameset docs that don't have a body + return; + } + + container = document.createElement("div"); + container.style.cssText = "border:0;width:0;height:0;position:absolute;top:0;left:-9999px;margin-top:1px"; + + body.appendChild( container ).appendChild( div ); + + // Support: IE8 + // Check if table cells still have offsetWidth/Height when they are set + // to display:none and there are still other visible table cells in a + // table row; if so, offsetWidth/Height are not reliable for use when + // determining if an element has been hidden directly using + // display:none (it is still safe to use offsets if a parent element is + // hidden; don safety goggles and see bug #4512 for more information). + div.innerHTML = "
    t
    "; + tds = div.getElementsByTagName("td"); + tds[ 0 ].style.cssText = "padding:0;margin:0;border:0;display:none"; + isSupported = ( tds[ 0 ].offsetHeight === 0 ); + + tds[ 0 ].style.display = ""; + tds[ 1 ].style.display = "none"; + + // Support: IE8 + // Check if empty table cells still have offsetWidth/Height + support.reliableHiddenOffsets = isSupported && ( tds[ 0 ].offsetHeight === 0 ); + + // Check box-sizing and margin behavior. + div.innerHTML = ""; + div.style.cssText = "box-sizing:border-box;-moz-box-sizing:border-box;-webkit-box-sizing:border-box;padding:1px;border:1px;display:block;width:4px;margin-top:1%;position:absolute;top:1%;"; + + // Workaround failing boxSizing test due to offsetWidth returning wrong value + // with some non-1 values of body zoom, ticket #13543 + jQuery.swap( body, body.style.zoom != null ? { zoom: 1 } : {}, function() { + support.boxSizing = div.offsetWidth === 4; + }); + + // Use window.getComputedStyle because jsdom on node.js will break without it. + if ( window.getComputedStyle ) { + support.pixelPosition = ( window.getComputedStyle( div, null ) || {} ).top !== "1%"; + support.boxSizingReliable = ( window.getComputedStyle( div, null ) || { width: "4px" } ).width === "4px"; + + // Check if div with explicit width and no margin-right incorrectly + // gets computed margin-right based on width of container. (#3333) + // Fails in WebKit before Feb 2011 nightlies + // WebKit Bug 13343 - getComputedStyle returns wrong value for margin-right + marginDiv = div.appendChild( document.createElement("div") ); + marginDiv.style.cssText = div.style.cssText = divReset; + marginDiv.style.marginRight = marginDiv.style.width = "0"; + div.style.width = "1px"; + + support.reliableMarginRight = + !parseFloat( ( window.getComputedStyle( marginDiv, null ) || {} ).marginRight ); + } + + if ( typeof div.style.zoom !== core_strundefined ) { + // Support: IE<8 + // Check if natively block-level elements act like inline-block + // elements when setting their display to 'inline' and giving + // them layout + div.innerHTML = ""; + div.style.cssText = divReset + "width:1px;padding:1px;display:inline;zoom:1"; + support.inlineBlockNeedsLayout = ( div.offsetWidth === 3 ); + + // Support: IE6 + // Check if elements with layout shrink-wrap their children + div.style.display = "block"; + div.innerHTML = "
    "; + div.firstChild.style.width = "5px"; + support.shrinkWrapBlocks = ( div.offsetWidth !== 3 ); + + if ( support.inlineBlockNeedsLayout ) { + // Prevent IE 6 from affecting layout for positioned elements #11048 + // Prevent IE from shrinking the body in IE 7 mode #12869 + // Support: IE<8 + body.style.zoom = 1; + } + } + + body.removeChild( container ); + + // Null elements to avoid leaks in IE + container = div = tds = marginDiv = null; + }); + + // Null elements to avoid leaks in IE + all = select = fragment = opt = a = input = null; + + return support; +})({}); + +var rbrace = /(?:\{[\s\S]*\}|\[[\s\S]*\])$/, + rmultiDash = /([A-Z])/g; + +function internalData( elem, name, data, pvt /* Internal Use Only */ ){ + if ( !jQuery.acceptData( elem ) ) { + return; + } + + var ret, thisCache, + internalKey = jQuery.expando, + + // We have to handle DOM nodes and JS objects differently because IE6-7 + // can't GC object references properly across the DOM-JS boundary + isNode = elem.nodeType, + + // Only DOM nodes need the global jQuery cache; JS object data is + // attached directly to the object so GC can occur automatically + cache = isNode ? jQuery.cache : elem, + + // Only defining an ID for JS objects if its cache already exists allows + // the code to shortcut on the same path as a DOM node with no cache + id = isNode ? elem[ internalKey ] : elem[ internalKey ] && internalKey; + + // Avoid doing any more work than we need to when trying to get data on an + // object that has no data at all + if ( (!id || !cache[id] || (!pvt && !cache[id].data)) && data === undefined && typeof name === "string" ) { + return; + } + + if ( !id ) { + // Only DOM nodes need a new unique ID for each element since their data + // ends up in the global cache + if ( isNode ) { + id = elem[ internalKey ] = core_deletedIds.pop() || jQuery.guid++; + } else { + id = internalKey; + } + } + + if ( !cache[ id ] ) { + // Avoid exposing jQuery metadata on plain JS objects when the object + // is serialized using JSON.stringify + cache[ id ] = isNode ? {} : { toJSON: jQuery.noop }; + } + + // An object can be passed to jQuery.data instead of a key/value pair; this gets + // shallow copied over onto the existing cache + if ( typeof name === "object" || typeof name === "function" ) { + if ( pvt ) { + cache[ id ] = jQuery.extend( cache[ id ], name ); + } else { + cache[ id ].data = jQuery.extend( cache[ id ].data, name ); + } + } + + thisCache = cache[ id ]; + + // jQuery data() is stored in a separate object inside the object's internal data + // cache in order to avoid key collisions between internal data and user-defined + // data. + if ( !pvt ) { + if ( !thisCache.data ) { + thisCache.data = {}; + } + + thisCache = thisCache.data; + } + + if ( data !== undefined ) { + thisCache[ jQuery.camelCase( name ) ] = data; + } + + // Check for both converted-to-camel and non-converted data property names + // If a data property was specified + if ( typeof name === "string" ) { + + // First Try to find as-is property data + ret = thisCache[ name ]; + + // Test for null|undefined property data + if ( ret == null ) { + + // Try to find the camelCased property + ret = thisCache[ jQuery.camelCase( name ) ]; + } + } else { + ret = thisCache; + } + + return ret; +} + +function internalRemoveData( elem, name, pvt ) { + if ( !jQuery.acceptData( elem ) ) { + return; + } + + var thisCache, i, + isNode = elem.nodeType, + + // See jQuery.data for more information + cache = isNode ? jQuery.cache : elem, + id = isNode ? elem[ jQuery.expando ] : jQuery.expando; + + // If there is already no cache entry for this object, there is no + // purpose in continuing + if ( !cache[ id ] ) { + return; + } + + if ( name ) { + + thisCache = pvt ? cache[ id ] : cache[ id ].data; + + if ( thisCache ) { + + // Support array or space separated string names for data keys + if ( !jQuery.isArray( name ) ) { + + // try the string as a key before any manipulation + if ( name in thisCache ) { + name = [ name ]; + } else { + + // split the camel cased version by spaces unless a key with the spaces exists + name = jQuery.camelCase( name ); + if ( name in thisCache ) { + name = [ name ]; + } else { + name = name.split(" "); + } + } + } else { + // If "name" is an array of keys... + // When data is initially created, via ("key", "val") signature, + // keys will be converted to camelCase. + // Since there is no way to tell _how_ a key was added, remove + // both plain key and camelCase key. #12786 + // This will only penalize the array argument path. + name = name.concat( jQuery.map( name, jQuery.camelCase ) ); + } + + i = name.length; + while ( i-- ) { + delete thisCache[ name[i] ]; + } + + // If there is no data left in the cache, we want to continue + // and let the cache object itself get destroyed + if ( pvt ? !isEmptyDataObject(thisCache) : !jQuery.isEmptyObject(thisCache) ) { + return; + } + } + } + + // See jQuery.data for more information + if ( !pvt ) { + delete cache[ id ].data; + + // Don't destroy the parent cache unless the internal data object + // had been the only thing left in it + if ( !isEmptyDataObject( cache[ id ] ) ) { + return; + } + } + + // Destroy the cache + if ( isNode ) { + jQuery.cleanData( [ elem ], true ); + + // Use delete when supported for expandos or `cache` is not a window per isWindow (#10080) + /* jshint eqeqeq: false */ + } else if ( jQuery.support.deleteExpando || cache != cache.window ) { + /* jshint eqeqeq: true */ + delete cache[ id ]; + + // When all else fails, null + } else { + cache[ id ] = null; + } +} + +jQuery.extend({ + cache: {}, + + // The following elements throw uncatchable exceptions if you + // attempt to add expando properties to them. + noData: { + "applet": true, + "embed": true, + // Ban all objects except for Flash (which handle expandos) + "object": "clsid:D27CDB6E-AE6D-11cf-96B8-444553540000" + }, + + hasData: function( elem ) { + elem = elem.nodeType ? jQuery.cache[ elem[jQuery.expando] ] : elem[ jQuery.expando ]; + return !!elem && !isEmptyDataObject( elem ); + }, + + data: function( elem, name, data ) { + return internalData( elem, name, data ); + }, + + removeData: function( elem, name ) { + return internalRemoveData( elem, name ); + }, + + // For internal use only. + _data: function( elem, name, data ) { + return internalData( elem, name, data, true ); + }, + + _removeData: function( elem, name ) { + return internalRemoveData( elem, name, true ); + }, + + // A method for determining if a DOM node can handle the data expando + acceptData: function( elem ) { + // Do not set data on non-element because it will not be cleared (#8335). + if ( elem.nodeType && elem.nodeType !== 1 && elem.nodeType !== 9 ) { + return false; + } + + var noData = elem.nodeName && jQuery.noData[ elem.nodeName.toLowerCase() ]; + + // nodes accept data unless otherwise specified; rejection can be conditional + return !noData || noData !== true && elem.getAttribute("classid") === noData; + } +}); + +jQuery.fn.extend({ + data: function( key, value ) { + var attrs, name, + data = null, + i = 0, + elem = this[0]; + + // Special expections of .data basically thwart jQuery.access, + // so implement the relevant behavior ourselves + + // Gets all values + if ( key === undefined ) { + if ( this.length ) { + data = jQuery.data( elem ); + + if ( elem.nodeType === 1 && !jQuery._data( elem, "parsedAttrs" ) ) { + attrs = elem.attributes; + for ( ; i < attrs.length; i++ ) { + name = attrs[i].name; + + if ( name.indexOf("data-") === 0 ) { + name = jQuery.camelCase( name.slice(5) ); + + dataAttr( elem, name, data[ name ] ); + } + } + jQuery._data( elem, "parsedAttrs", true ); + } + } + + return data; + } + + // Sets multiple values + if ( typeof key === "object" ) { + return this.each(function() { + jQuery.data( this, key ); + }); + } + + return arguments.length > 1 ? + + // Sets one value + this.each(function() { + jQuery.data( this, key, value ); + }) : + + // Gets one value + // Try to fetch any internally stored data first + elem ? dataAttr( elem, key, jQuery.data( elem, key ) ) : null; + }, + + removeData: function( key ) { + return this.each(function() { + jQuery.removeData( this, key ); + }); + } +}); + +function dataAttr( elem, key, data ) { + // If nothing was found internally, try to fetch any + // data from the HTML5 data-* attribute + if ( data === undefined && elem.nodeType === 1 ) { + + var name = "data-" + key.replace( rmultiDash, "-$1" ).toLowerCase(); + + data = elem.getAttribute( name ); + + if ( typeof data === "string" ) { + try { + data = data === "true" ? true : + data === "false" ? false : + data === "null" ? null : + // Only convert to a number if it doesn't change the string + +data + "" === data ? +data : + rbrace.test( data ) ? jQuery.parseJSON( data ) : + data; + } catch( e ) {} + + // Make sure we set the data so it isn't changed later + jQuery.data( elem, key, data ); + + } else { + data = undefined; + } + } + + return data; +} + +// checks a cache object for emptiness +function isEmptyDataObject( obj ) { + var name; + for ( name in obj ) { + + // if the public data object is empty, the private is still empty + if ( name === "data" && jQuery.isEmptyObject( obj[name] ) ) { + continue; + } + if ( name !== "toJSON" ) { + return false; + } + } + + return true; +} +jQuery.extend({ + queue: function( elem, type, data ) { + var queue; + + if ( elem ) { + type = ( type || "fx" ) + "queue"; + queue = jQuery._data( elem, type ); + + // Speed up dequeue by getting out quickly if this is just a lookup + if ( data ) { + if ( !queue || jQuery.isArray(data) ) { + queue = jQuery._data( elem, type, jQuery.makeArray(data) ); + } else { + queue.push( data ); + } + } + return queue || []; + } + }, + + dequeue: function( elem, type ) { + type = type || "fx"; + + var queue = jQuery.queue( elem, type ), + startLength = queue.length, + fn = queue.shift(), + hooks = jQuery._queueHooks( elem, type ), + next = function() { + jQuery.dequeue( elem, type ); + }; + + // If the fx queue is dequeued, always remove the progress sentinel + if ( fn === "inprogress" ) { + fn = queue.shift(); + startLength--; + } + + if ( fn ) { + + // Add a progress sentinel to prevent the fx queue from being + // automatically dequeued + if ( type === "fx" ) { + queue.unshift( "inprogress" ); + } + + // clear up the last queue stop function + delete hooks.stop; + fn.call( elem, next, hooks ); + } + + if ( !startLength && hooks ) { + hooks.empty.fire(); + } + }, + + // not intended for public consumption - generates a queueHooks object, or returns the current one + _queueHooks: function( elem, type ) { + var key = type + "queueHooks"; + return jQuery._data( elem, key ) || jQuery._data( elem, key, { + empty: jQuery.Callbacks("once memory").add(function() { + jQuery._removeData( elem, type + "queue" ); + jQuery._removeData( elem, key ); + }) + }); + } +}); + +jQuery.fn.extend({ + queue: function( type, data ) { + var setter = 2; + + if ( typeof type !== "string" ) { + data = type; + type = "fx"; + setter--; + } + + if ( arguments.length < setter ) { + return jQuery.queue( this[0], type ); + } + + return data === undefined ? + this : + this.each(function() { + var queue = jQuery.queue( this, type, data ); + + // ensure a hooks for this queue + jQuery._queueHooks( this, type ); + + if ( type === "fx" && queue[0] !== "inprogress" ) { + jQuery.dequeue( this, type ); + } + }); + }, + dequeue: function( type ) { + return this.each(function() { + jQuery.dequeue( this, type ); + }); + }, + // Based off of the plugin by Clint Helfers, with permission. + // http://blindsignals.com/index.php/2009/07/jquery-delay/ + delay: function( time, type ) { + time = jQuery.fx ? jQuery.fx.speeds[ time ] || time : time; + type = type || "fx"; + + return this.queue( type, function( next, hooks ) { + var timeout = setTimeout( next, time ); + hooks.stop = function() { + clearTimeout( timeout ); + }; + }); + }, + clearQueue: function( type ) { + return this.queue( type || "fx", [] ); + }, + // Get a promise resolved when queues of a certain type + // are emptied (fx is the type by default) + promise: function( type, obj ) { + var tmp, + count = 1, + defer = jQuery.Deferred(), + elements = this, + i = this.length, + resolve = function() { + if ( !( --count ) ) { + defer.resolveWith( elements, [ elements ] ); + } + }; + + if ( typeof type !== "string" ) { + obj = type; + type = undefined; + } + type = type || "fx"; + + while( i-- ) { + tmp = jQuery._data( elements[ i ], type + "queueHooks" ); + if ( tmp && tmp.empty ) { + count++; + tmp.empty.add( resolve ); + } + } + resolve(); + return defer.promise( obj ); + } +}); +var nodeHook, boolHook, + rclass = /[\t\r\n\f]/g, + rreturn = /\r/g, + rfocusable = /^(?:input|select|textarea|button|object)$/i, + rclickable = /^(?:a|area)$/i, + ruseDefault = /^(?:checked|selected)$/i, + getSetAttribute = jQuery.support.getSetAttribute, + getSetInput = jQuery.support.input; + +jQuery.fn.extend({ + attr: function( name, value ) { + return jQuery.access( this, jQuery.attr, name, value, arguments.length > 1 ); + }, + + removeAttr: function( name ) { + return this.each(function() { + jQuery.removeAttr( this, name ); + }); + }, + + prop: function( name, value ) { + return jQuery.access( this, jQuery.prop, name, value, arguments.length > 1 ); + }, + + removeProp: function( name ) { + name = jQuery.propFix[ name ] || name; + return this.each(function() { + // try/catch handles cases where IE balks (such as removing a property on window) + try { + this[ name ] = undefined; + delete this[ name ]; + } catch( e ) {} + }); + }, + + addClass: function( value ) { + var classes, elem, cur, clazz, j, + i = 0, + len = this.length, + proceed = typeof value === "string" && value; + + if ( jQuery.isFunction( value ) ) { + return this.each(function( j ) { + jQuery( this ).addClass( value.call( this, j, this.className ) ); + }); + } + + if ( proceed ) { + // The disjunction here is for better compressibility (see removeClass) + classes = ( value || "" ).match( core_rnotwhite ) || []; + + for ( ; i < len; i++ ) { + elem = this[ i ]; + cur = elem.nodeType === 1 && ( elem.className ? + ( " " + elem.className + " " ).replace( rclass, " " ) : + " " + ); + + if ( cur ) { + j = 0; + while ( (clazz = classes[j++]) ) { + if ( cur.indexOf( " " + clazz + " " ) < 0 ) { + cur += clazz + " "; + } + } + elem.className = jQuery.trim( cur ); + + } + } + } + + return this; + }, + + removeClass: function( value ) { + var classes, elem, cur, clazz, j, + i = 0, + len = this.length, + proceed = arguments.length === 0 || typeof value === "string" && value; + + if ( jQuery.isFunction( value ) ) { + return this.each(function( j ) { + jQuery( this ).removeClass( value.call( this, j, this.className ) ); + }); + } + if ( proceed ) { + classes = ( value || "" ).match( core_rnotwhite ) || []; + + for ( ; i < len; i++ ) { + elem = this[ i ]; + // This expression is here for better compressibility (see addClass) + cur = elem.nodeType === 1 && ( elem.className ? + ( " " + elem.className + " " ).replace( rclass, " " ) : + "" + ); + + if ( cur ) { + j = 0; + while ( (clazz = classes[j++]) ) { + // Remove *all* instances + while ( cur.indexOf( " " + clazz + " " ) >= 0 ) { + cur = cur.replace( " " + clazz + " ", " " ); + } + } + elem.className = value ? jQuery.trim( cur ) : ""; + } + } + } + + return this; + }, + + toggleClass: function( value, stateVal ) { + var type = typeof value; + + if ( typeof stateVal === "boolean" && type === "string" ) { + return stateVal ? this.addClass( value ) : this.removeClass( value ); + } + + if ( jQuery.isFunction( value ) ) { + return this.each(function( i ) { + jQuery( this ).toggleClass( value.call(this, i, this.className, stateVal), stateVal ); + }); + } + + return this.each(function() { + if ( type === "string" ) { + // toggle individual class names + var className, + i = 0, + self = jQuery( this ), + classNames = value.match( core_rnotwhite ) || []; + + while ( (className = classNames[ i++ ]) ) { + // check each className given, space separated list + if ( self.hasClass( className ) ) { + self.removeClass( className ); + } else { + self.addClass( className ); + } + } + + // Toggle whole class name + } else if ( type === core_strundefined || type === "boolean" ) { + if ( this.className ) { + // store className if set + jQuery._data( this, "__className__", this.className ); + } + + // If the element has a class name or if we're passed "false", + // then remove the whole classname (if there was one, the above saved it). + // Otherwise bring back whatever was previously saved (if anything), + // falling back to the empty string if nothing was stored. + this.className = this.className || value === false ? "" : jQuery._data( this, "__className__" ) || ""; + } + }); + }, + + hasClass: function( selector ) { + var className = " " + selector + " ", + i = 0, + l = this.length; + for ( ; i < l; i++ ) { + if ( this[i].nodeType === 1 && (" " + this[i].className + " ").replace(rclass, " ").indexOf( className ) >= 0 ) { + return true; + } + } + + return false; + }, + + val: function( value ) { + var ret, hooks, isFunction, + elem = this[0]; + + if ( !arguments.length ) { + if ( elem ) { + hooks = jQuery.valHooks[ elem.type ] || jQuery.valHooks[ elem.nodeName.toLowerCase() ]; + + if ( hooks && "get" in hooks && (ret = hooks.get( elem, "value" )) !== undefined ) { + return ret; + } + + ret = elem.value; + + return typeof ret === "string" ? + // handle most common string cases + ret.replace(rreturn, "") : + // handle cases where value is null/undef or number + ret == null ? "" : ret; + } + + return; + } + + isFunction = jQuery.isFunction( value ); + + return this.each(function( i ) { + var val; + + if ( this.nodeType !== 1 ) { + return; + } + + if ( isFunction ) { + val = value.call( this, i, jQuery( this ).val() ); + } else { + val = value; + } + + // Treat null/undefined as ""; convert numbers to string + if ( val == null ) { + val = ""; + } else if ( typeof val === "number" ) { + val += ""; + } else if ( jQuery.isArray( val ) ) { + val = jQuery.map(val, function ( value ) { + return value == null ? "" : value + ""; + }); + } + + hooks = jQuery.valHooks[ this.type ] || jQuery.valHooks[ this.nodeName.toLowerCase() ]; + + // If set returns undefined, fall back to normal setting + if ( !hooks || !("set" in hooks) || hooks.set( this, val, "value" ) === undefined ) { + this.value = val; + } + }); + } +}); + +jQuery.extend({ + valHooks: { + option: { + get: function( elem ) { + // Use proper attribute retrieval(#6932, #12072) + var val = jQuery.find.attr( elem, "value" ); + return val != null ? + val : + elem.text; + } + }, + select: { + get: function( elem ) { + var value, option, + options = elem.options, + index = elem.selectedIndex, + one = elem.type === "select-one" || index < 0, + values = one ? null : [], + max = one ? index + 1 : options.length, + i = index < 0 ? + max : + one ? index : 0; + + // Loop through all the selected options + for ( ; i < max; i++ ) { + option = options[ i ]; + + // oldIE doesn't update selected after form reset (#2551) + if ( ( option.selected || i === index ) && + // Don't return options that are disabled or in a disabled optgroup + ( jQuery.support.optDisabled ? !option.disabled : option.getAttribute("disabled") === null ) && + ( !option.parentNode.disabled || !jQuery.nodeName( option.parentNode, "optgroup" ) ) ) { + + // Get the specific value for the option + value = jQuery( option ).val(); + + // We don't need an array for one selects + if ( one ) { + return value; + } + + // Multi-Selects return an array + values.push( value ); + } + } + + return values; + }, + + set: function( elem, value ) { + var optionSet, option, + options = elem.options, + values = jQuery.makeArray( value ), + i = options.length; + + while ( i-- ) { + option = options[ i ]; + if ( (option.selected = jQuery.inArray( jQuery(option).val(), values ) >= 0) ) { + optionSet = true; + } + } + + // force browsers to behave consistently when non-matching value is set + if ( !optionSet ) { + elem.selectedIndex = -1; + } + return values; + } + } + }, + + attr: function( elem, name, value ) { + var hooks, ret, + nType = elem.nodeType; + + // don't get/set attributes on text, comment and attribute nodes + if ( !elem || nType === 3 || nType === 8 || nType === 2 ) { + return; + } + + // Fallback to prop when attributes are not supported + if ( typeof elem.getAttribute === core_strundefined ) { + return jQuery.prop( elem, name, value ); + } + + // All attributes are lowercase + // Grab necessary hook if one is defined + if ( nType !== 1 || !jQuery.isXMLDoc( elem ) ) { + name = name.toLowerCase(); + hooks = jQuery.attrHooks[ name ] || + ( jQuery.expr.match.bool.test( name ) ? boolHook : nodeHook ); + } + + if ( value !== undefined ) { + + if ( value === null ) { + jQuery.removeAttr( elem, name ); + + } else if ( hooks && "set" in hooks && (ret = hooks.set( elem, value, name )) !== undefined ) { + return ret; + + } else { + elem.setAttribute( name, value + "" ); + return value; + } + + } else if ( hooks && "get" in hooks && (ret = hooks.get( elem, name )) !== null ) { + return ret; + + } else { + ret = jQuery.find.attr( elem, name ); + + // Non-existent attributes return null, we normalize to undefined + return ret == null ? + undefined : + ret; + } + }, + + removeAttr: function( elem, value ) { + var name, propName, + i = 0, + attrNames = value && value.match( core_rnotwhite ); + + if ( attrNames && elem.nodeType === 1 ) { + while ( (name = attrNames[i++]) ) { + propName = jQuery.propFix[ name ] || name; + + // Boolean attributes get special treatment (#10870) + if ( jQuery.expr.match.bool.test( name ) ) { + // Set corresponding property to false + if ( getSetInput && getSetAttribute || !ruseDefault.test( name ) ) { + elem[ propName ] = false; + // Support: IE<9 + // Also clear defaultChecked/defaultSelected (if appropriate) + } else { + elem[ jQuery.camelCase( "default-" + name ) ] = + elem[ propName ] = false; + } + + // See #9699 for explanation of this approach (setting first, then removal) + } else { + jQuery.attr( elem, name, "" ); + } + + elem.removeAttribute( getSetAttribute ? name : propName ); + } + } + }, + + attrHooks: { + type: { + set: function( elem, value ) { + if ( !jQuery.support.radioValue && value === "radio" && jQuery.nodeName(elem, "input") ) { + // Setting the type on a radio button after the value resets the value in IE6-9 + // Reset value to default in case type is set after value during creation + var val = elem.value; + elem.setAttribute( "type", value ); + if ( val ) { + elem.value = val; + } + return value; + } + } + } + }, + + propFix: { + "for": "htmlFor", + "class": "className" + }, + + prop: function( elem, name, value ) { + var ret, hooks, notxml, + nType = elem.nodeType; + + // don't get/set properties on text, comment and attribute nodes + if ( !elem || nType === 3 || nType === 8 || nType === 2 ) { + return; + } + + notxml = nType !== 1 || !jQuery.isXMLDoc( elem ); + + if ( notxml ) { + // Fix name and attach hooks + name = jQuery.propFix[ name ] || name; + hooks = jQuery.propHooks[ name ]; + } + + if ( value !== undefined ) { + return hooks && "set" in hooks && (ret = hooks.set( elem, value, name )) !== undefined ? + ret : + ( elem[ name ] = value ); + + } else { + return hooks && "get" in hooks && (ret = hooks.get( elem, name )) !== null ? + ret : + elem[ name ]; + } + }, + + propHooks: { + tabIndex: { + get: function( elem ) { + // elem.tabIndex doesn't always return the correct value when it hasn't been explicitly set + // http://fluidproject.org/blog/2008/01/09/getting-setting-and-removing-tabindex-values-with-javascript/ + // Use proper attribute retrieval(#12072) + var tabindex = jQuery.find.attr( elem, "tabindex" ); + + return tabindex ? + parseInt( tabindex, 10 ) : + rfocusable.test( elem.nodeName ) || rclickable.test( elem.nodeName ) && elem.href ? + 0 : + -1; + } + } + } +}); + +// Hooks for boolean attributes +boolHook = { + set: function( elem, value, name ) { + if ( value === false ) { + // Remove boolean attributes when set to false + jQuery.removeAttr( elem, name ); + } else if ( getSetInput && getSetAttribute || !ruseDefault.test( name ) ) { + // IE<8 needs the *property* name + elem.setAttribute( !getSetAttribute && jQuery.propFix[ name ] || name, name ); + + // Use defaultChecked and defaultSelected for oldIE + } else { + elem[ jQuery.camelCase( "default-" + name ) ] = elem[ name ] = true; + } + + return name; + } +}; +jQuery.each( jQuery.expr.match.bool.source.match( /\w+/g ), function( i, name ) { + var getter = jQuery.expr.attrHandle[ name ] || jQuery.find.attr; + + jQuery.expr.attrHandle[ name ] = getSetInput && getSetAttribute || !ruseDefault.test( name ) ? + function( elem, name, isXML ) { + var fn = jQuery.expr.attrHandle[ name ], + ret = isXML ? + undefined : + /* jshint eqeqeq: false */ + (jQuery.expr.attrHandle[ name ] = undefined) != + getter( elem, name, isXML ) ? + + name.toLowerCase() : + null; + jQuery.expr.attrHandle[ name ] = fn; + return ret; + } : + function( elem, name, isXML ) { + return isXML ? + undefined : + elem[ jQuery.camelCase( "default-" + name ) ] ? + name.toLowerCase() : + null; + }; +}); + +// fix oldIE attroperties +if ( !getSetInput || !getSetAttribute ) { + jQuery.attrHooks.value = { + set: function( elem, value, name ) { + if ( jQuery.nodeName( elem, "input" ) ) { + // Does not return so that setAttribute is also used + elem.defaultValue = value; + } else { + // Use nodeHook if defined (#1954); otherwise setAttribute is fine + return nodeHook && nodeHook.set( elem, value, name ); + } + } + }; +} + +// IE6/7 do not support getting/setting some attributes with get/setAttribute +if ( !getSetAttribute ) { + + // Use this for any attribute in IE6/7 + // This fixes almost every IE6/7 issue + nodeHook = { + set: function( elem, value, name ) { + // Set the existing or create a new attribute node + var ret = elem.getAttributeNode( name ); + if ( !ret ) { + elem.setAttributeNode( + (ret = elem.ownerDocument.createAttribute( name )) + ); + } + + ret.value = value += ""; + + // Break association with cloned elements by also using setAttribute (#9646) + return name === "value" || value === elem.getAttribute( name ) ? + value : + undefined; + } + }; + jQuery.expr.attrHandle.id = jQuery.expr.attrHandle.name = jQuery.expr.attrHandle.coords = + // Some attributes are constructed with empty-string values when not defined + function( elem, name, isXML ) { + var ret; + return isXML ? + undefined : + (ret = elem.getAttributeNode( name )) && ret.value !== "" ? + ret.value : + null; + }; + jQuery.valHooks.button = { + get: function( elem, name ) { + var ret = elem.getAttributeNode( name ); + return ret && ret.specified ? + ret.value : + undefined; + }, + set: nodeHook.set + }; + + // Set contenteditable to false on removals(#10429) + // Setting to empty string throws an error as an invalid value + jQuery.attrHooks.contenteditable = { + set: function( elem, value, name ) { + nodeHook.set( elem, value === "" ? false : value, name ); + } + }; + + // Set width and height to auto instead of 0 on empty string( Bug #8150 ) + // This is for removals + jQuery.each([ "width", "height" ], function( i, name ) { + jQuery.attrHooks[ name ] = { + set: function( elem, value ) { + if ( value === "" ) { + elem.setAttribute( name, "auto" ); + return value; + } + } + }; + }); +} + + +// Some attributes require a special call on IE +// http://msdn.microsoft.com/en-us/library/ms536429%28VS.85%29.aspx +if ( !jQuery.support.hrefNormalized ) { + // href/src property should get the full normalized URL (#10299/#12915) + jQuery.each([ "href", "src" ], function( i, name ) { + jQuery.propHooks[ name ] = { + get: function( elem ) { + return elem.getAttribute( name, 4 ); + } + }; + }); +} + +if ( !jQuery.support.style ) { + jQuery.attrHooks.style = { + get: function( elem ) { + // Return undefined in the case of empty string + // Note: IE uppercases css property names, but if we were to .toLowerCase() + // .cssText, that would destroy case senstitivity in URL's, like in "background" + return elem.style.cssText || undefined; + }, + set: function( elem, value ) { + return ( elem.style.cssText = value + "" ); + } + }; +} + +// Safari mis-reports the default selected property of an option +// Accessing the parent's selectedIndex property fixes it +if ( !jQuery.support.optSelected ) { + jQuery.propHooks.selected = { + get: function( elem ) { + var parent = elem.parentNode; + + if ( parent ) { + parent.selectedIndex; + + // Make sure that it also works with optgroups, see #5701 + if ( parent.parentNode ) { + parent.parentNode.selectedIndex; + } + } + return null; + } + }; +} + +jQuery.each([ + "tabIndex", + "readOnly", + "maxLength", + "cellSpacing", + "cellPadding", + "rowSpan", + "colSpan", + "useMap", + "frameBorder", + "contentEditable" +], function() { + jQuery.propFix[ this.toLowerCase() ] = this; +}); + +// IE6/7 call enctype encoding +if ( !jQuery.support.enctype ) { + jQuery.propFix.enctype = "encoding"; +} + +// Radios and checkboxes getter/setter +jQuery.each([ "radio", "checkbox" ], function() { + jQuery.valHooks[ this ] = { + set: function( elem, value ) { + if ( jQuery.isArray( value ) ) { + return ( elem.checked = jQuery.inArray( jQuery(elem).val(), value ) >= 0 ); + } + } + }; + if ( !jQuery.support.checkOn ) { + jQuery.valHooks[ this ].get = function( elem ) { + // Support: Webkit + // "" is returned instead of "on" if a value isn't specified + return elem.getAttribute("value") === null ? "on" : elem.value; + }; + } +}); +var rformElems = /^(?:input|select|textarea)$/i, + rkeyEvent = /^key/, + rmouseEvent = /^(?:mouse|contextmenu)|click/, + rfocusMorph = /^(?:focusinfocus|focusoutblur)$/, + rtypenamespace = /^([^.]*)(?:\.(.+)|)$/; + +function returnTrue() { + return true; +} + +function returnFalse() { + return false; +} + +function safeActiveElement() { + try { + return document.activeElement; + } catch ( err ) { } +} + +/* + * Helper functions for managing events -- not part of the public interface. + * Props to Dean Edwards' addEvent library for many of the ideas. + */ +jQuery.event = { + + global: {}, + + add: function( elem, types, handler, data, selector ) { + var tmp, events, t, handleObjIn, + special, eventHandle, handleObj, + handlers, type, namespaces, origType, + elemData = jQuery._data( elem ); + + // Don't attach events to noData or text/comment nodes (but allow plain objects) + if ( !elemData ) { + return; + } + + // Caller can pass in an object of custom data in lieu of the handler + if ( handler.handler ) { + handleObjIn = handler; + handler = handleObjIn.handler; + selector = handleObjIn.selector; + } + + // Make sure that the handler has a unique ID, used to find/remove it later + if ( !handler.guid ) { + handler.guid = jQuery.guid++; + } + + // Init the element's event structure and main handler, if this is the first + if ( !(events = elemData.events) ) { + events = elemData.events = {}; + } + if ( !(eventHandle = elemData.handle) ) { + eventHandle = elemData.handle = function( e ) { + // Discard the second event of a jQuery.event.trigger() and + // when an event is called after a page has unloaded + return typeof jQuery !== core_strundefined && (!e || jQuery.event.triggered !== e.type) ? + jQuery.event.dispatch.apply( eventHandle.elem, arguments ) : + undefined; + }; + // Add elem as a property of the handle fn to prevent a memory leak with IE non-native events + eventHandle.elem = elem; + } + + // Handle multiple events separated by a space + types = ( types || "" ).match( core_rnotwhite ) || [""]; + t = types.length; + while ( t-- ) { + tmp = rtypenamespace.exec( types[t] ) || []; + type = origType = tmp[1]; + namespaces = ( tmp[2] || "" ).split( "." ).sort(); + + // There *must* be a type, no attaching namespace-only handlers + if ( !type ) { + continue; + } + + // If event changes its type, use the special event handlers for the changed type + special = jQuery.event.special[ type ] || {}; + + // If selector defined, determine special event api type, otherwise given type + type = ( selector ? special.delegateType : special.bindType ) || type; + + // Update special based on newly reset type + special = jQuery.event.special[ type ] || {}; + + // handleObj is passed to all event handlers + handleObj = jQuery.extend({ + type: type, + origType: origType, + data: data, + handler: handler, + guid: handler.guid, + selector: selector, + needsContext: selector && jQuery.expr.match.needsContext.test( selector ), + namespace: namespaces.join(".") + }, handleObjIn ); + + // Init the event handler queue if we're the first + if ( !(handlers = events[ type ]) ) { + handlers = events[ type ] = []; + handlers.delegateCount = 0; + + // Only use addEventListener/attachEvent if the special events handler returns false + if ( !special.setup || special.setup.call( elem, data, namespaces, eventHandle ) === false ) { + // Bind the global event handler to the element + if ( elem.addEventListener ) { + elem.addEventListener( type, eventHandle, false ); + + } else if ( elem.attachEvent ) { + elem.attachEvent( "on" + type, eventHandle ); + } + } + } + + if ( special.add ) { + special.add.call( elem, handleObj ); + + if ( !handleObj.handler.guid ) { + handleObj.handler.guid = handler.guid; + } + } + + // Add to the element's handler list, delegates in front + if ( selector ) { + handlers.splice( handlers.delegateCount++, 0, handleObj ); + } else { + handlers.push( handleObj ); + } + + // Keep track of which events have ever been used, for event optimization + jQuery.event.global[ type ] = true; + } + + // Nullify elem to prevent memory leaks in IE + elem = null; + }, + + // Detach an event or set of events from an element + remove: function( elem, types, handler, selector, mappedTypes ) { + var j, handleObj, tmp, + origCount, t, events, + special, handlers, type, + namespaces, origType, + elemData = jQuery.hasData( elem ) && jQuery._data( elem ); + + if ( !elemData || !(events = elemData.events) ) { + return; + } + + // Once for each type.namespace in types; type may be omitted + types = ( types || "" ).match( core_rnotwhite ) || [""]; + t = types.length; + while ( t-- ) { + tmp = rtypenamespace.exec( types[t] ) || []; + type = origType = tmp[1]; + namespaces = ( tmp[2] || "" ).split( "." ).sort(); + + // Unbind all events (on this namespace, if provided) for the element + if ( !type ) { + for ( type in events ) { + jQuery.event.remove( elem, type + types[ t ], handler, selector, true ); + } + continue; + } + + special = jQuery.event.special[ type ] || {}; + type = ( selector ? special.delegateType : special.bindType ) || type; + handlers = events[ type ] || []; + tmp = tmp[2] && new RegExp( "(^|\\.)" + namespaces.join("\\.(?:.*\\.|)") + "(\\.|$)" ); + + // Remove matching events + origCount = j = handlers.length; + while ( j-- ) { + handleObj = handlers[ j ]; + + if ( ( mappedTypes || origType === handleObj.origType ) && + ( !handler || handler.guid === handleObj.guid ) && + ( !tmp || tmp.test( handleObj.namespace ) ) && + ( !selector || selector === handleObj.selector || selector === "**" && handleObj.selector ) ) { + handlers.splice( j, 1 ); + + if ( handleObj.selector ) { + handlers.delegateCount--; + } + if ( special.remove ) { + special.remove.call( elem, handleObj ); + } + } + } + + // Remove generic event handler if we removed something and no more handlers exist + // (avoids potential for endless recursion during removal of special event handlers) + if ( origCount && !handlers.length ) { + if ( !special.teardown || special.teardown.call( elem, namespaces, elemData.handle ) === false ) { + jQuery.removeEvent( elem, type, elemData.handle ); + } + + delete events[ type ]; + } + } + + // Remove the expando if it's no longer used + if ( jQuery.isEmptyObject( events ) ) { + delete elemData.handle; + + // removeData also checks for emptiness and clears the expando if empty + // so use it instead of delete + jQuery._removeData( elem, "events" ); + } + }, + + trigger: function( event, data, elem, onlyHandlers ) { + var handle, ontype, cur, + bubbleType, special, tmp, i, + eventPath = [ elem || document ], + type = core_hasOwn.call( event, "type" ) ? event.type : event, + namespaces = core_hasOwn.call( event, "namespace" ) ? event.namespace.split(".") : []; + + cur = tmp = elem = elem || document; + + // Don't do events on text and comment nodes + if ( elem.nodeType === 3 || elem.nodeType === 8 ) { + return; + } + + // focus/blur morphs to focusin/out; ensure we're not firing them right now + if ( rfocusMorph.test( type + jQuery.event.triggered ) ) { + return; + } + + if ( type.indexOf(".") >= 0 ) { + // Namespaced trigger; create a regexp to match event type in handle() + namespaces = type.split("."); + type = namespaces.shift(); + namespaces.sort(); + } + ontype = type.indexOf(":") < 0 && "on" + type; + + // Caller can pass in a jQuery.Event object, Object, or just an event type string + event = event[ jQuery.expando ] ? + event : + new jQuery.Event( type, typeof event === "object" && event ); + + // Trigger bitmask: & 1 for native handlers; & 2 for jQuery (always true) + event.isTrigger = onlyHandlers ? 2 : 3; + event.namespace = namespaces.join("."); + event.namespace_re = event.namespace ? + new RegExp( "(^|\\.)" + namespaces.join("\\.(?:.*\\.|)") + "(\\.|$)" ) : + null; + + // Clean up the event in case it is being reused + event.result = undefined; + if ( !event.target ) { + event.target = elem; + } + + // Clone any incoming data and prepend the event, creating the handler arg list + data = data == null ? + [ event ] : + jQuery.makeArray( data, [ event ] ); + + // Allow special events to draw outside the lines + special = jQuery.event.special[ type ] || {}; + if ( !onlyHandlers && special.trigger && special.trigger.apply( elem, data ) === false ) { + return; + } + + // Determine event propagation path in advance, per W3C events spec (#9951) + // Bubble up to document, then to window; watch for a global ownerDocument var (#9724) + if ( !onlyHandlers && !special.noBubble && !jQuery.isWindow( elem ) ) { + + bubbleType = special.delegateType || type; + if ( !rfocusMorph.test( bubbleType + type ) ) { + cur = cur.parentNode; + } + for ( ; cur; cur = cur.parentNode ) { + eventPath.push( cur ); + tmp = cur; + } + + // Only add window if we got to document (e.g., not plain obj or detached DOM) + if ( tmp === (elem.ownerDocument || document) ) { + eventPath.push( tmp.defaultView || tmp.parentWindow || window ); + } + } + + // Fire handlers on the event path + i = 0; + while ( (cur = eventPath[i++]) && !event.isPropagationStopped() ) { + + event.type = i > 1 ? + bubbleType : + special.bindType || type; + + // jQuery handler + handle = ( jQuery._data( cur, "events" ) || {} )[ event.type ] && jQuery._data( cur, "handle" ); + if ( handle ) { + handle.apply( cur, data ); + } + + // Native handler + handle = ontype && cur[ ontype ]; + if ( handle && jQuery.acceptData( cur ) && handle.apply && handle.apply( cur, data ) === false ) { + event.preventDefault(); + } + } + event.type = type; + + // If nobody prevented the default action, do it now + if ( !onlyHandlers && !event.isDefaultPrevented() ) { + + if ( (!special._default || special._default.apply( eventPath.pop(), data ) === false) && + jQuery.acceptData( elem ) ) { + + // Call a native DOM method on the target with the same name name as the event. + // Can't use an .isFunction() check here because IE6/7 fails that test. + // Don't do default actions on window, that's where global variables be (#6170) + if ( ontype && elem[ type ] && !jQuery.isWindow( elem ) ) { + + // Don't re-trigger an onFOO event when we call its FOO() method + tmp = elem[ ontype ]; + + if ( tmp ) { + elem[ ontype ] = null; + } + + // Prevent re-triggering of the same event, since we already bubbled it above + jQuery.event.triggered = type; + try { + elem[ type ](); + } catch ( e ) { + // IE<9 dies on focus/blur to hidden element (#1486,#12518) + // only reproducible on winXP IE8 native, not IE9 in IE8 mode + } + jQuery.event.triggered = undefined; + + if ( tmp ) { + elem[ ontype ] = tmp; + } + } + } + } + + return event.result; + }, + + dispatch: function( event ) { + + // Make a writable jQuery.Event from the native event object + event = jQuery.event.fix( event ); + + var i, ret, handleObj, matched, j, + handlerQueue = [], + args = core_slice.call( arguments ), + handlers = ( jQuery._data( this, "events" ) || {} )[ event.type ] || [], + special = jQuery.event.special[ event.type ] || {}; + + // Use the fix-ed jQuery.Event rather than the (read-only) native event + args[0] = event; + event.delegateTarget = this; + + // Call the preDispatch hook for the mapped type, and let it bail if desired + if ( special.preDispatch && special.preDispatch.call( this, event ) === false ) { + return; + } + + // Determine handlers + handlerQueue = jQuery.event.handlers.call( this, event, handlers ); + + // Run delegates first; they may want to stop propagation beneath us + i = 0; + while ( (matched = handlerQueue[ i++ ]) && !event.isPropagationStopped() ) { + event.currentTarget = matched.elem; + + j = 0; + while ( (handleObj = matched.handlers[ j++ ]) && !event.isImmediatePropagationStopped() ) { + + // Triggered event must either 1) have no namespace, or + // 2) have namespace(s) a subset or equal to those in the bound event (both can have no namespace). + if ( !event.namespace_re || event.namespace_re.test( handleObj.namespace ) ) { + + event.handleObj = handleObj; + event.data = handleObj.data; + + ret = ( (jQuery.event.special[ handleObj.origType ] || {}).handle || handleObj.handler ) + .apply( matched.elem, args ); + + if ( ret !== undefined ) { + if ( (event.result = ret) === false ) { + event.preventDefault(); + event.stopPropagation(); + } + } + } + } + } + + // Call the postDispatch hook for the mapped type + if ( special.postDispatch ) { + special.postDispatch.call( this, event ); + } + + return event.result; + }, + + handlers: function( event, handlers ) { + var sel, handleObj, matches, i, + handlerQueue = [], + delegateCount = handlers.delegateCount, + cur = event.target; + + // Find delegate handlers + // Black-hole SVG instance trees (#13180) + // Avoid non-left-click bubbling in Firefox (#3861) + if ( delegateCount && cur.nodeType && (!event.button || event.type !== "click") ) { + + /* jshint eqeqeq: false */ + for ( ; cur != this; cur = cur.parentNode || this ) { + /* jshint eqeqeq: true */ + + // Don't check non-elements (#13208) + // Don't process clicks on disabled elements (#6911, #8165, #11382, #11764) + if ( cur.nodeType === 1 && (cur.disabled !== true || event.type !== "click") ) { + matches = []; + for ( i = 0; i < delegateCount; i++ ) { + handleObj = handlers[ i ]; + + // Don't conflict with Object.prototype properties (#13203) + sel = handleObj.selector + " "; + + if ( matches[ sel ] === undefined ) { + matches[ sel ] = handleObj.needsContext ? + jQuery( sel, this ).index( cur ) >= 0 : + jQuery.find( sel, this, null, [ cur ] ).length; + } + if ( matches[ sel ] ) { + matches.push( handleObj ); + } + } + if ( matches.length ) { + handlerQueue.push({ elem: cur, handlers: matches }); + } + } + } + } + + // Add the remaining (directly-bound) handlers + if ( delegateCount < handlers.length ) { + handlerQueue.push({ elem: this, handlers: handlers.slice( delegateCount ) }); + } + + return handlerQueue; + }, + + fix: function( event ) { + if ( event[ jQuery.expando ] ) { + return event; + } + + // Create a writable copy of the event object and normalize some properties + var i, prop, copy, + type = event.type, + originalEvent = event, + fixHook = this.fixHooks[ type ]; + + if ( !fixHook ) { + this.fixHooks[ type ] = fixHook = + rmouseEvent.test( type ) ? this.mouseHooks : + rkeyEvent.test( type ) ? this.keyHooks : + {}; + } + copy = fixHook.props ? this.props.concat( fixHook.props ) : this.props; + + event = new jQuery.Event( originalEvent ); + + i = copy.length; + while ( i-- ) { + prop = copy[ i ]; + event[ prop ] = originalEvent[ prop ]; + } + + // Support: IE<9 + // Fix target property (#1925) + if ( !event.target ) { + event.target = originalEvent.srcElement || document; + } + + // Support: Chrome 23+, Safari? + // Target should not be a text node (#504, #13143) + if ( event.target.nodeType === 3 ) { + event.target = event.target.parentNode; + } + + // Support: IE<9 + // For mouse/key events, metaKey==false if it's undefined (#3368, #11328) + event.metaKey = !!event.metaKey; + + return fixHook.filter ? fixHook.filter( event, originalEvent ) : event; + }, + + // Includes some event props shared by KeyEvent and MouseEvent + props: "altKey bubbles cancelable ctrlKey currentTarget eventPhase metaKey relatedTarget shiftKey target timeStamp view which".split(" "), + + fixHooks: {}, + + keyHooks: { + props: "char charCode key keyCode".split(" "), + filter: function( event, original ) { + + // Add which for key events + if ( event.which == null ) { + event.which = original.charCode != null ? original.charCode : original.keyCode; + } + + return event; + } + }, + + mouseHooks: { + props: "button buttons clientX clientY fromElement offsetX offsetY pageX pageY screenX screenY toElement".split(" "), + filter: function( event, original ) { + var body, eventDoc, doc, + button = original.button, + fromElement = original.fromElement; + + // Calculate pageX/Y if missing and clientX/Y available + if ( event.pageX == null && original.clientX != null ) { + eventDoc = event.target.ownerDocument || document; + doc = eventDoc.documentElement; + body = eventDoc.body; + + event.pageX = original.clientX + ( doc && doc.scrollLeft || body && body.scrollLeft || 0 ) - ( doc && doc.clientLeft || body && body.clientLeft || 0 ); + event.pageY = original.clientY + ( doc && doc.scrollTop || body && body.scrollTop || 0 ) - ( doc && doc.clientTop || body && body.clientTop || 0 ); + } + + // Add relatedTarget, if necessary + if ( !event.relatedTarget && fromElement ) { + event.relatedTarget = fromElement === event.target ? original.toElement : fromElement; + } + + // Add which for click: 1 === left; 2 === middle; 3 === right + // Note: button is not normalized, so don't use it + if ( !event.which && button !== undefined ) { + event.which = ( button & 1 ? 1 : ( button & 2 ? 3 : ( button & 4 ? 2 : 0 ) ) ); + } + + return event; + } + }, + + special: { + load: { + // Prevent triggered image.load events from bubbling to window.load + noBubble: true + }, + focus: { + // Fire native event if possible so blur/focus sequence is correct + trigger: function() { + if ( this !== safeActiveElement() && this.focus ) { + try { + this.focus(); + return false; + } catch ( e ) { + // Support: IE<9 + // If we error on focus to hidden element (#1486, #12518), + // let .trigger() run the handlers + } + } + }, + delegateType: "focusin" + }, + blur: { + trigger: function() { + if ( this === safeActiveElement() && this.blur ) { + this.blur(); + return false; + } + }, + delegateType: "focusout" + }, + click: { + // For checkbox, fire native event so checked state will be right + trigger: function() { + if ( jQuery.nodeName( this, "input" ) && this.type === "checkbox" && this.click ) { + this.click(); + return false; + } + }, + + // For cross-browser consistency, don't fire native .click() on links + _default: function( event ) { + return jQuery.nodeName( event.target, "a" ); + } + }, + + beforeunload: { + postDispatch: function( event ) { + + // Even when returnValue equals to undefined Firefox will still show alert + if ( event.result !== undefined ) { + event.originalEvent.returnValue = event.result; + } + } + } + }, + + simulate: function( type, elem, event, bubble ) { + // Piggyback on a donor event to simulate a different one. + // Fake originalEvent to avoid donor's stopPropagation, but if the + // simulated event prevents default then we do the same on the donor. + var e = jQuery.extend( + new jQuery.Event(), + event, + { + type: type, + isSimulated: true, + originalEvent: {} + } + ); + if ( bubble ) { + jQuery.event.trigger( e, null, elem ); + } else { + jQuery.event.dispatch.call( elem, e ); + } + if ( e.isDefaultPrevented() ) { + event.preventDefault(); + } + } +}; + +jQuery.removeEvent = document.removeEventListener ? + function( elem, type, handle ) { + if ( elem.removeEventListener ) { + elem.removeEventListener( type, handle, false ); + } + } : + function( elem, type, handle ) { + var name = "on" + type; + + if ( elem.detachEvent ) { + + // #8545, #7054, preventing memory leaks for custom events in IE6-8 + // detachEvent needed property on element, by name of that event, to properly expose it to GC + if ( typeof elem[ name ] === core_strundefined ) { + elem[ name ] = null; + } + + elem.detachEvent( name, handle ); + } + }; + +jQuery.Event = function( src, props ) { + // Allow instantiation without the 'new' keyword + if ( !(this instanceof jQuery.Event) ) { + return new jQuery.Event( src, props ); + } + + // Event object + if ( src && src.type ) { + this.originalEvent = src; + this.type = src.type; + + // Events bubbling up the document may have been marked as prevented + // by a handler lower down the tree; reflect the correct value. + this.isDefaultPrevented = ( src.defaultPrevented || src.returnValue === false || + src.getPreventDefault && src.getPreventDefault() ) ? returnTrue : returnFalse; + + // Event type + } else { + this.type = src; + } + + // Put explicitly provided properties onto the event object + if ( props ) { + jQuery.extend( this, props ); + } + + // Create a timestamp if incoming event doesn't have one + this.timeStamp = src && src.timeStamp || jQuery.now(); + + // Mark it as fixed + this[ jQuery.expando ] = true; +}; + +// jQuery.Event is based on DOM3 Events as specified by the ECMAScript Language Binding +// http://www.w3.org/TR/2003/WD-DOM-Level-3-Events-20030331/ecma-script-binding.html +jQuery.Event.prototype = { + isDefaultPrevented: returnFalse, + isPropagationStopped: returnFalse, + isImmediatePropagationStopped: returnFalse, + + preventDefault: function() { + var e = this.originalEvent; + + this.isDefaultPrevented = returnTrue; + if ( !e ) { + return; + } + + // If preventDefault exists, run it on the original event + if ( e.preventDefault ) { + e.preventDefault(); + + // Support: IE + // Otherwise set the returnValue property of the original event to false + } else { + e.returnValue = false; + } + }, + stopPropagation: function() { + var e = this.originalEvent; + + this.isPropagationStopped = returnTrue; + if ( !e ) { + return; + } + // If stopPropagation exists, run it on the original event + if ( e.stopPropagation ) { + e.stopPropagation(); + } + + // Support: IE + // Set the cancelBubble property of the original event to true + e.cancelBubble = true; + }, + stopImmediatePropagation: function() { + this.isImmediatePropagationStopped = returnTrue; + this.stopPropagation(); + } +}; + +// Create mouseenter/leave events using mouseover/out and event-time checks +jQuery.each({ + mouseenter: "mouseover", + mouseleave: "mouseout" +}, function( orig, fix ) { + jQuery.event.special[ orig ] = { + delegateType: fix, + bindType: fix, + + handle: function( event ) { + var ret, + target = this, + related = event.relatedTarget, + handleObj = event.handleObj; + + // For mousenter/leave call the handler if related is outside the target. + // NB: No relatedTarget if the mouse left/entered the browser window + if ( !related || (related !== target && !jQuery.contains( target, related )) ) { + event.type = handleObj.origType; + ret = handleObj.handler.apply( this, arguments ); + event.type = fix; + } + return ret; + } + }; +}); + +// IE submit delegation +if ( !jQuery.support.submitBubbles ) { + + jQuery.event.special.submit = { + setup: function() { + // Only need this for delegated form submit events + if ( jQuery.nodeName( this, "form" ) ) { + return false; + } + + // Lazy-add a submit handler when a descendant form may potentially be submitted + jQuery.event.add( this, "click._submit keypress._submit", function( e ) { + // Node name check avoids a VML-related crash in IE (#9807) + var elem = e.target, + form = jQuery.nodeName( elem, "input" ) || jQuery.nodeName( elem, "button" ) ? elem.form : undefined; + if ( form && !jQuery._data( form, "submitBubbles" ) ) { + jQuery.event.add( form, "submit._submit", function( event ) { + event._submit_bubble = true; + }); + jQuery._data( form, "submitBubbles", true ); + } + }); + // return undefined since we don't need an event listener + }, + + postDispatch: function( event ) { + // If form was submitted by the user, bubble the event up the tree + if ( event._submit_bubble ) { + delete event._submit_bubble; + if ( this.parentNode && !event.isTrigger ) { + jQuery.event.simulate( "submit", this.parentNode, event, true ); + } + } + }, + + teardown: function() { + // Only need this for delegated form submit events + if ( jQuery.nodeName( this, "form" ) ) { + return false; + } + + // Remove delegated handlers; cleanData eventually reaps submit handlers attached above + jQuery.event.remove( this, "._submit" ); + } + }; +} + +// IE change delegation and checkbox/radio fix +if ( !jQuery.support.changeBubbles ) { + + jQuery.event.special.change = { + + setup: function() { + + if ( rformElems.test( this.nodeName ) ) { + // IE doesn't fire change on a check/radio until blur; trigger it on click + // after a propertychange. Eat the blur-change in special.change.handle. + // This still fires onchange a second time for check/radio after blur. + if ( this.type === "checkbox" || this.type === "radio" ) { + jQuery.event.add( this, "propertychange._change", function( event ) { + if ( event.originalEvent.propertyName === "checked" ) { + this._just_changed = true; + } + }); + jQuery.event.add( this, "click._change", function( event ) { + if ( this._just_changed && !event.isTrigger ) { + this._just_changed = false; + } + // Allow triggered, simulated change events (#11500) + jQuery.event.simulate( "change", this, event, true ); + }); + } + return false; + } + // Delegated event; lazy-add a change handler on descendant inputs + jQuery.event.add( this, "beforeactivate._change", function( e ) { + var elem = e.target; + + if ( rformElems.test( elem.nodeName ) && !jQuery._data( elem, "changeBubbles" ) ) { + jQuery.event.add( elem, "change._change", function( event ) { + if ( this.parentNode && !event.isSimulated && !event.isTrigger ) { + jQuery.event.simulate( "change", this.parentNode, event, true ); + } + }); + jQuery._data( elem, "changeBubbles", true ); + } + }); + }, + + handle: function( event ) { + var elem = event.target; + + // Swallow native change events from checkbox/radio, we already triggered them above + if ( this !== elem || event.isSimulated || event.isTrigger || (elem.type !== "radio" && elem.type !== "checkbox") ) { + return event.handleObj.handler.apply( this, arguments ); + } + }, + + teardown: function() { + jQuery.event.remove( this, "._change" ); + + return !rformElems.test( this.nodeName ); + } + }; +} + +// Create "bubbling" focus and blur events +if ( !jQuery.support.focusinBubbles ) { + jQuery.each({ focus: "focusin", blur: "focusout" }, function( orig, fix ) { + + // Attach a single capturing handler while someone wants focusin/focusout + var attaches = 0, + handler = function( event ) { + jQuery.event.simulate( fix, event.target, jQuery.event.fix( event ), true ); + }; + + jQuery.event.special[ fix ] = { + setup: function() { + if ( attaches++ === 0 ) { + document.addEventListener( orig, handler, true ); + } + }, + teardown: function() { + if ( --attaches === 0 ) { + document.removeEventListener( orig, handler, true ); + } + } + }; + }); +} + +jQuery.fn.extend({ + + on: function( types, selector, data, fn, /*INTERNAL*/ one ) { + var type, origFn; + + // Types can be a map of types/handlers + if ( typeof types === "object" ) { + // ( types-Object, selector, data ) + if ( typeof selector !== "string" ) { + // ( types-Object, data ) + data = data || selector; + selector = undefined; + } + for ( type in types ) { + this.on( type, selector, data, types[ type ], one ); + } + return this; + } + + if ( data == null && fn == null ) { + // ( types, fn ) + fn = selector; + data = selector = undefined; + } else if ( fn == null ) { + if ( typeof selector === "string" ) { + // ( types, selector, fn ) + fn = data; + data = undefined; + } else { + // ( types, data, fn ) + fn = data; + data = selector; + selector = undefined; + } + } + if ( fn === false ) { + fn = returnFalse; + } else if ( !fn ) { + return this; + } + + if ( one === 1 ) { + origFn = fn; + fn = function( event ) { + // Can use an empty set, since event contains the info + jQuery().off( event ); + return origFn.apply( this, arguments ); + }; + // Use same guid so caller can remove using origFn + fn.guid = origFn.guid || ( origFn.guid = jQuery.guid++ ); + } + return this.each( function() { + jQuery.event.add( this, types, fn, data, selector ); + }); + }, + one: function( types, selector, data, fn ) { + return this.on( types, selector, data, fn, 1 ); + }, + off: function( types, selector, fn ) { + var handleObj, type; + if ( types && types.preventDefault && types.handleObj ) { + // ( event ) dispatched jQuery.Event + handleObj = types.handleObj; + jQuery( types.delegateTarget ).off( + handleObj.namespace ? handleObj.origType + "." + handleObj.namespace : handleObj.origType, + handleObj.selector, + handleObj.handler + ); + return this; + } + if ( typeof types === "object" ) { + // ( types-object [, selector] ) + for ( type in types ) { + this.off( type, selector, types[ type ] ); + } + return this; + } + if ( selector === false || typeof selector === "function" ) { + // ( types [, fn] ) + fn = selector; + selector = undefined; + } + if ( fn === false ) { + fn = returnFalse; + } + return this.each(function() { + jQuery.event.remove( this, types, fn, selector ); + }); + }, + + trigger: function( type, data ) { + return this.each(function() { + jQuery.event.trigger( type, data, this ); + }); + }, + triggerHandler: function( type, data ) { + var elem = this[0]; + if ( elem ) { + return jQuery.event.trigger( type, data, elem, true ); + } + } +}); +var isSimple = /^.[^:#\[\.,]*$/, + rparentsprev = /^(?:parents|prev(?:Until|All))/, + rneedsContext = jQuery.expr.match.needsContext, + // methods guaranteed to produce a unique set when starting from a unique set + guaranteedUnique = { + children: true, + contents: true, + next: true, + prev: true + }; + +jQuery.fn.extend({ + find: function( selector ) { + var i, + ret = [], + self = this, + len = self.length; + + if ( typeof selector !== "string" ) { + return this.pushStack( jQuery( selector ).filter(function() { + for ( i = 0; i < len; i++ ) { + if ( jQuery.contains( self[ i ], this ) ) { + return true; + } + } + }) ); + } + + for ( i = 0; i < len; i++ ) { + jQuery.find( selector, self[ i ], ret ); + } + + // Needed because $( selector, context ) becomes $( context ).find( selector ) + ret = this.pushStack( len > 1 ? jQuery.unique( ret ) : ret ); + ret.selector = this.selector ? this.selector + " " + selector : selector; + return ret; + }, + + has: function( target ) { + var i, + targets = jQuery( target, this ), + len = targets.length; + + return this.filter(function() { + for ( i = 0; i < len; i++ ) { + if ( jQuery.contains( this, targets[i] ) ) { + return true; + } + } + }); + }, + + not: function( selector ) { + return this.pushStack( winnow(this, selector || [], true) ); + }, + + filter: function( selector ) { + return this.pushStack( winnow(this, selector || [], false) ); + }, + + is: function( selector ) { + return !!winnow( + this, + + // If this is a positional/relative selector, check membership in the returned set + // so $("p:first").is("p:last") won't return true for a doc with two "p". + typeof selector === "string" && rneedsContext.test( selector ) ? + jQuery( selector ) : + selector || [], + false + ).length; + }, + + closest: function( selectors, context ) { + var cur, + i = 0, + l = this.length, + ret = [], + pos = rneedsContext.test( selectors ) || typeof selectors !== "string" ? + jQuery( selectors, context || this.context ) : + 0; + + for ( ; i < l; i++ ) { + for ( cur = this[i]; cur && cur !== context; cur = cur.parentNode ) { + // Always skip document fragments + if ( cur.nodeType < 11 && (pos ? + pos.index(cur) > -1 : + + // Don't pass non-elements to Sizzle + cur.nodeType === 1 && + jQuery.find.matchesSelector(cur, selectors)) ) { + + cur = ret.push( cur ); + break; + } + } + } + + return this.pushStack( ret.length > 1 ? jQuery.unique( ret ) : ret ); + }, + + // Determine the position of an element within + // the matched set of elements + index: function( elem ) { + + // No argument, return index in parent + if ( !elem ) { + return ( this[0] && this[0].parentNode ) ? this.first().prevAll().length : -1; + } + + // index in selector + if ( typeof elem === "string" ) { + return jQuery.inArray( this[0], jQuery( elem ) ); + } + + // Locate the position of the desired element + return jQuery.inArray( + // If it receives a jQuery object, the first element is used + elem.jquery ? elem[0] : elem, this ); + }, + + add: function( selector, context ) { + var set = typeof selector === "string" ? + jQuery( selector, context ) : + jQuery.makeArray( selector && selector.nodeType ? [ selector ] : selector ), + all = jQuery.merge( this.get(), set ); + + return this.pushStack( jQuery.unique(all) ); + }, + + addBack: function( selector ) { + return this.add( selector == null ? + this.prevObject : this.prevObject.filter(selector) + ); + } +}); + +function sibling( cur, dir ) { + do { + cur = cur[ dir ]; + } while ( cur && cur.nodeType !== 1 ); + + return cur; +} + +jQuery.each({ + parent: function( elem ) { + var parent = elem.parentNode; + return parent && parent.nodeType !== 11 ? parent : null; + }, + parents: function( elem ) { + return jQuery.dir( elem, "parentNode" ); + }, + parentsUntil: function( elem, i, until ) { + return jQuery.dir( elem, "parentNode", until ); + }, + next: function( elem ) { + return sibling( elem, "nextSibling" ); + }, + prev: function( elem ) { + return sibling( elem, "previousSibling" ); + }, + nextAll: function( elem ) { + return jQuery.dir( elem, "nextSibling" ); + }, + prevAll: function( elem ) { + return jQuery.dir( elem, "previousSibling" ); + }, + nextUntil: function( elem, i, until ) { + return jQuery.dir( elem, "nextSibling", until ); + }, + prevUntil: function( elem, i, until ) { + return jQuery.dir( elem, "previousSibling", until ); + }, + siblings: function( elem ) { + return jQuery.sibling( ( elem.parentNode || {} ).firstChild, elem ); + }, + children: function( elem ) { + return jQuery.sibling( elem.firstChild ); + }, + contents: function( elem ) { + return jQuery.nodeName( elem, "iframe" ) ? + elem.contentDocument || elem.contentWindow.document : + jQuery.merge( [], elem.childNodes ); + } +}, function( name, fn ) { + jQuery.fn[ name ] = function( until, selector ) { + var ret = jQuery.map( this, fn, until ); + + if ( name.slice( -5 ) !== "Until" ) { + selector = until; + } + + if ( selector && typeof selector === "string" ) { + ret = jQuery.filter( selector, ret ); + } + + if ( this.length > 1 ) { + // Remove duplicates + if ( !guaranteedUnique[ name ] ) { + ret = jQuery.unique( ret ); + } + + // Reverse order for parents* and prev-derivatives + if ( rparentsprev.test( name ) ) { + ret = ret.reverse(); + } + } + + return this.pushStack( ret ); + }; +}); + +jQuery.extend({ + filter: function( expr, elems, not ) { + var elem = elems[ 0 ]; + + if ( not ) { + expr = ":not(" + expr + ")"; + } + + return elems.length === 1 && elem.nodeType === 1 ? + jQuery.find.matchesSelector( elem, expr ) ? [ elem ] : [] : + jQuery.find.matches( expr, jQuery.grep( elems, function( elem ) { + return elem.nodeType === 1; + })); + }, + + dir: function( elem, dir, until ) { + var matched = [], + cur = elem[ dir ]; + + while ( cur && cur.nodeType !== 9 && (until === undefined || cur.nodeType !== 1 || !jQuery( cur ).is( until )) ) { + if ( cur.nodeType === 1 ) { + matched.push( cur ); + } + cur = cur[dir]; + } + return matched; + }, + + sibling: function( n, elem ) { + var r = []; + + for ( ; n; n = n.nextSibling ) { + if ( n.nodeType === 1 && n !== elem ) { + r.push( n ); + } + } + + return r; + } +}); + +// Implement the identical functionality for filter and not +function winnow( elements, qualifier, not ) { + if ( jQuery.isFunction( qualifier ) ) { + return jQuery.grep( elements, function( elem, i ) { + /* jshint -W018 */ + return !!qualifier.call( elem, i, elem ) !== not; + }); + + } + + if ( qualifier.nodeType ) { + return jQuery.grep( elements, function( elem ) { + return ( elem === qualifier ) !== not; + }); + + } + + if ( typeof qualifier === "string" ) { + if ( isSimple.test( qualifier ) ) { + return jQuery.filter( qualifier, elements, not ); + } + + qualifier = jQuery.filter( qualifier, elements ); + } + + return jQuery.grep( elements, function( elem ) { + return ( jQuery.inArray( elem, qualifier ) >= 0 ) !== not; + }); +} +function createSafeFragment( document ) { + var list = nodeNames.split( "|" ), + safeFrag = document.createDocumentFragment(); + + if ( safeFrag.createElement ) { + while ( list.length ) { + safeFrag.createElement( + list.pop() + ); + } + } + return safeFrag; +} + +var nodeNames = "abbr|article|aside|audio|bdi|canvas|data|datalist|details|figcaption|figure|footer|" + + "header|hgroup|mark|meter|nav|output|progress|section|summary|time|video", + rinlinejQuery = / jQuery\d+="(?:null|\d+)"/g, + rnoshimcache = new RegExp("<(?:" + nodeNames + ")[\\s/>]", "i"), + rleadingWhitespace = /^\s+/, + rxhtmlTag = /<(?!area|br|col|embed|hr|img|input|link|meta|param)(([\w:]+)[^>]*)\/>/gi, + rtagName = /<([\w:]+)/, + rtbody = /\s*$/g, + + // We have to close these tags to support XHTML (#13200) + wrapMap = { + option: [ 1, "" ], + legend: [ 1, "
    ", "
    " ], + area: [ 1, "", "" ], + param: [ 1, "", "" ], + thead: [ 1, "", "
    " ], + tr: [ 2, "", "
    " ], + col: [ 2, "", "
    " ], + td: [ 3, "", "
    " ], + + // IE6-8 can't serialize link, script, style, or any html5 (NoScope) tags, + // unless wrapped in a div with non-breaking characters in front of it. + _default: jQuery.support.htmlSerialize ? [ 0, "", "" ] : [ 1, "X
    ", "
    " ] + }, + safeFragment = createSafeFragment( document ), + fragmentDiv = safeFragment.appendChild( document.createElement("div") ); + +wrapMap.optgroup = wrapMap.option; +wrapMap.tbody = wrapMap.tfoot = wrapMap.colgroup = wrapMap.caption = wrapMap.thead; +wrapMap.th = wrapMap.td; + +jQuery.fn.extend({ + text: function( value ) { + return jQuery.access( this, function( value ) { + return value === undefined ? + jQuery.text( this ) : + this.empty().append( ( this[0] && this[0].ownerDocument || document ).createTextNode( value ) ); + }, null, value, arguments.length ); + }, + + append: function() { + return this.domManip( arguments, function( elem ) { + if ( this.nodeType === 1 || this.nodeType === 11 || this.nodeType === 9 ) { + var target = manipulationTarget( this, elem ); + target.appendChild( elem ); + } + }); + }, + + prepend: function() { + return this.domManip( arguments, function( elem ) { + if ( this.nodeType === 1 || this.nodeType === 11 || this.nodeType === 9 ) { + var target = manipulationTarget( this, elem ); + target.insertBefore( elem, target.firstChild ); + } + }); + }, + + before: function() { + return this.domManip( arguments, function( elem ) { + if ( this.parentNode ) { + this.parentNode.insertBefore( elem, this ); + } + }); + }, + + after: function() { + return this.domManip( arguments, function( elem ) { + if ( this.parentNode ) { + this.parentNode.insertBefore( elem, this.nextSibling ); + } + }); + }, + + // keepData is for internal use only--do not document + remove: function( selector, keepData ) { + var elem, + elems = selector ? jQuery.filter( selector, this ) : this, + i = 0; + + for ( ; (elem = elems[i]) != null; i++ ) { + + if ( !keepData && elem.nodeType === 1 ) { + jQuery.cleanData( getAll( elem ) ); + } + + if ( elem.parentNode ) { + if ( keepData && jQuery.contains( elem.ownerDocument, elem ) ) { + setGlobalEval( getAll( elem, "script" ) ); + } + elem.parentNode.removeChild( elem ); + } + } + + return this; + }, + + empty: function() { + var elem, + i = 0; + + for ( ; (elem = this[i]) != null; i++ ) { + // Remove element nodes and prevent memory leaks + if ( elem.nodeType === 1 ) { + jQuery.cleanData( getAll( elem, false ) ); + } + + // Remove any remaining nodes + while ( elem.firstChild ) { + elem.removeChild( elem.firstChild ); + } + + // If this is a select, ensure that it displays empty (#12336) + // Support: IE<9 + if ( elem.options && jQuery.nodeName( elem, "select" ) ) { + elem.options.length = 0; + } + } + + return this; + }, + + clone: function( dataAndEvents, deepDataAndEvents ) { + dataAndEvents = dataAndEvents == null ? false : dataAndEvents; + deepDataAndEvents = deepDataAndEvents == null ? dataAndEvents : deepDataAndEvents; + + return this.map( function () { + return jQuery.clone( this, dataAndEvents, deepDataAndEvents ); + }); + }, + + html: function( value ) { + return jQuery.access( this, function( value ) { + var elem = this[0] || {}, + i = 0, + l = this.length; + + if ( value === undefined ) { + return elem.nodeType === 1 ? + elem.innerHTML.replace( rinlinejQuery, "" ) : + undefined; + } + + // See if we can take a shortcut and just use innerHTML + if ( typeof value === "string" && !rnoInnerhtml.test( value ) && + ( jQuery.support.htmlSerialize || !rnoshimcache.test( value ) ) && + ( jQuery.support.leadingWhitespace || !rleadingWhitespace.test( value ) ) && + !wrapMap[ ( rtagName.exec( value ) || ["", ""] )[1].toLowerCase() ] ) { + + value = value.replace( rxhtmlTag, "<$1>" ); + + try { + for (; i < l; i++ ) { + // Remove element nodes and prevent memory leaks + elem = this[i] || {}; + if ( elem.nodeType === 1 ) { + jQuery.cleanData( getAll( elem, false ) ); + elem.innerHTML = value; + } + } + + elem = 0; + + // If using innerHTML throws an exception, use the fallback method + } catch(e) {} + } + + if ( elem ) { + this.empty().append( value ); + } + }, null, value, arguments.length ); + }, + + replaceWith: function() { + var + // Snapshot the DOM in case .domManip sweeps something relevant into its fragment + args = jQuery.map( this, function( elem ) { + return [ elem.nextSibling, elem.parentNode ]; + }), + i = 0; + + // Make the changes, replacing each context element with the new content + this.domManip( arguments, function( elem ) { + var next = args[ i++ ], + parent = args[ i++ ]; + + if ( parent ) { + // Don't use the snapshot next if it has moved (#13810) + if ( next && next.parentNode !== parent ) { + next = this.nextSibling; + } + jQuery( this ).remove(); + parent.insertBefore( elem, next ); + } + // Allow new content to include elements from the context set + }, true ); + + // Force removal if there was no new content (e.g., from empty arguments) + return i ? this : this.remove(); + }, + + detach: function( selector ) { + return this.remove( selector, true ); + }, + + domManip: function( args, callback, allowIntersection ) { + + // Flatten any nested arrays + args = core_concat.apply( [], args ); + + var first, node, hasScripts, + scripts, doc, fragment, + i = 0, + l = this.length, + set = this, + iNoClone = l - 1, + value = args[0], + isFunction = jQuery.isFunction( value ); + + // We can't cloneNode fragments that contain checked, in WebKit + if ( isFunction || !( l <= 1 || typeof value !== "string" || jQuery.support.checkClone || !rchecked.test( value ) ) ) { + return this.each(function( index ) { + var self = set.eq( index ); + if ( isFunction ) { + args[0] = value.call( this, index, self.html() ); + } + self.domManip( args, callback, allowIntersection ); + }); + } + + if ( l ) { + fragment = jQuery.buildFragment( args, this[ 0 ].ownerDocument, false, !allowIntersection && this ); + first = fragment.firstChild; + + if ( fragment.childNodes.length === 1 ) { + fragment = first; + } + + if ( first ) { + scripts = jQuery.map( getAll( fragment, "script" ), disableScript ); + hasScripts = scripts.length; + + // Use the original fragment for the last item instead of the first because it can end up + // being emptied incorrectly in certain situations (#8070). + for ( ; i < l; i++ ) { + node = fragment; + + if ( i !== iNoClone ) { + node = jQuery.clone( node, true, true ); + + // Keep references to cloned scripts for later restoration + if ( hasScripts ) { + jQuery.merge( scripts, getAll( node, "script" ) ); + } + } + + callback.call( this[i], node, i ); + } + + if ( hasScripts ) { + doc = scripts[ scripts.length - 1 ].ownerDocument; + + // Reenable scripts + jQuery.map( scripts, restoreScript ); + + // Evaluate executable scripts on first document insertion + for ( i = 0; i < hasScripts; i++ ) { + node = scripts[ i ]; + if ( rscriptType.test( node.type || "" ) && + !jQuery._data( node, "globalEval" ) && jQuery.contains( doc, node ) ) { + + if ( node.src ) { + // Hope ajax is available... + jQuery._evalUrl( node.src ); + } else { + jQuery.globalEval( ( node.text || node.textContent || node.innerHTML || "" ).replace( rcleanScript, "" ) ); + } + } + } + } + + // Fix #11809: Avoid leaking memory + fragment = first = null; + } + } + + return this; + } +}); + +// Support: IE<8 +// Manipulating tables requires a tbody +function manipulationTarget( elem, content ) { + return jQuery.nodeName( elem, "table" ) && + jQuery.nodeName( content.nodeType === 1 ? content : content.firstChild, "tr" ) ? + + elem.getElementsByTagName("tbody")[0] || + elem.appendChild( elem.ownerDocument.createElement("tbody") ) : + elem; +} + +// Replace/restore the type attribute of script elements for safe DOM manipulation +function disableScript( elem ) { + elem.type = (jQuery.find.attr( elem, "type" ) !== null) + "/" + elem.type; + return elem; +} +function restoreScript( elem ) { + var match = rscriptTypeMasked.exec( elem.type ); + if ( match ) { + elem.type = match[1]; + } else { + elem.removeAttribute("type"); + } + return elem; +} + +// Mark scripts as having already been evaluated +function setGlobalEval( elems, refElements ) { + var elem, + i = 0; + for ( ; (elem = elems[i]) != null; i++ ) { + jQuery._data( elem, "globalEval", !refElements || jQuery._data( refElements[i], "globalEval" ) ); + } +} + +function cloneCopyEvent( src, dest ) { + + if ( dest.nodeType !== 1 || !jQuery.hasData( src ) ) { + return; + } + + var type, i, l, + oldData = jQuery._data( src ), + curData = jQuery._data( dest, oldData ), + events = oldData.events; + + if ( events ) { + delete curData.handle; + curData.events = {}; + + for ( type in events ) { + for ( i = 0, l = events[ type ].length; i < l; i++ ) { + jQuery.event.add( dest, type, events[ type ][ i ] ); + } + } + } + + // make the cloned public data object a copy from the original + if ( curData.data ) { + curData.data = jQuery.extend( {}, curData.data ); + } +} + +function fixCloneNodeIssues( src, dest ) { + var nodeName, e, data; + + // We do not need to do anything for non-Elements + if ( dest.nodeType !== 1 ) { + return; + } + + nodeName = dest.nodeName.toLowerCase(); + + // IE6-8 copies events bound via attachEvent when using cloneNode. + if ( !jQuery.support.noCloneEvent && dest[ jQuery.expando ] ) { + data = jQuery._data( dest ); + + for ( e in data.events ) { + jQuery.removeEvent( dest, e, data.handle ); + } + + // Event data gets referenced instead of copied if the expando gets copied too + dest.removeAttribute( jQuery.expando ); + } + + // IE blanks contents when cloning scripts, and tries to evaluate newly-set text + if ( nodeName === "script" && dest.text !== src.text ) { + disableScript( dest ).text = src.text; + restoreScript( dest ); + + // IE6-10 improperly clones children of object elements using classid. + // IE10 throws NoModificationAllowedError if parent is null, #12132. + } else if ( nodeName === "object" ) { + if ( dest.parentNode ) { + dest.outerHTML = src.outerHTML; + } + + // This path appears unavoidable for IE9. When cloning an object + // element in IE9, the outerHTML strategy above is not sufficient. + // If the src has innerHTML and the destination does not, + // copy the src.innerHTML into the dest.innerHTML. #10324 + if ( jQuery.support.html5Clone && ( src.innerHTML && !jQuery.trim(dest.innerHTML) ) ) { + dest.innerHTML = src.innerHTML; + } + + } else if ( nodeName === "input" && manipulation_rcheckableType.test( src.type ) ) { + // IE6-8 fails to persist the checked state of a cloned checkbox + // or radio button. Worse, IE6-7 fail to give the cloned element + // a checked appearance if the defaultChecked value isn't also set + + dest.defaultChecked = dest.checked = src.checked; + + // IE6-7 get confused and end up setting the value of a cloned + // checkbox/radio button to an empty string instead of "on" + if ( dest.value !== src.value ) { + dest.value = src.value; + } + + // IE6-8 fails to return the selected option to the default selected + // state when cloning options + } else if ( nodeName === "option" ) { + dest.defaultSelected = dest.selected = src.defaultSelected; + + // IE6-8 fails to set the defaultValue to the correct value when + // cloning other types of input fields + } else if ( nodeName === "input" || nodeName === "textarea" ) { + dest.defaultValue = src.defaultValue; + } +} + +jQuery.each({ + appendTo: "append", + prependTo: "prepend", + insertBefore: "before", + insertAfter: "after", + replaceAll: "replaceWith" +}, function( name, original ) { + jQuery.fn[ name ] = function( selector ) { + var elems, + i = 0, + ret = [], + insert = jQuery( selector ), + last = insert.length - 1; + + for ( ; i <= last; i++ ) { + elems = i === last ? this : this.clone(true); + jQuery( insert[i] )[ original ]( elems ); + + // Modern browsers can apply jQuery collections as arrays, but oldIE needs a .get() + core_push.apply( ret, elems.get() ); + } + + return this.pushStack( ret ); + }; +}); + +function getAll( context, tag ) { + var elems, elem, + i = 0, + found = typeof context.getElementsByTagName !== core_strundefined ? context.getElementsByTagName( tag || "*" ) : + typeof context.querySelectorAll !== core_strundefined ? context.querySelectorAll( tag || "*" ) : + undefined; + + if ( !found ) { + for ( found = [], elems = context.childNodes || context; (elem = elems[i]) != null; i++ ) { + if ( !tag || jQuery.nodeName( elem, tag ) ) { + found.push( elem ); + } else { + jQuery.merge( found, getAll( elem, tag ) ); + } + } + } + + return tag === undefined || tag && jQuery.nodeName( context, tag ) ? + jQuery.merge( [ context ], found ) : + found; +} + +// Used in buildFragment, fixes the defaultChecked property +function fixDefaultChecked( elem ) { + if ( manipulation_rcheckableType.test( elem.type ) ) { + elem.defaultChecked = elem.checked; + } +} + +jQuery.extend({ + clone: function( elem, dataAndEvents, deepDataAndEvents ) { + var destElements, node, clone, i, srcElements, + inPage = jQuery.contains( elem.ownerDocument, elem ); + + if ( jQuery.support.html5Clone || jQuery.isXMLDoc(elem) || !rnoshimcache.test( "<" + elem.nodeName + ">" ) ) { + clone = elem.cloneNode( true ); + + // IE<=8 does not properly clone detached, unknown element nodes + } else { + fragmentDiv.innerHTML = elem.outerHTML; + fragmentDiv.removeChild( clone = fragmentDiv.firstChild ); + } + + if ( (!jQuery.support.noCloneEvent || !jQuery.support.noCloneChecked) && + (elem.nodeType === 1 || elem.nodeType === 11) && !jQuery.isXMLDoc(elem) ) { + + // We eschew Sizzle here for performance reasons: http://jsperf.com/getall-vs-sizzle/2 + destElements = getAll( clone ); + srcElements = getAll( elem ); + + // Fix all IE cloning issues + for ( i = 0; (node = srcElements[i]) != null; ++i ) { + // Ensure that the destination node is not null; Fixes #9587 + if ( destElements[i] ) { + fixCloneNodeIssues( node, destElements[i] ); + } + } + } + + // Copy the events from the original to the clone + if ( dataAndEvents ) { + if ( deepDataAndEvents ) { + srcElements = srcElements || getAll( elem ); + destElements = destElements || getAll( clone ); + + for ( i = 0; (node = srcElements[i]) != null; i++ ) { + cloneCopyEvent( node, destElements[i] ); + } + } else { + cloneCopyEvent( elem, clone ); + } + } + + // Preserve script evaluation history + destElements = getAll( clone, "script" ); + if ( destElements.length > 0 ) { + setGlobalEval( destElements, !inPage && getAll( elem, "script" ) ); + } + + destElements = srcElements = node = null; + + // Return the cloned set + return clone; + }, + + buildFragment: function( elems, context, scripts, selection ) { + var j, elem, contains, + tmp, tag, tbody, wrap, + l = elems.length, + + // Ensure a safe fragment + safe = createSafeFragment( context ), + + nodes = [], + i = 0; + + for ( ; i < l; i++ ) { + elem = elems[ i ]; + + if ( elem || elem === 0 ) { + + // Add nodes directly + if ( jQuery.type( elem ) === "object" ) { + jQuery.merge( nodes, elem.nodeType ? [ elem ] : elem ); + + // Convert non-html into a text node + } else if ( !rhtml.test( elem ) ) { + nodes.push( context.createTextNode( elem ) ); + + // Convert html into DOM nodes + } else { + tmp = tmp || safe.appendChild( context.createElement("div") ); + + // Deserialize a standard representation + tag = ( rtagName.exec( elem ) || ["", ""] )[1].toLowerCase(); + wrap = wrapMap[ tag ] || wrapMap._default; + + tmp.innerHTML = wrap[1] + elem.replace( rxhtmlTag, "<$1>" ) + wrap[2]; + + // Descend through wrappers to the right content + j = wrap[0]; + while ( j-- ) { + tmp = tmp.lastChild; + } + + // Manually add leading whitespace removed by IE + if ( !jQuery.support.leadingWhitespace && rleadingWhitespace.test( elem ) ) { + nodes.push( context.createTextNode( rleadingWhitespace.exec( elem )[0] ) ); + } + + // Remove IE's autoinserted from table fragments + if ( !jQuery.support.tbody ) { + + // String was a , *may* have spurious + elem = tag === "table" && !rtbody.test( elem ) ? + tmp.firstChild : + + // String was a bare or + wrap[1] === "
    " && !rtbody.test( elem ) ? + tmp : + 0; + + j = elem && elem.childNodes.length; + while ( j-- ) { + if ( jQuery.nodeName( (tbody = elem.childNodes[j]), "tbody" ) && !tbody.childNodes.length ) { + elem.removeChild( tbody ); + } + } + } + + jQuery.merge( nodes, tmp.childNodes ); + + // Fix #12392 for WebKit and IE > 9 + tmp.textContent = ""; + + // Fix #12392 for oldIE + while ( tmp.firstChild ) { + tmp.removeChild( tmp.firstChild ); + } + + // Remember the top-level container for proper cleanup + tmp = safe.lastChild; + } + } + } + + // Fix #11356: Clear elements from fragment + if ( tmp ) { + safe.removeChild( tmp ); + } + + // Reset defaultChecked for any radios and checkboxes + // about to be appended to the DOM in IE 6/7 (#8060) + if ( !jQuery.support.appendChecked ) { + jQuery.grep( getAll( nodes, "input" ), fixDefaultChecked ); + } + + i = 0; + while ( (elem = nodes[ i++ ]) ) { + + // #4087 - If origin and destination elements are the same, and this is + // that element, do not do anything + if ( selection && jQuery.inArray( elem, selection ) !== -1 ) { + continue; + } + + contains = jQuery.contains( elem.ownerDocument, elem ); + + // Append to fragment + tmp = getAll( safe.appendChild( elem ), "script" ); + + // Preserve script evaluation history + if ( contains ) { + setGlobalEval( tmp ); + } + + // Capture executables + if ( scripts ) { + j = 0; + while ( (elem = tmp[ j++ ]) ) { + if ( rscriptType.test( elem.type || "" ) ) { + scripts.push( elem ); + } + } + } + } + + tmp = null; + + return safe; + }, + + cleanData: function( elems, /* internal */ acceptData ) { + var elem, type, id, data, + i = 0, + internalKey = jQuery.expando, + cache = jQuery.cache, + deleteExpando = jQuery.support.deleteExpando, + special = jQuery.event.special; + + for ( ; (elem = elems[i]) != null; i++ ) { + + if ( acceptData || jQuery.acceptData( elem ) ) { + + id = elem[ internalKey ]; + data = id && cache[ id ]; + + if ( data ) { + if ( data.events ) { + for ( type in data.events ) { + if ( special[ type ] ) { + jQuery.event.remove( elem, type ); + + // This is a shortcut to avoid jQuery.event.remove's overhead + } else { + jQuery.removeEvent( elem, type, data.handle ); + } + } + } + + // Remove cache only if it was not already removed by jQuery.event.remove + if ( cache[ id ] ) { + + delete cache[ id ]; + + // IE does not allow us to delete expando properties from nodes, + // nor does it have a removeAttribute function on Document nodes; + // we must handle all of these cases + if ( deleteExpando ) { + delete elem[ internalKey ]; + + } else if ( typeof elem.removeAttribute !== core_strundefined ) { + elem.removeAttribute( internalKey ); + + } else { + elem[ internalKey ] = null; + } + + core_deletedIds.push( id ); + } + } + } + } + }, + + _evalUrl: function( url ) { + return jQuery.ajax({ + url: url, + type: "GET", + dataType: "script", + async: false, + global: false, + "throws": true + }); + } +}); +jQuery.fn.extend({ + wrapAll: function( html ) { + if ( jQuery.isFunction( html ) ) { + return this.each(function(i) { + jQuery(this).wrapAll( html.call(this, i) ); + }); + } + + if ( this[0] ) { + // The elements to wrap the target around + var wrap = jQuery( html, this[0].ownerDocument ).eq(0).clone(true); + + if ( this[0].parentNode ) { + wrap.insertBefore( this[0] ); + } + + wrap.map(function() { + var elem = this; + + while ( elem.firstChild && elem.firstChild.nodeType === 1 ) { + elem = elem.firstChild; + } + + return elem; + }).append( this ); + } + + return this; + }, + + wrapInner: function( html ) { + if ( jQuery.isFunction( html ) ) { + return this.each(function(i) { + jQuery(this).wrapInner( html.call(this, i) ); + }); + } + + return this.each(function() { + var self = jQuery( this ), + contents = self.contents(); + + if ( contents.length ) { + contents.wrapAll( html ); + + } else { + self.append( html ); + } + }); + }, + + wrap: function( html ) { + var isFunction = jQuery.isFunction( html ); + + return this.each(function(i) { + jQuery( this ).wrapAll( isFunction ? html.call(this, i) : html ); + }); + }, + + unwrap: function() { + return this.parent().each(function() { + if ( !jQuery.nodeName( this, "body" ) ) { + jQuery( this ).replaceWith( this.childNodes ); + } + }).end(); + } +}); +var iframe, getStyles, curCSS, + ralpha = /alpha\([^)]*\)/i, + ropacity = /opacity\s*=\s*([^)]*)/, + rposition = /^(top|right|bottom|left)$/, + // swappable if display is none or starts with table except "table", "table-cell", or "table-caption" + // see here for display values: https://developer.mozilla.org/en-US/docs/CSS/display + rdisplayswap = /^(none|table(?!-c[ea]).+)/, + rmargin = /^margin/, + rnumsplit = new RegExp( "^(" + core_pnum + ")(.*)$", "i" ), + rnumnonpx = new RegExp( "^(" + core_pnum + ")(?!px)[a-z%]+$", "i" ), + rrelNum = new RegExp( "^([+-])=(" + core_pnum + ")", "i" ), + elemdisplay = { BODY: "block" }, + + cssShow = { position: "absolute", visibility: "hidden", display: "block" }, + cssNormalTransform = { + letterSpacing: 0, + fontWeight: 400 + }, + + cssExpand = [ "Top", "Right", "Bottom", "Left" ], + cssPrefixes = [ "Webkit", "O", "Moz", "ms" ]; + +// return a css property mapped to a potentially vendor prefixed property +function vendorPropName( style, name ) { + + // shortcut for names that are not vendor prefixed + if ( name in style ) { + return name; + } + + // check for vendor prefixed names + var capName = name.charAt(0).toUpperCase() + name.slice(1), + origName = name, + i = cssPrefixes.length; + + while ( i-- ) { + name = cssPrefixes[ i ] + capName; + if ( name in style ) { + return name; + } + } + + return origName; +} + +function isHidden( elem, el ) { + // isHidden might be called from jQuery#filter function; + // in that case, element will be second argument + elem = el || elem; + return jQuery.css( elem, "display" ) === "none" || !jQuery.contains( elem.ownerDocument, elem ); +} + +function showHide( elements, show ) { + var display, elem, hidden, + values = [], + index = 0, + length = elements.length; + + for ( ; index < length; index++ ) { + elem = elements[ index ]; + if ( !elem.style ) { + continue; + } + + values[ index ] = jQuery._data( elem, "olddisplay" ); + display = elem.style.display; + if ( show ) { + // Reset the inline display of this element to learn if it is + // being hidden by cascaded rules or not + if ( !values[ index ] && display === "none" ) { + elem.style.display = ""; + } + + // Set elements which have been overridden with display: none + // in a stylesheet to whatever the default browser style is + // for such an element + if ( elem.style.display === "" && isHidden( elem ) ) { + values[ index ] = jQuery._data( elem, "olddisplay", css_defaultDisplay(elem.nodeName) ); + } + } else { + + if ( !values[ index ] ) { + hidden = isHidden( elem ); + + if ( display && display !== "none" || !hidden ) { + jQuery._data( elem, "olddisplay", hidden ? display : jQuery.css( elem, "display" ) ); + } + } + } + } + + // Set the display of most of the elements in a second loop + // to avoid the constant reflow + for ( index = 0; index < length; index++ ) { + elem = elements[ index ]; + if ( !elem.style ) { + continue; + } + if ( !show || elem.style.display === "none" || elem.style.display === "" ) { + elem.style.display = show ? values[ index ] || "" : "none"; + } + } + + return elements; +} + +jQuery.fn.extend({ + css: function( name, value ) { + return jQuery.access( this, function( elem, name, value ) { + var len, styles, + map = {}, + i = 0; + + if ( jQuery.isArray( name ) ) { + styles = getStyles( elem ); + len = name.length; + + for ( ; i < len; i++ ) { + map[ name[ i ] ] = jQuery.css( elem, name[ i ], false, styles ); + } + + return map; + } + + return value !== undefined ? + jQuery.style( elem, name, value ) : + jQuery.css( elem, name ); + }, name, value, arguments.length > 1 ); + }, + show: function() { + return showHide( this, true ); + }, + hide: function() { + return showHide( this ); + }, + toggle: function( state ) { + if ( typeof state === "boolean" ) { + return state ? this.show() : this.hide(); + } + + return this.each(function() { + if ( isHidden( this ) ) { + jQuery( this ).show(); + } else { + jQuery( this ).hide(); + } + }); + } +}); + +jQuery.extend({ + // Add in style property hooks for overriding the default + // behavior of getting and setting a style property + cssHooks: { + opacity: { + get: function( elem, computed ) { + if ( computed ) { + // We should always get a number back from opacity + var ret = curCSS( elem, "opacity" ); + return ret === "" ? "1" : ret; + } + } + } + }, + + // Don't automatically add "px" to these possibly-unitless properties + cssNumber: { + "columnCount": true, + "fillOpacity": true, + "fontWeight": true, + "lineHeight": true, + "opacity": true, + "order": true, + "orphans": true, + "widows": true, + "zIndex": true, + "zoom": true + }, + + // Add in properties whose names you wish to fix before + // setting or getting the value + cssProps: { + // normalize float css property + "float": jQuery.support.cssFloat ? "cssFloat" : "styleFloat" + }, + + // Get and set the style property on a DOM Node + style: function( elem, name, value, extra ) { + // Don't set styles on text and comment nodes + if ( !elem || elem.nodeType === 3 || elem.nodeType === 8 || !elem.style ) { + return; + } + + // Make sure that we're working with the right name + var ret, type, hooks, + origName = jQuery.camelCase( name ), + style = elem.style; + + name = jQuery.cssProps[ origName ] || ( jQuery.cssProps[ origName ] = vendorPropName( style, origName ) ); + + // gets hook for the prefixed version + // followed by the unprefixed version + hooks = jQuery.cssHooks[ name ] || jQuery.cssHooks[ origName ]; + + // Check if we're setting a value + if ( value !== undefined ) { + type = typeof value; + + // convert relative number strings (+= or -=) to relative numbers. #7345 + if ( type === "string" && (ret = rrelNum.exec( value )) ) { + value = ( ret[1] + 1 ) * ret[2] + parseFloat( jQuery.css( elem, name ) ); + // Fixes bug #9237 + type = "number"; + } + + // Make sure that NaN and null values aren't set. See: #7116 + if ( value == null || type === "number" && isNaN( value ) ) { + return; + } + + // If a number was passed in, add 'px' to the (except for certain CSS properties) + if ( type === "number" && !jQuery.cssNumber[ origName ] ) { + value += "px"; + } + + // Fixes #8908, it can be done more correctly by specifing setters in cssHooks, + // but it would mean to define eight (for every problematic property) identical functions + if ( !jQuery.support.clearCloneStyle && value === "" && name.indexOf("background") === 0 ) { + style[ name ] = "inherit"; + } + + // If a hook was provided, use that value, otherwise just set the specified value + if ( !hooks || !("set" in hooks) || (value = hooks.set( elem, value, extra )) !== undefined ) { + + // Wrapped to prevent IE from throwing errors when 'invalid' values are provided + // Fixes bug #5509 + try { + style[ name ] = value; + } catch(e) {} + } + + } else { + // If a hook was provided get the non-computed value from there + if ( hooks && "get" in hooks && (ret = hooks.get( elem, false, extra )) !== undefined ) { + return ret; + } + + // Otherwise just get the value from the style object + return style[ name ]; + } + }, + + css: function( elem, name, extra, styles ) { + var num, val, hooks, + origName = jQuery.camelCase( name ); + + // Make sure that we're working with the right name + name = jQuery.cssProps[ origName ] || ( jQuery.cssProps[ origName ] = vendorPropName( elem.style, origName ) ); + + // gets hook for the prefixed version + // followed by the unprefixed version + hooks = jQuery.cssHooks[ name ] || jQuery.cssHooks[ origName ]; + + // If a hook was provided get the computed value from there + if ( hooks && "get" in hooks ) { + val = hooks.get( elem, true, extra ); + } + + // Otherwise, if a way to get the computed value exists, use that + if ( val === undefined ) { + val = curCSS( elem, name, styles ); + } + + //convert "normal" to computed value + if ( val === "normal" && name in cssNormalTransform ) { + val = cssNormalTransform[ name ]; + } + + // Return, converting to number if forced or a qualifier was provided and val looks numeric + if ( extra === "" || extra ) { + num = parseFloat( val ); + return extra === true || jQuery.isNumeric( num ) ? num || 0 : val; + } + return val; + } +}); + +// NOTE: we've included the "window" in window.getComputedStyle +// because jsdom on node.js will break without it. +if ( window.getComputedStyle ) { + getStyles = function( elem ) { + return window.getComputedStyle( elem, null ); + }; + + curCSS = function( elem, name, _computed ) { + var width, minWidth, maxWidth, + computed = _computed || getStyles( elem ), + + // getPropertyValue is only needed for .css('filter') in IE9, see #12537 + ret = computed ? computed.getPropertyValue( name ) || computed[ name ] : undefined, + style = elem.style; + + if ( computed ) { + + if ( ret === "" && !jQuery.contains( elem.ownerDocument, elem ) ) { + ret = jQuery.style( elem, name ); + } + + // A tribute to the "awesome hack by Dean Edwards" + // Chrome < 17 and Safari 5.0 uses "computed value" instead of "used value" for margin-right + // Safari 5.1.7 (at least) returns percentage for a larger set of values, but width seems to be reliably pixels + // this is against the CSSOM draft spec: http://dev.w3.org/csswg/cssom/#resolved-values + if ( rnumnonpx.test( ret ) && rmargin.test( name ) ) { + + // Remember the original values + width = style.width; + minWidth = style.minWidth; + maxWidth = style.maxWidth; + + // Put in the new values to get a computed value out + style.minWidth = style.maxWidth = style.width = ret; + ret = computed.width; + + // Revert the changed values + style.width = width; + style.minWidth = minWidth; + style.maxWidth = maxWidth; + } + } + + return ret; + }; +} else if ( document.documentElement.currentStyle ) { + getStyles = function( elem ) { + return elem.currentStyle; + }; + + curCSS = function( elem, name, _computed ) { + var left, rs, rsLeft, + computed = _computed || getStyles( elem ), + ret = computed ? computed[ name ] : undefined, + style = elem.style; + + // Avoid setting ret to empty string here + // so we don't default to auto + if ( ret == null && style && style[ name ] ) { + ret = style[ name ]; + } + + // From the awesome hack by Dean Edwards + // http://erik.eae.net/archives/2007/07/27/18.54.15/#comment-102291 + + // If we're not dealing with a regular pixel number + // but a number that has a weird ending, we need to convert it to pixels + // but not position css attributes, as those are proportional to the parent element instead + // and we can't measure the parent instead because it might trigger a "stacking dolls" problem + if ( rnumnonpx.test( ret ) && !rposition.test( name ) ) { + + // Remember the original values + left = style.left; + rs = elem.runtimeStyle; + rsLeft = rs && rs.left; + + // Put in the new values to get a computed value out + if ( rsLeft ) { + rs.left = elem.currentStyle.left; + } + style.left = name === "fontSize" ? "1em" : ret; + ret = style.pixelLeft + "px"; + + // Revert the changed values + style.left = left; + if ( rsLeft ) { + rs.left = rsLeft; + } + } + + return ret === "" ? "auto" : ret; + }; +} + +function setPositiveNumber( elem, value, subtract ) { + var matches = rnumsplit.exec( value ); + return matches ? + // Guard against undefined "subtract", e.g., when used as in cssHooks + Math.max( 0, matches[ 1 ] - ( subtract || 0 ) ) + ( matches[ 2 ] || "px" ) : + value; +} + +function augmentWidthOrHeight( elem, name, extra, isBorderBox, styles ) { + var i = extra === ( isBorderBox ? "border" : "content" ) ? + // If we already have the right measurement, avoid augmentation + 4 : + // Otherwise initialize for horizontal or vertical properties + name === "width" ? 1 : 0, + + val = 0; + + for ( ; i < 4; i += 2 ) { + // both box models exclude margin, so add it if we want it + if ( extra === "margin" ) { + val += jQuery.css( elem, extra + cssExpand[ i ], true, styles ); + } + + if ( isBorderBox ) { + // border-box includes padding, so remove it if we want content + if ( extra === "content" ) { + val -= jQuery.css( elem, "padding" + cssExpand[ i ], true, styles ); + } + + // at this point, extra isn't border nor margin, so remove border + if ( extra !== "margin" ) { + val -= jQuery.css( elem, "border" + cssExpand[ i ] + "Width", true, styles ); + } + } else { + // at this point, extra isn't content, so add padding + val += jQuery.css( elem, "padding" + cssExpand[ i ], true, styles ); + + // at this point, extra isn't content nor padding, so add border + if ( extra !== "padding" ) { + val += jQuery.css( elem, "border" + cssExpand[ i ] + "Width", true, styles ); + } + } + } + + return val; +} + +function getWidthOrHeight( elem, name, extra ) { + + // Start with offset property, which is equivalent to the border-box value + var valueIsBorderBox = true, + val = name === "width" ? elem.offsetWidth : elem.offsetHeight, + styles = getStyles( elem ), + isBorderBox = jQuery.support.boxSizing && jQuery.css( elem, "boxSizing", false, styles ) === "border-box"; + + // some non-html elements return undefined for offsetWidth, so check for null/undefined + // svg - https://bugzilla.mozilla.org/show_bug.cgi?id=649285 + // MathML - https://bugzilla.mozilla.org/show_bug.cgi?id=491668 + if ( val <= 0 || val == null ) { + // Fall back to computed then uncomputed css if necessary + val = curCSS( elem, name, styles ); + if ( val < 0 || val == null ) { + val = elem.style[ name ]; + } + + // Computed unit is not pixels. Stop here and return. + if ( rnumnonpx.test(val) ) { + return val; + } + + // we need the check for style in case a browser which returns unreliable values + // for getComputedStyle silently falls back to the reliable elem.style + valueIsBorderBox = isBorderBox && ( jQuery.support.boxSizingReliable || val === elem.style[ name ] ); + + // Normalize "", auto, and prepare for extra + val = parseFloat( val ) || 0; + } + + // use the active box-sizing model to add/subtract irrelevant styles + return ( val + + augmentWidthOrHeight( + elem, + name, + extra || ( isBorderBox ? "border" : "content" ), + valueIsBorderBox, + styles + ) + ) + "px"; +} + +// Try to determine the default display value of an element +function css_defaultDisplay( nodeName ) { + var doc = document, + display = elemdisplay[ nodeName ]; + + if ( !display ) { + display = actualDisplay( nodeName, doc ); + + // If the simple way fails, read from inside an iframe + if ( display === "none" || !display ) { + // Use the already-created iframe if possible + iframe = ( iframe || + jQuery("