diff --git a/CHANGES.md b/CHANGES.md index 97e1c6e4..c0c6ad24 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,9 +5,12 @@ Release Notes. Apollo Java 2.4.0 ------------------ + * [Fix the Cannot enhance @Configuration bean definition issue](https://github.com/apolloconfig/apollo-java/pull/82) * [Feature openapi query namespace support not fill item](https://github.com/apolloconfig/apollo-java/pull/83) * [Add more observability in apollo config client](https://github.com/apolloconfig/apollo-java/pull/74) +* [Feature Support Kubernetes ConfigMap cache for Apollo java, golang client](https://github.com/apolloconfig/apollo-java/pull/79) + ------------------ All issues and pull requests are [here](https://github.com/apolloconfig/apollo-java/milestone/4?closed=1) diff --git a/apollo-client/pom.xml b/apollo-client/pom.xml index f643256a..1b51a8d9 100644 --- a/apollo-client/pom.xml +++ b/apollo-client/pom.xml @@ -70,6 +70,11 @@ spring-boot-configuration-processor true + + io.kubernetes + client-java + true + org.eclipse.jetty diff --git a/apollo-client/src/main/java/com/ctrip/framework/apollo/enums/ConfigSourceType.java b/apollo-client/src/main/java/com/ctrip/framework/apollo/enums/ConfigSourceType.java index b8249472..d3e3507e 100644 --- a/apollo-client/src/main/java/com/ctrip/framework/apollo/enums/ConfigSourceType.java +++ b/apollo-client/src/main/java/com/ctrip/framework/apollo/enums/ConfigSourceType.java @@ -22,7 +22,10 @@ * @since 1.1.0 */ public enum ConfigSourceType { - REMOTE("Loaded from remote config service"), LOCAL("Loaded from local cache"), NONE("Load failed"); + REMOTE("Loaded from remote config service"), + LOCAL("Loaded from local cache"), + CONFIGMAP("Loaded from k8s config map"), + NONE("Load failed"); private final String description; diff --git a/apollo-client/src/main/java/com/ctrip/framework/apollo/internals/K8sConfigMapConfigRepository.java b/apollo-client/src/main/java/com/ctrip/framework/apollo/internals/K8sConfigMapConfigRepository.java new file mode 100644 index 00000000..5b5f9124 --- /dev/null +++ b/apollo-client/src/main/java/com/ctrip/framework/apollo/internals/K8sConfigMapConfigRepository.java @@ -0,0 +1,263 @@ +/* + * Copyright 2022 Apollo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.ctrip.framework.apollo.internals; + +import com.ctrip.framework.apollo.kubernetes.KubernetesManager; +import com.ctrip.framework.apollo.build.ApolloInjector; +import com.ctrip.framework.apollo.core.ConfigConsts; +import com.ctrip.framework.apollo.core.utils.DeferredLoggerFactory; +import com.ctrip.framework.apollo.core.utils.StringUtils; +import com.ctrip.framework.apollo.enums.ConfigSourceType; +import com.ctrip.framework.apollo.exceptions.ApolloConfigException; +import com.ctrip.framework.apollo.tracer.Tracer; +import com.ctrip.framework.apollo.tracer.spi.Transaction; +import com.ctrip.framework.apollo.util.ConfigUtil; +import com.ctrip.framework.apollo.util.ExceptionUtil; +import com.ctrip.framework.apollo.util.escape.EscapeUtil; +import com.google.common.base.Preconditions; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import org.slf4j.Logger; + +import java.lang.reflect.Type; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +/** + * @author dyx1234 + */ +public class K8sConfigMapConfigRepository extends AbstractConfigRepository + implements RepositoryChangeListener { + private static final Logger logger = DeferredLoggerFactory.getLogger(K8sConfigMapConfigRepository.class); + private final String namespace; + private String configMapName; + private String configMapKey; + private final String k8sNamespace; + private final ConfigUtil configUtil; + private final KubernetesManager kubernetesManager; + private volatile Properties configMapProperties; + private volatile ConfigRepository upstream; + private volatile ConfigSourceType sourceType = ConfigSourceType.CONFIGMAP; + private static final Gson GSON = new Gson(); + + + public K8sConfigMapConfigRepository(String namespace, ConfigRepository upstream) { + this.namespace = namespace; + configUtil = ApolloInjector.getInstance(ConfigUtil.class); + kubernetesManager = ApolloInjector.getInstance(KubernetesManager.class); + k8sNamespace = configUtil.getK8sNamespace(); + + this.setConfigMapKey(configUtil.getCluster(), namespace); + this.setConfigMapName(configUtil.getAppId(), false); + this.setUpstreamRepository(upstream); + } + + private void setConfigMapKey(String cluster, String namespace) { + // cluster: User Definition >idc>default + if (StringUtils.isBlank(cluster)) { + configMapKey = EscapeUtil.createConfigMapKey("default", namespace); + return; + } + configMapKey = EscapeUtil.createConfigMapKey(cluster, namespace); + } + + private void setConfigMapName(String appId, boolean syncImmediately) { + Preconditions.checkNotNull(appId, "AppId cannot be null"); + configMapName = ConfigConsts.APOLLO_CONFIG_CACHE + appId; + this.checkConfigMapName(configMapName); + if (syncImmediately) { + this.sync(); + } + } + + private void checkConfigMapName(String configMapName) { + if (StringUtils.isBlank(configMapName)) { + throw new IllegalArgumentException("ConfigMap name cannot be null"); + } + if (kubernetesManager.checkConfigMapExist(k8sNamespace, configMapName)) { + return; + } + // Create an empty configmap, write the new value in the update event + Transaction transaction = Tracer.newTransaction("Apollo.ConfigService", "createK8sConfigMap"); + transaction.addData("configMapName", configMapName); + try { + kubernetesManager.createConfigMap(k8sNamespace, configMapName, null); + transaction.setStatus(Transaction.SUCCESS); + } catch (Throwable ex) { + Tracer.logEvent("ApolloConfigException", ExceptionUtil.getDetailMessage(ex)); + transaction.setStatus(ex); + throw new ApolloConfigException("Create configmap failed!", ex); + } finally { + transaction.complete(); + } + } + + @Override + public Properties getConfig() { + if (configMapProperties == null) { + sync(); + } + Properties result = propertiesFactory.getPropertiesInstance(); + result.putAll(configMapProperties); + return result; + } + + /** + * Update the memory when the configuration center changes + * + * @param upstreamConfigRepository the upstream repo + */ + @Override + public void setUpstreamRepository(ConfigRepository upstreamConfigRepository) { + if (upstreamConfigRepository == null) { + return; + } + //clear previous listener + if (upstream != null) { + upstream.removeChangeListener(this); + } + upstream = upstreamConfigRepository; + upstreamConfigRepository.addChangeListener(this); + } + + @Override + public ConfigSourceType getSourceType() { + return sourceType; + } + + /** + * Sync the configmap + */ + @Override + protected void sync() { + // Chain recovery, first read from upstream data source + boolean syncFromUpstreamResultSuccess = trySyncFromUpstream(); + + if (syncFromUpstreamResultSuccess) { + return; + } + + Transaction transaction = Tracer.newTransaction("Apollo.ConfigService", "syncK8sConfigMap"); + Throwable exception = null; + try { + configMapProperties = loadFromK8sConfigMap(); + sourceType = ConfigSourceType.CONFIGMAP; + transaction.setStatus(Transaction.SUCCESS); + } catch (Throwable ex) { + Tracer.logEvent("ApolloConfigException", ExceptionUtil.getDetailMessage(ex)); + transaction.setStatus(ex); + exception = ex; + } finally { + transaction.complete(); + } + + if (configMapProperties == null) { + sourceType = ConfigSourceType.NONE; + throw new ApolloConfigException( + "Load config from Kubernetes ConfigMap failed!", exception); + } + } + + Properties loadFromK8sConfigMap() { + Preconditions.checkNotNull(configMapName, "ConfigMap name cannot be null"); + + try { + String jsonConfig = kubernetesManager.getValueFromConfigMap(k8sNamespace, configMapName, configMapKey); + + // Convert jsonConfig to properties + Properties properties = propertiesFactory.getPropertiesInstance(); + if (jsonConfig != null && !jsonConfig.isEmpty()) { + Type type = new TypeToken>() {}.getType(); + Map configMap = GSON.fromJson(jsonConfig, type); + configMap.forEach(properties::setProperty); + } + return properties; + } catch (Exception ex) { + Tracer.logError(ex); + throw new ApolloConfigException(String + .format("Load config from Kubernetes ConfigMap %s failed!", configMapName), ex); + } + } + + private boolean trySyncFromUpstream() { + if (upstream == null) { + return false; + } + try { + updateConfigMapProperties(upstream.getConfig(), upstream.getSourceType()); + return true; + } catch (Throwable ex) { + Tracer.logError(ex); + logger.warn("Sync config from upstream repository {} failed, reason: {}", upstream.getClass(), + ExceptionUtil.getDetailMessage(ex)); + } + return false; + } + + private synchronized void updateConfigMapProperties(Properties newProperties, ConfigSourceType sourceType) { + this.sourceType = sourceType; + if (newProperties == null || newProperties.equals(configMapProperties)) { + return; + } + this.configMapProperties = newProperties; + persistConfigMap(configMapProperties); + } + + /** + * Update the memory + * + * @param namespace the namespace of this repository change + * @param newProperties the properties after change + */ + @Override + public void onRepositoryChange(String namespace, Properties newProperties) { + if (newProperties == null || newProperties.equals(configMapProperties)) { + return; + } + Properties newFileProperties = propertiesFactory.getPropertiesInstance(); + newFileProperties.putAll(newProperties); + updateConfigMapProperties(newFileProperties, upstream.getSourceType()); + this.fireRepositoryChange(namespace, newProperties); + } + + void persistConfigMap(Properties properties) { + Transaction transaction = Tracer.newTransaction("Apollo.ConfigService", "persistK8sConfigMap"); + transaction.addData("configMapName", configMapName); + transaction.addData("k8sNamespace", k8sNamespace); + try { + // Convert properties to a JSON string using Gson + String jsonConfig = GSON.toJson(properties); + Map data = new HashMap<>(); + data.put(configMapKey, jsonConfig); + + // update configmap + kubernetesManager.updateConfigMap(k8sNamespace, configMapName, data); + transaction.setStatus(Transaction.SUCCESS); + } catch (Exception ex) { + ApolloConfigException exception = + new ApolloConfigException( + String.format("Persist config to Kubernetes ConfigMap %s failed!", configMapName), ex); + Tracer.logError(exception); + transaction.setStatus(exception); + logger.error("Persist config to Kubernetes ConfigMap failed!", exception); + } finally { + transaction.complete(); + } + } + +} diff --git a/apollo-client/src/main/java/com/ctrip/framework/apollo/kubernetes/KubernetesManager.java b/apollo-client/src/main/java/com/ctrip/framework/apollo/kubernetes/KubernetesManager.java new file mode 100644 index 00000000..4cf96f38 --- /dev/null +++ b/apollo-client/src/main/java/com/ctrip/framework/apollo/kubernetes/KubernetesManager.java @@ -0,0 +1,208 @@ +/* + * Copyright 2022 Apollo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.ctrip.framework.apollo.kubernetes; + +import com.ctrip.framework.apollo.core.utils.StringUtils; +import com.ctrip.framework.apollo.exceptions.ApolloConfigException; +import io.kubernetes.client.openapi.ApiClient; +import io.kubernetes.client.openapi.ApiException; +import io.kubernetes.client.openapi.apis.CoreV1Api; +import io.kubernetes.client.openapi.models.V1ConfigMap; +import io.kubernetes.client.openapi.models.V1ObjectMeta; +import io.kubernetes.client.util.Config; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.TimeUnit; + +@Service +public class KubernetesManager { + private static final Logger logger = LoggerFactory.getLogger(KubernetesManager.class); + + private ApiClient client; + private CoreV1Api coreV1Api; + + public KubernetesManager() { + try { + client = Config.defaultClient(); + coreV1Api = new CoreV1Api(client); + } catch (Exception e) { + String errorMessage = "Failed to initialize Kubernetes client: " + e.getMessage(); + logger.error(errorMessage, e); + throw new RuntimeException(errorMessage, e); + } + } + + public KubernetesManager(CoreV1Api coreV1Api) { + this.coreV1Api = coreV1Api; + } + + private V1ConfigMap buildConfigMap(String name, String namespace, Map data) { + V1ObjectMeta metadata = new V1ObjectMeta() + .name(name) + .namespace(namespace); + + return new V1ConfigMap() + .apiVersion("v1") + .kind("ConfigMap") + .metadata(metadata) + .data(data); + } + + /** + * Creates a Kubernetes ConfigMap. + * + * @param k8sNamespace the namespace of the ConfigMap + * @param name the name of the ConfigMap + * @param data the data to be stored in the ConfigMap + * @return the name of the created ConfigMap + * @throws RuntimeException if an error occurs while creating the ConfigMap + */ + public String createConfigMap(String k8sNamespace, String name, Map data) { + if (StringUtils.isEmpty(k8sNamespace) || StringUtils.isEmpty(name)) { + logger.error("create configmap failed due to null or empty parameter: k8sNamespace={}, name={}", k8sNamespace, name); + return null; + } + V1ConfigMap configMap = buildConfigMap(name, k8sNamespace, data); + try { + coreV1Api.createNamespacedConfigMap(k8sNamespace, configMap, null, null, null, null); + logger.info("ConfigMap created successfully: name: {}, namespace: {}", name, k8sNamespace); + return name; + } catch (Exception e) { + logger.error("Failed to create ConfigMap: {}", e.getMessage(), e); + throw new RuntimeException("Failed to create ConfigMap: " + e.getMessage(), e); + } + } + + /** + * get value from config map + * + * @param k8sNamespace k8sNamespace + * @param name config map name + * @param key config map key (cluster+namespace) + * @return value(json string) + */ + public String getValueFromConfigMap(String k8sNamespace, String name, String key) { + if (StringUtils.isEmpty(k8sNamespace) || StringUtils.isEmpty(name) || StringUtils.isEmpty(key)) { + logger.error("Parameters can not be null or empty: k8sNamespace={}, name={}", k8sNamespace, name); + return null; + } + try { + V1ConfigMap configMap = coreV1Api.readNamespacedConfigMap(name, k8sNamespace, null); + if (!Objects.requireNonNull(configMap.getData()).containsKey(key)) { + logger.error("Specified key not found in ConfigMap: {}, k8sNamespace: {}, name: {}", name, k8sNamespace, name); + } + return configMap.getData().get(key); + } catch (Exception e) { + logger.error("Error occurred while getting value from ConfigMap: {}", e.getMessage(), e); + return null; + } + } + + /** + * update config map + * + * @param k8sNamespace configmap namespace + * @param name config map name + * @param data new data + * @return config map name + */ + // Set the retry times using the client retry mechanism (CAS) + public boolean updateConfigMap(String k8sNamespace, String name, Map data) throws ApiException { + if (StringUtils.isEmpty(k8sNamespace) || StringUtils.isEmpty(name)) { + logger.error("Parameters can not be null or empty: k8sNamespace={}, name={}", k8sNamespace, name); + return false; + } + + int maxRetries = 5; + int retryCount = 0; + long waitTime = 100; + + while (retryCount < maxRetries) { + try { + V1ConfigMap configmap = coreV1Api.readNamespacedConfigMap(name, k8sNamespace, null); + Map existingData = configmap.getData(); + if (existingData == null) { + existingData = new HashMap<>(); + } + + // Determine if the data contains its own kv and de-weight it + Map finalExistingData = existingData; + boolean containsEntry = data.entrySet().stream() + .allMatch(entry -> entry.getValue().equals(finalExistingData.get(entry.getKey()))); + + if (containsEntry) { + logger.info("Data is identical or already contains the entry, no update needed."); + return true; + } + + // Add new entries to the existing data + existingData.putAll(data); + configmap.setData(existingData); + + coreV1Api.replaceNamespacedConfigMap(name, k8sNamespace, configmap, null, null, null, null); + return true; + } catch (ApiException e) { + if (e.getCode() == 409) { + retryCount++; + logger.warn("Conflict occurred, retrying... ({})", retryCount); + try { + // Scramble the time, so that different machines in the distributed retry time is different + // The random ratio ranges from 0.9 to 1.1 + TimeUnit.MILLISECONDS.sleep((long) (waitTime * (0.9 + Math.random() * 0.2))); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + } + waitTime = Math.min(waitTime * 2, 1000); + } else { + logger.error("Error updating ConfigMap: {}", e.getMessage(), e); + throw e; + } + } + } + String errorMessage = String.format("Failed to update ConfigMap after %d retries: k8sNamespace=%s, name=%s", maxRetries, k8sNamespace, name); + logger.error(errorMessage); + throw new ApolloConfigException(errorMessage); + } + + /** + * check config map exist + * + * @param k8sNamespace config map namespace + * @param configMapName config map name + * @return true if config map exist, false otherwise + */ + public boolean checkConfigMapExist(String k8sNamespace, String configMapName) { + if (StringUtils.isEmpty(k8sNamespace) || StringUtils.isEmpty(configMapName)) { + logger.error("Parameters can not be null or empty: k8sNamespace={}, configMapName={}", k8sNamespace, configMapName); + return false; + } + try { + logger.info("Check whether ConfigMap exists, configMapName: {}", configMapName); + coreV1Api.readNamespacedConfigMap(configMapName, k8sNamespace, null); + return true; + } catch (Exception e) { + // configmap not exist + logger.info("ConfigMap not existence"); + return false; + } + } +} diff --git a/apollo-client/src/main/java/com/ctrip/framework/apollo/spi/DefaultConfigFactory.java b/apollo-client/src/main/java/com/ctrip/framework/apollo/spi/DefaultConfigFactory.java index 36c300f4..d74b6c63 100644 --- a/apollo-client/src/main/java/com/ctrip/framework/apollo/spi/DefaultConfigFactory.java +++ b/apollo-client/src/main/java/com/ctrip/framework/apollo/spi/DefaultConfigFactory.java @@ -33,6 +33,7 @@ import com.ctrip.framework.apollo.internals.XmlConfigFile; import com.ctrip.framework.apollo.internals.YamlConfigFile; import com.ctrip.framework.apollo.internals.YmlConfigFile; +import com.ctrip.framework.apollo.internals.K8sConfigMapConfigRepository; import com.ctrip.framework.apollo.util.ConfigUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -111,7 +112,9 @@ public ConfigFile createConfigFile(String namespace, ConfigFileFormat configFile } ConfigRepository createConfigRepository(String namespace) { - if (m_configUtil.isPropertyFileCacheEnabled()) { + if (m_configUtil.isPropertyKubernetesCacheEnabled()) { + return createConfigMapConfigRepository(namespace); + } else if (m_configUtil.isPropertyFileCacheEnabled()) { return createLocalConfigRepository(namespace); } return createRemoteConfigRepository(namespace); @@ -133,6 +136,15 @@ LocalFileConfigRepository createLocalConfigRepository(String namespace) { return new LocalFileConfigRepository(namespace, createRemoteConfigRepository(namespace)); } + /** + * Creates a Kubernetes config map repository for a given namespace + * @param namespace the namespace of the repository + * @return the newly created repository for the given namespace + */ + private ConfigRepository createConfigMapConfigRepository(String namespace) { + return new K8sConfigMapConfigRepository(namespace, createLocalConfigRepository(namespace)); + } + RemoteConfigRepository createRemoteConfigRepository(String namespace) { return new RemoteConfigRepository(namespace); } diff --git a/apollo-client/src/main/java/com/ctrip/framework/apollo/util/ConfigUtil.java b/apollo-client/src/main/java/com/ctrip/framework/apollo/util/ConfigUtil.java index 83a324f0..e322491c 100644 --- a/apollo-client/src/main/java/com/ctrip/framework/apollo/util/ConfigUtil.java +++ b/apollo-client/src/main/java/com/ctrip/framework/apollo/util/ConfigUtil.java @@ -71,6 +71,7 @@ public class ConfigUtil { private boolean propertyNamesCacheEnabled = false; private boolean propertyFileCacheEnabled = true; private boolean overrideSystemProperties = true; + private boolean propertyKubernetesCacheEnabled = false; private boolean clientMonitorEnabled = false; private boolean clientMonitorJmxEnabled = false; private String monitorExternalType = "NONE"; @@ -91,6 +92,7 @@ public ConfigUtil() { initPropertyNamesCacheEnabled(); initPropertyFileCacheEnabled(); initOverrideSystemProperties(); + initPropertyKubernetesCacheEnabled(); initClientMonitorEnabled(); initClientMonitorJmxEnabled(); initClientMonitorExternalType(); @@ -376,6 +378,34 @@ private String getDeprecatedCustomizedCacheRoot() { return cacheRoot; } + public String getK8sNamespace() { + String k8sNamespace = getCacheKubernetesNamespace(); + + if (!Strings.isNullOrEmpty(k8sNamespace)) { + return k8sNamespace; + } + + return ConfigConsts.KUBERNETES_CACHE_CONFIG_MAP_NAMESPACE_DEFAULT; + } + + private String getCacheKubernetesNamespace() { + // 1. Get from System Property + String k8sNamespace = System.getProperty(ApolloClientSystemConsts.APOLLO_CACHE_KUBERNETES_NAMESPACE); + if (Strings.isNullOrEmpty(k8sNamespace)) { + // 2. Get from OS environment variable + k8sNamespace = System.getenv(ApolloClientSystemConsts.APOLLO_CACHE_KUBERNETES_NAMESPACE_ENVIRONMENT_VARIABLES); + } + if (Strings.isNullOrEmpty(k8sNamespace)) { + // 3. Get from server.properties + k8sNamespace = Foundation.server().getProperty(ApolloClientSystemConsts.APOLLO_CACHE_KUBERNETES_NAMESPACE, null); + } + if (Strings.isNullOrEmpty(k8sNamespace)) { + // 4. Get from app.properties + k8sNamespace = Foundation.app().getProperty(ApolloClientSystemConsts.APOLLO_CACHE_KUBERNETES_NAMESPACE, null); + } + return k8sNamespace; + } + public boolean isInLocalMode() { try { return Env.LOCAL == getApolloEnv(); @@ -479,6 +509,10 @@ public boolean isPropertyFileCacheEnabled() { return propertyFileCacheEnabled; } + public boolean isPropertyKubernetesCacheEnabled() { + return propertyKubernetesCacheEnabled; + } + public boolean isOverrideSystemProperties() { return overrideSystemProperties; } @@ -500,11 +534,15 @@ private void initOverrideSystemProperties() { ApolloClientSystemConsts.APOLLO_OVERRIDE_SYSTEM_PROPERTIES, overrideSystemProperties); } - - + + private void initPropertyKubernetesCacheEnabled() { + propertyKubernetesCacheEnabled = getPropertyBoolean(ApolloClientSystemConsts.APOLLO_KUBERNETES_CACHE_ENABLE, + ApolloClientSystemConsts.APOLLO_KUBERNETES_CACHE_ENABLE_ENVIRONMENT_VARIABLES, + propertyKubernetesCacheEnabled); + } + private void initClientMonitorExternalType() { monitorExternalType = System.getProperty(ApolloClientSystemConsts.APOLLO_CLIENT_MONITOR_EXTERNAL_TYPE); - if (Strings.isNullOrEmpty(monitorExternalType)) { monitorExternalType = Foundation.app() .getProperty(ApolloClientSystemConsts.APOLLO_CLIENT_MONITOR_EXTERNAL_TYPE, "NONE"); diff --git a/apollo-client/src/main/java/com/ctrip/framework/apollo/util/escape/EscapeUtil.java b/apollo-client/src/main/java/com/ctrip/framework/apollo/util/escape/EscapeUtil.java new file mode 100644 index 00000000..8a499115 --- /dev/null +++ b/apollo-client/src/main/java/com/ctrip/framework/apollo/util/escape/EscapeUtil.java @@ -0,0 +1,42 @@ +/* + * Copyright 2022 Apollo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.ctrip.framework.apollo.util.escape; + +/** + * @author dyx1234 + */ +public class EscapeUtil { + + private static final String SINGLE_UNDERSCORE = "_"; + private static final String DOUBLE_UNDERSCORE = "__"; + private static final String TRIPLE_UNDERSCORE = "___"; + + // Escapes a single underscore in a namespace + public static String escapeNamespace(String namespace) { + if (namespace == null || namespace.isEmpty()) { + throw new IllegalArgumentException("Namespace cannot be null or empty"); + } + return namespace.replace(SINGLE_UNDERSCORE, DOUBLE_UNDERSCORE); + } + + // Concatenate the cluster and the escaped namespace, using three underscores as delimiters + public static String createConfigMapKey(String cluster, String namespace) { + String escapedNamespace = escapeNamespace(namespace); + return String.join(TRIPLE_UNDERSCORE, cluster, escapedNamespace); + } + +} diff --git a/apollo-client/src/main/resources/META-INF/additional-spring-configuration-metadata.json b/apollo-client/src/main/resources/META-INF/additional-spring-configuration-metadata.json index 0bb9b644..1a268b52 100644 --- a/apollo-client/src/main/resources/META-INF/additional-spring-configuration-metadata.json +++ b/apollo-client/src/main/resources/META-INF/additional-spring-configuration-metadata.json @@ -112,6 +112,20 @@ "description": "enable property names cache.", "defaultValue": false }, + { + "name": "apollo.cache.kubernetes.enable", + "type": "java.lang.Boolean", + "sourceType": "com.ctrip.framework.apollo.util.ConfigUtil", + "description": "enable kubernetes configmap cache.", + "defaultValue": false + }, + { + "name": "apollo.cache.kubernetes.namespace", + "type": "java.lang.String", + "sourceType": "com.ctrip.framework.apollo.util.ConfigUtil", + "description": "kubernetes configmap namespace.", + "defaultValue": "default" + }, { "name": "apollo.property.order.enable", "type": "java.lang.Boolean", diff --git a/apollo-client/src/test/java/com/ctrip/framework/apollo/internals/K8sConfigMapConfigRepositoryTest.java b/apollo-client/src/test/java/com/ctrip/framework/apollo/internals/K8sConfigMapConfigRepositoryTest.java new file mode 100644 index 00000000..f0c02071 --- /dev/null +++ b/apollo-client/src/test/java/com/ctrip/framework/apollo/internals/K8sConfigMapConfigRepositoryTest.java @@ -0,0 +1,193 @@ +/* + * Copyright 2022 Apollo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.ctrip.framework.apollo.internals; + +import com.ctrip.framework.apollo.build.MockInjector; +import com.ctrip.framework.apollo.enums.ConfigSourceType; +import com.ctrip.framework.apollo.kubernetes.KubernetesManager; +import com.ctrip.framework.apollo.util.ConfigUtil; +import com.ctrip.framework.apollo.util.escape.EscapeUtil; +import io.kubernetes.client.openapi.ApiException; +import io.kubernetes.client.openapi.models.V1ConfigMap; +import io.kubernetes.client.openapi.models.V1ObjectMeta; +import org.junit.Before; +import org.junit.Test; +import org.springframework.test.util.ReflectionTestUtils; + +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.*; + +public class K8sConfigMapConfigRepositoryTest { + private static String someAppId = "someApp"; + private static String someCluster = "someCluster"; + private String someNamespace = "default"; + private static final String someConfigmapName = "apollo-configcache-someApp"; + + private static final String defaultKey = "defaultKey"; + private static final String defaultValue = "defaultValue"; + private static final String defaultJsonValue = "{\"id\":123,\"name\":\"John Doe\",\"email\":\"john.doe@example.com\"}"; + + private ConfigRepository upstreamRepo; + private Properties someProperties; + private ConfigSourceType someSourceType = ConfigSourceType.LOCAL; + private V1ConfigMap configMap; + private Map data; + private KubernetesManager kubernetesManager; + private K8sConfigMapConfigRepository k8sConfigMapConfigRepository; + + + @Before + public void setUp() { + // mock configUtil + MockInjector.setInstance(ConfigUtil.class, new MockConfigUtil()); + // mock kubernetesManager + kubernetesManager = mock(KubernetesManager.class); + MockInjector.setInstance(KubernetesManager.class, kubernetesManager); + + // mock upstream + someProperties = new Properties(); + someProperties.setProperty(defaultKey, defaultValue); + upstreamRepo = mock(ConfigRepository.class); + when(upstreamRepo.getConfig()).thenReturn(someProperties); + when(upstreamRepo.getSourceType()).thenReturn(someSourceType); + + // make configmap + data = new HashMap<>(); + data.put(defaultKey, defaultJsonValue); + configMap = new V1ConfigMap() + .metadata(new V1ObjectMeta().name(someAppId).namespace(someNamespace)) + .data(data); + + k8sConfigMapConfigRepository = new K8sConfigMapConfigRepository(someNamespace, upstreamRepo); + } + + /** + * 测试setConfigMapKey方法，当cluster和namespace都为正常值时 + */ + @Test + public void testSetConfigMapKeyUnderNormalConditions() throws Throwable { + // arrange + String cluster = "testCluster"; + String namespace = "test_Namespace_1"; + String escapedKey = "testCluster___test__Namespace__1"; + + // act + ReflectionTestUtils.invokeMethod(k8sConfigMapConfigRepository, "setConfigMapKey", cluster, namespace); + + // assert + String expectedConfigMapKey = EscapeUtil.createConfigMapKey(cluster, namespace); + assertEquals(escapedKey, ReflectionTestUtils.getField(k8sConfigMapConfigRepository, "configMapKey")); + assertEquals(expectedConfigMapKey, ReflectionTestUtils.getField(k8sConfigMapConfigRepository, "configMapKey")); + } + + /** + * 测试sync方法成功从上游数据源同步 + */ + @Test + public void testSyncSuccessFromUpstream() throws Throwable { + // arrange + k8sConfigMapConfigRepository.setUpstreamRepository(upstreamRepo); + + // act + k8sConfigMapConfigRepository.sync(); + + // assert + verify(upstreamRepo, times(1)).getConfig(); + } + + + /** + * 测试sync方法从上游数据源同步失败，成功从Kubernetes的ConfigMap中加载 + */ + @Test + public void testSyncFailFromUpstreamSuccessFromConfigMap() throws Throwable { + // arrange + ConfigRepository upstream = mock(ConfigRepository.class); + when(upstream.getConfig()).thenThrow(new RuntimeException("Upstream sync failed")); + k8sConfigMapConfigRepository.setUpstreamRepository(upstream); + when(kubernetesManager.getValueFromConfigMap(anyString(), anyString(), anyString())).thenReturn(data.get(defaultKey)); + + // act + k8sConfigMapConfigRepository.sync(); + + // assert + verify(kubernetesManager, times(1)).getValueFromConfigMap(anyString(), anyString(), anyString()); + } + + @Test + public void testGetConfig() { + // Arrange + Properties expectedProperties = new Properties(); + expectedProperties.setProperty(defaultKey, defaultValue); + when(upstreamRepo.getConfig()).thenReturn(expectedProperties); + // Act + Properties actualProperties = k8sConfigMapConfigRepository.getConfig(); + // Assert + assertNotNull(actualProperties); + assertEquals(defaultValue, actualProperties.getProperty(defaultKey)); + } + + @Test + public void testPersistConfigMap() throws ApiException { + // Arrange + Properties properties = new Properties(); + properties.setProperty(defaultKey, defaultValue); + // Act + k8sConfigMapConfigRepository.persistConfigMap(properties); + // Assert + verify(kubernetesManager, times(1)).updateConfigMap(anyString(), anyString(), anyMap()); + } + + @Test + public void testOnRepositoryChange() throws ApiException { + // Arrange + Properties newProperties = new Properties(); + newProperties.setProperty(defaultKey, defaultValue); + // Act + k8sConfigMapConfigRepository.onRepositoryChange(someNamespace, newProperties); + // Assert + verify(kubernetesManager, times(1)).updateConfigMap(anyString(), anyString(), anyMap()); + } + + @Test + public void testLoadFromK8sConfigMapSuccess() { + when(kubernetesManager.getValueFromConfigMap(anyString(), anyString(), anyString())).thenReturn(defaultJsonValue); + + Properties properties = k8sConfigMapConfigRepository.loadFromK8sConfigMap(); + + assertNotNull(properties); + } + + public static class MockConfigUtil extends ConfigUtil { + @Override + public String getAppId() { + return someAppId; + } + + @Override + public String getCluster() { + return someCluster; + } + } + +} diff --git a/apollo-client/src/test/java/com/ctrip/framework/apollo/kubernetes/KubernetesManagerTest.java b/apollo-client/src/test/java/com/ctrip/framework/apollo/kubernetes/KubernetesManagerTest.java new file mode 100644 index 00000000..c756d35d --- /dev/null +++ b/apollo-client/src/test/java/com/ctrip/framework/apollo/kubernetes/KubernetesManagerTest.java @@ -0,0 +1,208 @@ +/* + * Copyright 2022 Apollo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.ctrip.framework.apollo.kubernetes; + +import com.ctrip.framework.apollo.build.MockInjector; +import io.kubernetes.client.openapi.ApiException; +import io.kubernetes.client.openapi.apis.CoreV1Api; +import io.kubernetes.client.openapi.models.V1ConfigMap; +import io.kubernetes.client.openapi.models.V1ObjectMeta; +import org.junit.Before; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.mockito.Mockito.*; +import static org.junit.Assert.*; + +public class KubernetesManagerTest { + + private CoreV1Api coreV1Api; + private KubernetesManager kubernetesManager; + + @Before + public void setUp() { + coreV1Api = mock(CoreV1Api.class); + kubernetesManager = new KubernetesManager(coreV1Api); + + MockInjector.setInstance(KubernetesManager.class, kubernetesManager); + MockInjector.setInstance(CoreV1Api.class, coreV1Api); + } + + /** + * 测试 createConfigMap 成功创建配置 + */ + @Test + public void testCreateConfigMapSuccess() throws Exception { + // arrange + String namespace = "default"; + String name = "testConfigMap"; + Map data = new HashMap<>(); + data.put("key", "value"); + V1ConfigMap configMap = new V1ConfigMap() + .metadata(new V1ObjectMeta().name(name).namespace(namespace)) + .data(data); + + when(coreV1Api.createNamespacedConfigMap(eq(namespace), eq(configMap), isNull(), isNull(), isNull(),isNull())).thenReturn(configMap); + + // act + String result = kubernetesManager.createConfigMap(namespace, name, data); + + // assert + verify(coreV1Api, times(1)).createNamespacedConfigMap(eq(namespace), any(V1ConfigMap.class), isNull(), isNull(), isNull(),isNull()); + assert name.equals(result); + } + + /** + * 测试 createConfigMap 传入 null 作为数据，正常执行 + */ + @Test + public void testCreateConfigMapNullData() throws Exception { + // arrange + String namespace = "default"; + String name = "testConfigMap"; + Map data = null; + + // act + String result = kubernetesManager.createConfigMap(namespace, name, data); + + // assert + verify(coreV1Api, times(1)).createNamespacedConfigMap(eq(namespace), any(V1ConfigMap.class), isNull(), isNull(), isNull(),isNull()); + assert name.equals(result); + } + + /** + * 测试getValueFromConfigMap方法，当ConfigMap存在且包含指定key时返回正确的value + */ + @Test + public void testGetValueFromConfigMapReturnsValue() throws Exception { + // arrange + String namespace = "default"; + String name = "testConfigMap"; + String key = "testKey"; + String expectedValue = "testValue"; + V1ConfigMap configMap = new V1ConfigMap(); + configMap.putDataItem(key, expectedValue); + + when(coreV1Api.readNamespacedConfigMap(name, namespace, null)).thenReturn(configMap); + + // act + String actualValue = kubernetesManager.getValueFromConfigMap(namespace, name, key); + + // assert + assertEquals(expectedValue, actualValue); + } + + /** + * 测试getValueFromConfigMap方法，当ConfigMap不存在指定key时返回null + */ + @Test + public void testGetValueFromConfigMapKeyNotFound() throws Exception { + // arrange + String namespace = "default"; + String name = "testConfigMap"; + String key = "nonExistingKey"; + V1ConfigMap configMap = new V1ConfigMap(); + when(coreV1Api.readNamespacedConfigMap(name, namespace, null)).thenReturn(configMap); + + // act + String actualValue = kubernetesManager.getValueFromConfigMap(namespace, name, key); + + // assert + assertNull(actualValue); + } + + /** + * 测试updateConfigMap成功的情况 + */ + @Test + public void testUpdateConfigMapSuccess() throws Exception { + // arrange + String namespace = "default"; + String name = "testConfigMap"; + Map data = new HashMap<>(); + data.put("key", "value"); + V1ConfigMap configMap = new V1ConfigMap(); + configMap.metadata(new V1ObjectMeta().name(name).namespace(namespace)); + configMap.data(data); + + when(coreV1Api.readNamespacedConfigMap(name, namespace, null)).thenReturn(configMap); + when(coreV1Api.replaceNamespacedConfigMap(name, namespace, configMap, null, null, null, null)).thenReturn(configMap); + + // act + Boolean success = kubernetesManager.updateConfigMap(namespace, name, data); + + // assert + assertTrue(success); + } + + /** + * 测试ConfigMap存在时，checkConfigMapExist方法返回true + */ + @Test + public void testCheckConfigMapExistWhenConfigMapExists() throws Exception { + // arrange + String namespace = "default"; + String name = "testConfigMap"; + + // 创建一个模拟的 V1ConfigMap 实例 + V1ConfigMap mockConfigMap = new V1ConfigMap(); + mockConfigMap.setMetadata(new V1ObjectMeta().name(name).namespace(namespace)); + doReturn(mockConfigMap).when(coreV1Api).readNamespacedConfigMap(name, namespace, null); + + // act + boolean result = kubernetesManager.checkConfigMapExist(namespace, name); + + // assert + assertEquals(true, result); + } + + /** + * 测试ConfigMap不存在的情况，返回false + */ + @Test + public void testCheckConfigMapExistWhenConfigMapDoesNotExist() throws Exception { + // arrange + String namespace = "default"; + String name = "testConfigMap"; + doThrow(new ApiException("ConfigMap not exist")).when(coreV1Api).readNamespacedConfigMap(name, namespace, null); + + // act + boolean result = kubernetesManager.checkConfigMapExist(namespace, name); + + // assert + assertFalse(result); + } + + /** + * 测试参数k8sNamespace和configMapName都为空时，checkConfigMapExist方法返回false + */ + @Test + public void testCheckConfigMapExistWithEmptyNamespaceAndName() { + // arrange + String namespace = ""; + String name = ""; + + // act + boolean result = kubernetesManager.checkConfigMapExist(namespace, name); + + // assert + assertFalse(result); + } + +} diff --git a/apollo-client/src/test/java/com/ctrip/framework/apollo/util/ConfigUtilTest.java b/apollo-client/src/test/java/com/ctrip/framework/apollo/util/ConfigUtilTest.java index 18cda2dd..e9f8ea1e 100644 --- a/apollo-client/src/test/java/com/ctrip/framework/apollo/util/ConfigUtilTest.java +++ b/apollo-client/src/test/java/com/ctrip/framework/apollo/util/ConfigUtilTest.java @@ -47,6 +47,8 @@ public void tearDown() throws Exception { System.clearProperty(ApolloClientSystemConsts.APOLLO_CACHE_DIR); System.clearProperty(PropertiesFactory.APOLLO_PROPERTY_ORDER_ENABLE); System.clearProperty(ApolloClientSystemConsts.APOLLO_PROPERTY_NAMES_CACHE_ENABLE); + System.clearProperty(ApolloClientSystemConsts.APOLLO_CACHE_KUBERNETES_NAMESPACE); + System.clearProperty(ApolloClientSystemConsts.APOLLO_KUBERNETES_CACHE_ENABLE); } @Test @@ -243,6 +245,35 @@ public void testDefaultLocalCacheDir() throws Exception { assertEquals("/opt/data/" + someAppId, configUtil.getDefaultLocalCacheDir()); } + @Test + public void testK8sNamespaceWithSystemProperty() { + String someK8sNamespace = "someK8sNamespace"; + + System.setProperty(ApolloClientSystemConsts.APOLLO_CACHE_KUBERNETES_NAMESPACE, someK8sNamespace); + + ConfigUtil configUtil = new ConfigUtil(); + + assertEquals(someK8sNamespace, configUtil.getK8sNamespace()); + } + + @Test + public void testK8sNamespaceWithDefault() { + ConfigUtil configUtil = new ConfigUtil(); + + assertEquals(ConfigConsts.KUBERNETES_CACHE_CONFIG_MAP_NAMESPACE_DEFAULT, configUtil.getK8sNamespace()); + } + + @Test + public void testKubernetesCacheEnabledWithSystemProperty() { + boolean someKubernetesCacheEnabled = true; + + System.setProperty(ApolloClientSystemConsts.APOLLO_KUBERNETES_CACHE_ENABLE, String.valueOf(someKubernetesCacheEnabled)); + + ConfigUtil configUtil = new ConfigUtil(); + + assertTrue(configUtil.isPropertyKubernetesCacheEnabled()); + } + @Test public void testCustomizePropertiesOrdered() { boolean propertiesOrdered = true; diff --git a/apollo-core/src/main/java/com/ctrip/framework/apollo/core/ApolloClientSystemConsts.java b/apollo-core/src/main/java/com/ctrip/framework/apollo/core/ApolloClientSystemConsts.java index 2bf8f850..c199c1a2 100644 --- a/apollo-core/src/main/java/com/ctrip/framework/apollo/core/ApolloClientSystemConsts.java +++ b/apollo-core/src/main/java/com/ctrip/framework/apollo/core/ApolloClientSystemConsts.java @@ -73,6 +73,16 @@ public class ApolloClientSystemConsts { @Deprecated public static final String DEPRECATED_APOLLO_CACHE_DIR_ENVIRONMENT_VARIABLES = "APOLLO_CACHEDIR"; + /** + * kubernetes configmap cache namespace + */ + public static final String APOLLO_CACHE_KUBERNETES_NAMESPACE = "apollo.cache.kubernetes.namespace"; + + /** + * kubernetes configmap cache namespace environment variables + */ + public static final String APOLLO_CACHE_KUBERNETES_NAMESPACE_ENVIRONMENT_VARIABLES = "APOLLO_CACHE_KUBERNETES_NAMESPACE"; + /** * apollo client access key */ @@ -157,6 +167,16 @@ public class ApolloClientSystemConsts { */ public static final String APOLLO_CACHE_FILE_ENABLE_ENVIRONMENT_VARIABLES = "APOLLO_CACHE_FILE_ENABLE"; + /** + * enable property names cache + */ + public static final String APOLLO_KUBERNETES_CACHE_ENABLE = "apollo.cache.kubernetes.enable"; + + /** + * enable property names cache environment variables + */ + public static final String APOLLO_KUBERNETES_CACHE_ENABLE_ENVIRONMENT_VARIABLES = "APOLLO_KUBERNETES_CACHE_ENABLE"; + /** * enable apollo overrideSystemProperties */ diff --git a/apollo-core/src/main/java/com/ctrip/framework/apollo/core/ConfigConsts.java b/apollo-core/src/main/java/com/ctrip/framework/apollo/core/ConfigConsts.java index 19f33278..621c2c97 100644 --- a/apollo-core/src/main/java/com/ctrip/framework/apollo/core/ConfigConsts.java +++ b/apollo-core/src/main/java/com/ctrip/framework/apollo/core/ConfigConsts.java @@ -20,10 +20,12 @@ public interface ConfigConsts { String NAMESPACE_APPLICATION = "application"; String CLUSTER_NAME_DEFAULT = "default"; String CLUSTER_NAMESPACE_SEPARATOR = "+"; + String APOLLO_CONFIG_CACHE = "apollo-configcache-"; String APOLLO_CLUSTER_KEY = "apollo.cluster"; String APOLLO_META_KEY = "apollo.meta"; String CONFIG_FILE_CONTENT_KEY = "content"; String NO_APPID_PLACEHOLDER = "ApolloNoAppIdPlaceHolder"; + String KUBERNETES_CACHE_CONFIG_MAP_NAMESPACE_DEFAULT = "default"; String APOLLO_AUTO_UPDATE_INJECTED_SPRING_PROPERTIES = "ApolloAutoUpdateInjectedSpringProperties"; long NOTIFICATION_ID_PLACEHOLDER = -1; } diff --git a/pom.xml b/pom.xml index 31321a3c..a255aabd 100644 --- a/pom.xml +++ b/pom.xml @@ -164,6 +164,12 @@ pom import + + io.kubernetes + client-java + 18.0.0 + true +