diff --git a/pinot-common/src/main/java/org/apache/pinot/common/utils/tls/RenewableTlsUtils.java b/pinot-common/src/main/java/org/apache/pinot/common/utils/tls/RenewableTlsUtils.java index 8c50dfaf592f..32903c8471ae 100644 --- a/pinot-common/src/main/java/org/apache/pinot/common/utils/tls/RenewableTlsUtils.java +++ b/pinot-common/src/main/java/org/apache/pinot/common/utils/tls/RenewableTlsUtils.java @@ -37,6 +37,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import nl.altindag.ssl.SSLFactory; import nl.altindag.ssl.keymanager.HotSwappableX509ExtendedKeyManager; import nl.altindag.ssl.trustmanager.HotSwappableX509ExtendedTrustManager; @@ -52,6 +53,8 @@ public class RenewableTlsUtils { private static final Logger LOGGER = LoggerFactory.getLogger(RenewableTlsUtils.class); private static final String FILE_SCHEME = "file"; + private static final int CERT_RELOAD_JOB_INTERVAL_IN_MINUTES = 1440; + private static final int CERT_RELOAD_JOB_INITAL_DELAY_IN_MINUTES = 20; private RenewableTlsUtils() { // left blank @@ -216,6 +219,26 @@ static void enableAutoRenewalFromFileStoreForSSLFactory(SSLFactory sslFactory, S throw new RuntimeException(e); } }); + + // Reloading SSL Factory blindly every day in a new scheduled thread + // to prevent any discrepancies with the FileWatcher. There has been scenarios where + // the FileWatcher is not reliable in detecting changes and reloading the certs as + // expected when changed. The certs were reloaded successfully in few components but + // it was never detected and run in others. This will result in few components with + // stale certificates. In order to prevent this issue, we are adding a new scheduled + // thread which will run once a day and reload the certs. + + Executors.newSingleThreadScheduledExecutor().scheduleAtFixedRate(() -> { + LOGGER.info("Creating a scheduled thread to reloadSsl once a day"); + try { + reloadSslFactory(sslFactory, + keyStoreType, keyStorePath, keyStorePassword, + trustStoreType, trustStorePath, trustStorePassword, + sslContextProtocol, secureRandom, insecureModeSupplier); + } catch (Exception e) { + throw new RuntimeException(e); + } + }, CERT_RELOAD_JOB_INITAL_DELAY_IN_MINUTES, CERT_RELOAD_JOB_INTERVAL_IN_MINUTES, TimeUnit.MINUTES); } catch (Exception e) { throw new RuntimeException(e); } @@ -233,8 +256,6 @@ static void reloadSslFactoryWhenFileStoreChanges(SSLFactory baseSslFactory, Map> watchKeyPathMap = new HashMap<>(); registerFile(watchService, watchKeyPathMap, keyStorePath); registerFile(watchService, watchKeyPathMap, trustStorePath); - int maxSslFactoryReloadingAttempts = 3; - int sslFactoryReloadingRetryDelayMs = 1000; WatchKey key; while ((key = watchService.take()) != null) { for (WatchEvent event : key.pollEvents()) { @@ -243,38 +264,54 @@ static void reloadSslFactoryWhenFileStoreChanges(SSLFactory baseSslFactory, LOGGER.info("Detected change in file: {}, try to renew SSLFactory {} " + "(built from key store {} and truststore {})", changedFile, baseSslFactory, keyStorePath, trustStorePath); - try { - // Need to retry a few times because when one file (key store or trust store) is updated, the other file - // (trust store or key store) may not have been fully written yet, so we need to wait a bit and retry. - RetryPolicies.fixedDelayRetryPolicy(maxSslFactoryReloadingAttempts, sslFactoryReloadingRetryDelayMs) - .attempt(() -> { - try { - SSLFactory updatedSslFactory = - createSSLFactory(keyStoreType, keyStorePath, keyStorePassword, trustStoreType, trustStorePath, - trustStorePassword, sslContextProtocol, secureRandom, false, insecureModeSupplier.get()); - SSLFactoryUtils.reload(baseSslFactory, updatedSslFactory); - LOGGER.info("Successfully renewed SSLFactory {} (built from key store {} and truststore {}) on file" - + " {} changes", baseSslFactory, keyStorePath, trustStorePath, changedFile); - return true; - } catch (Exception e) { - LOGGER.info( - "Encountered issues when renewing SSLFactory {} (built from key store {} and truststore {}) on " - + "file {} changes", baseSslFactory, keyStorePath, trustStorePath, changedFile, e); - return false; - } - }); - } catch (Exception e) { - LOGGER.error( - "Failed to renew SSLFactory {} (built from key store {} and truststore {}) on file {} changes after {} " - + "retries", baseSslFactory, keyStorePath, trustStorePath, changedFile, - maxSslFactoryReloadingAttempts, e); - } + + reloadSslFactory(baseSslFactory, keyStoreType, keyStorePath, keyStorePassword, trustStoreType, + trustStorePath, trustStorePassword, sslContextProtocol, secureRandom, insecureModeSupplier); } } key.reset(); } } + private static synchronized void reloadSslFactory(SSLFactory baseSslFactory, + String keyStoreType, String keyStorePath, String keyStorePassword, + String trustStoreType, String trustStorePath, String trustStorePassword, + String sslContextProtocol, SecureRandom secureRandom, Supplier insecureModeSupplier) { + LOGGER.info("reloadSslFactory :: Enable auto renewal of SSLFactory {}, key store {}, trust store {}", + baseSslFactory, keyStorePath, trustStorePath); + int maxSslFactoryReloadingAttempts = 3; + int sslFactoryReloadingRetryDelayMs = 1000; + + try { + // Need to retry a few times because when one file (key store or trust store) is updated, the other file + // (trust store or key store) may not have been fully written yet, so we need to wait a bit and retry. + RetryPolicies.fixedDelayRetryPolicy(maxSslFactoryReloadingAttempts, sslFactoryReloadingRetryDelayMs) + .attempt(() -> { + try { + SSLFactory updatedSslFactory = + createSSLFactory(keyStoreType, keyStorePath, keyStorePassword, trustStoreType, trustStorePath, + trustStorePassword, sslContextProtocol, secureRandom, false, insecureModeSupplier.get()); + SSLFactoryUtils.reload(baseSslFactory, updatedSslFactory); + LOGGER.info("reloadSslFactory :: Successfully renewed SSLFactory {} " + + "(built from key store {} and " + + "truststore {}) on file", baseSslFactory, keyStorePath, trustStorePath); + return true; + } catch (Exception e) { + LOGGER.info( + "reloadSslFactory :: Encountered issues when renewing SSLFactory " + + "{} (built from key store {} and " + + "truststore {}) on ", baseSslFactory, keyStorePath, trustStorePath, e); + return false; + } + }); + } catch (Exception e) { + LOGGER.error( + "reloadSslFactory :: Failed to renew SSLFactory {} (built from key store {} and truststore {}) after {} " + + "retries", baseSslFactory, keyStorePath, trustStorePath, + maxSslFactoryReloadingAttempts, e); + } + } + @VisibleForTesting static void registerFile(WatchService watchService, Map> keyPathMap, String filePath) throws IOException, URISyntaxException {