Force cert reload on a daily basis. (#14535)

pinot-common/src/main/java/org/apache/pinot/common/utils/tls/RenewableTlsUtils.java

@@ -37,6 +37,7 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
 import nl.altindag.ssl.SSLFactory;
 import nl.altindag.ssl.keymanager.HotSwappableX509ExtendedKeyManager;
 import nl.altindag.ssl.trustmanager.HotSwappableX509ExtendedTrustManager;
@@ -52,6 +53,8 @@
 public class RenewableTlsUtils {
   private static final Logger LOGGER = LoggerFactory.getLogger(RenewableTlsUtils.class);
   private static final String FILE_SCHEME = "file";
+  private static final int CERT_RELOAD_JOB_INTERVAL_IN_MINUTES = 1440;
+  private static final int CERT_RELOAD_JOB_INITAL_DELAY_IN_MINUTES = 20;
 
   private RenewableTlsUtils() {
     // left blank
@@ -216,6 +219,26 @@ static void enableAutoRenewalFromFileStoreForSSLFactory(SSLFactory sslFactory, S
           throw new RuntimeException(e);
         }
       });
+
+      // Reloading SSL Factory blindly every day in a new scheduled thread
+      // to prevent any discrepancies with the FileWatcher. There has been scenarios where
+      // the FileWatcher is not reliable in detecting changes and reloading the certs as
+      // expected when changed. The certs were reloaded successfully in few components but
+      // it was never detected and run in others. This will result in few components with
+      // stale certificates. In order to prevent this issue, we are adding a new scheduled
+      // thread which will run once a day and reload the certs.
+
+      Executors.newSingleThreadScheduledExecutor().scheduleAtFixedRate(() -> {
+        LOGGER.info("Creating a scheduled thread to reloadSsl once a day");
+        try {
+          reloadSslFactory(sslFactory,
+              keyStoreType, keyStorePath, keyStorePassword,
+              trustStoreType, trustStorePath, trustStorePassword,
+              sslContextProtocol, secureRandom, insecureModeSupplier);
+        } catch (Exception e) {
+          throw new RuntimeException(e);
+        }
+      }, CERT_RELOAD_JOB_INITAL_DELAY_IN_MINUTES, CERT_RELOAD_JOB_INTERVAL_IN_MINUTES, TimeUnit.MINUTES);
     } catch (Exception e) {
       throw new RuntimeException(e);
     }
@@ -233,8 +256,6 @@ static void reloadSslFactoryWhenFileStoreChanges(SSLFactory baseSslFactory,
     Map<WatchKey, Set<Path>> watchKeyPathMap = new HashMap<>();
     registerFile(watchService, watchKeyPathMap, keyStorePath);
     registerFile(watchService, watchKeyPathMap, trustStorePath);
-    int maxSslFactoryReloadingAttempts = 3;
-    int sslFactoryReloadingRetryDelayMs = 1000;
     WatchKey key;
     while ((key = watchService.take()) != null) {
       for (WatchEvent<?> event : key.pollEvents()) {
@@ -243,38 +264,54 @@ static void reloadSslFactoryWhenFileStoreChanges(SSLFactory baseSslFactory,
           LOGGER.info("Detected change in file: {}, try to renew SSLFactory {} "
                   + "(built from key store {} and truststore {})",
               changedFile, baseSslFactory, keyStorePath, trustStorePath);
-          try {
-            // Need to retry a few times because when one file (key store or trust store) is updated, the other file
-            // (trust store or key store) may not have been fully written yet, so we need to wait a bit and retry.
-            RetryPolicies.fixedDelayRetryPolicy(maxSslFactoryReloadingAttempts, sslFactoryReloadingRetryDelayMs)
-                .attempt(() -> {
-                  try {
-                    SSLFactory updatedSslFactory =
-                        createSSLFactory(keyStoreType, keyStorePath, keyStorePassword, trustStoreType, trustStorePath,
-                            trustStorePassword, sslContextProtocol, secureRandom, false, insecureModeSupplier.get());
-                    SSLFactoryUtils.reload(baseSslFactory, updatedSslFactory);
-                    LOGGER.info("Successfully renewed SSLFactory {} (built from key store {} and truststore {}) on file"
-                        + " {} changes", baseSslFactory, keyStorePath, trustStorePath, changedFile);
-                    return true;
-                  } catch (Exception e) {
-                    LOGGER.info(
-                        "Encountered issues when renewing SSLFactory {} (built from key store {} and truststore {}) on "
-                            + "file {} changes", baseSslFactory, keyStorePath, trustStorePath, changedFile, e);
-                    return false;
-                  }
-                });
-          } catch (Exception e) {
-            LOGGER.error(
-                "Failed to renew SSLFactory {} (built from key store {} and truststore {}) on file {} changes after {} "
-                    + "retries", baseSslFactory, keyStorePath, trustStorePath, changedFile,
-                maxSslFactoryReloadingAttempts, e);
-          }
+
+          reloadSslFactory(baseSslFactory, keyStoreType, keyStorePath, keyStorePassword, trustStoreType,
+              trustStorePath, trustStorePassword, sslContextProtocol, secureRandom, insecureModeSupplier);
         }
       }
       key.reset();
     }
   }
 
+  private static synchronized void reloadSslFactory(SSLFactory baseSslFactory,
+      String keyStoreType, String keyStorePath, String keyStorePassword,
+      String trustStoreType, String trustStorePath, String trustStorePassword,
+      String sslContextProtocol, SecureRandom secureRandom, Supplier<Boolean> insecureModeSupplier) {
+    LOGGER.info("reloadSslFactory :: Enable auto renewal of SSLFactory {}, key store {}, trust store {}",
+        baseSslFactory, keyStorePath, trustStorePath);
+    int maxSslFactoryReloadingAttempts = 3;
+    int sslFactoryReloadingRetryDelayMs = 1000;
+
+    try {
+      // Need to retry a few times because when one file (key store or trust store) is updated, the other file
+      // (trust store or key store) may not have been fully written yet, so we need to wait a bit and retry.
+      RetryPolicies.fixedDelayRetryPolicy(maxSslFactoryReloadingAttempts, sslFactoryReloadingRetryDelayMs)
+          .attempt(() -> {
+            try {
+              SSLFactory updatedSslFactory =
+                  createSSLFactory(keyStoreType, keyStorePath, keyStorePassword, trustStoreType, trustStorePath,
+                      trustStorePassword, sslContextProtocol, secureRandom, false, insecureModeSupplier.get());
+              SSLFactoryUtils.reload(baseSslFactory, updatedSslFactory);
+              LOGGER.info("reloadSslFactory :: Successfully renewed SSLFactory {} "
+                  + "(built from key store {} and "
+                  + "truststore {}) on file", baseSslFactory, keyStorePath, trustStorePath);
+              return true;
+            } catch (Exception e) {
+              LOGGER.info(
+                  "reloadSslFactory :: Encountered issues when renewing SSLFactory "
+                  + "{} (built from key store {} and "
+                  + "truststore {}) on ", baseSslFactory, keyStorePath, trustStorePath, e);
+              return false;
+            }
+          });
+    } catch (Exception e) {
+      LOGGER.error(
+          "reloadSslFactory :: Failed to renew SSLFactory {} (built from key store {} and truststore {}) after {} "
+              + "retries", baseSslFactory, keyStorePath, trustStorePath,
+          maxSslFactoryReloadingAttempts, e);
+    }
+  }
+
   @VisibleForTesting
   static void registerFile(WatchService watchService, Map<WatchKey, Set<Path>> keyPathMap, String filePath)
       throws IOException, URISyntaxException {