Skip to content

Commit

Permalink
[Improve][Connector-V2] Enable the fs cache for paimon sink to avoid …
Browse files Browse the repository at this point in the history
…affecting other hadoop filesystem jobs

xx
  • Loading branch information
dailai committed Apr 2, 2024
1 parent 7038a7d commit 8cc8559
Showing 1 changed file with 5 additions and 0 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
@Slf4j
public class PaimonSecurityContext extends SecurityContext {
private static final String KRB5_CONF_KEY = "java.security.krb5.conf";
private static final String FS_DISABLE_CACHE = "fs.hdfs.impl.disable.cache";
private static final List<String> HADOOP_CONF_FILES =
ImmutableList.of("core-site.xml", "hdfs-site.xml", "hive-site.xml");

Expand Down Expand Up @@ -89,6 +90,10 @@ public static PaimonHadoopConfiguration loadHadoopConfig(PaimonConfig paimonConf
});
}
paimonConfig.getHadoopConfProps().forEach((k, v) -> configuration.set(k, v));
// This configuration is enabled to avoid affecting other hadoop filesystem jobs
// refer:
// org.apache.seatunnel.connectors.seatunnel.file.hadoop.HadoopFileSystemProxy.createConfiguration
configuration.setBoolean(FS_DISABLE_CACHE, true);
log.info("Hadoop config initialized: {}", configuration.getClass().getName());
return configuration;
}
Expand Down

0 comments on commit 8cc8559

Please sign in to comment.