From 4e2f4ef1169ad66eed3f4351288888366d5b01df Mon Sep 17 00:00:00 2001 From: Eric Date: Thu, 14 Sep 2023 19:41:41 +0800 Subject: [PATCH] Revert "[fix][hive-source][bug] fix An error occurred reading an empty directory (#5427)" This reverts commit de7b86a5dd24d9cff4fa28ca7b8eadbd50ed8bb9. --- .../source/reader/AbstractReadStrategy.java | 22 ++++++++++++------- .../file/source/reader/ExcelReadStrategy.java | 3 +-- .../file/source/reader/TextReadStrategy.java | 3 +-- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategy.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategy.java index 1dc8e7a2ad9..e4e1694f30d 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategy.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategy.java @@ -24,6 +24,8 @@ import org.apache.seatunnel.api.table.type.SeaTunnelRowType; import org.apache.seatunnel.connectors.seatunnel.file.config.BaseSourceConfig; import org.apache.seatunnel.connectors.seatunnel.file.config.HadoopConf; +import org.apache.seatunnel.connectors.seatunnel.file.exception.FileConnectorErrorCode; +import org.apache.seatunnel.connectors.seatunnel.file.exception.FileConnectorException; import org.apache.seatunnel.connectors.seatunnel.file.sink.util.FileSystemUtils; import org.apache.hadoop.conf.Configuration; @@ -151,9 +153,15 @@ public List getFileNamesByPath(HadoopConf hadoopConf, String path) throw } } } - if (this.fileNames.isEmpty()) { - log.error("The current directory is empty " + path); + + if (fileNames.isEmpty()) { + throw new FileConnectorException( + FileConnectorErrorCode.FILE_LIST_EMPTY, + "The target file list is empty," + + "SeaTunnel will not be able to sync empty table, " + + "please check the configuration parameters such as: [file_filter_pattern]"); } + return fileNames; } @@ -188,12 +196,10 @@ public SeaTunnelRowType getActualSeaTunnelRowTypeInfo() { protected Map parsePartitionsByPath(String path) { LinkedHashMap partitions = new LinkedHashMap<>(); - if (path != null && !path.isEmpty()) { - Arrays.stream(path.split("/", -1)) - .filter(split -> split.contains("=")) - .map(split -> split.split("=", -1)) - .forEach(kv -> partitions.put(kv[0], kv[1])); - } + Arrays.stream(path.split("/", -1)) + .filter(split -> split.contains("=")) + .map(split -> split.split("=", -1)) + .forEach(kv -> partitions.put(kv[0], kv[1])); return partitions; } diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/ExcelReadStrategy.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/ExcelReadStrategy.java index 649216c752c..0b1cfc083ba 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/ExcelReadStrategy.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/ExcelReadStrategy.java @@ -136,8 +136,7 @@ public void setSeaTunnelRowTypeInfo(SeaTunnelRowType seaTunnelRowType) { "Schmea information is not set or incorrect schmea settings"); } SeaTunnelRowType userDefinedRowTypeWithPartition = - mergePartitionTypes( - fileNames.size() > 0 ? fileNames.get(0) : null, seaTunnelRowType); + mergePartitionTypes(fileNames.get(0), seaTunnelRowType); // column projection if (pluginConfig.hasPath(BaseSourceConfig.READ_COLUMNS.key())) { // get the read column index from user-defined row type diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/TextReadStrategy.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/TextReadStrategy.java index eefa8e4de76..4b931cb8902 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/TextReadStrategy.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/TextReadStrategy.java @@ -138,8 +138,7 @@ public SeaTunnelRowType getSeaTunnelRowTypeInfo(HadoopConf hadoopConf, String pa @Override public void setSeaTunnelRowTypeInfo(SeaTunnelRowType seaTunnelRowType) { SeaTunnelRowType userDefinedRowTypeWithPartition = - mergePartitionTypes( - fileNames.size() > 0 ? fileNames.get(0) : null, seaTunnelRowType); + mergePartitionTypes(fileNames.get(0), seaTunnelRowType); if (pluginConfig.hasPath(BaseSourceConfig.DELIMITER.key())) { fieldDelimiter = pluginConfig.getString(BaseSourceConfig.DELIMITER.key()); } else {