[improve] upgrade parquet version to 1.13.0

liunaijie · Oct 29, 2024 · 95c9d11 · 95c9d11
1 parent a8d0d4c
commit 95c9d11
Show file tree

Hide file tree

Showing 4 changed files with 10 additions and 3 deletions.
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/pom.xml b/seatunnel-connectors-v2/connector-file/connector-file-base/pom.xml
@@ -33,7 +33,7 @@
         <orc.version>1.5.6</orc.version>
         <commons.collecton4.version>4.4</commons.collecton4.version>
         <commons.lang3.version>3.4</commons.lang3.version>
-        <parquet-avro.version>1.12.3</parquet-avro.version>
+        <parquet-avro.version>1.13.0</parquet-avro.version>
         <poi.version>4.1.2</poi.version>
         <poi-ooxml.version>4.1.2</poi-ooxml.version>
         <hadoop-minikdc.version>3.1.4</hadoop-minikdc.version>

diff --git a/...e/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/BaseSinkConfig.java b/...e/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/BaseSinkConfig.java
@@ -80,7 +80,8 @@ public class BaseSinkConfig extends KerberosConfig {
                                     CompressFormat.LZ4,
                                     CompressFormat.GZIP,
                                     CompressFormat.BROTLI,
-                                    CompressFormat.ZSTD))
+                                    CompressFormat.ZSTD,
+                                    CompressFormat.LZ4_RAW))
                     .defaultValue(CompressFormat.NONE)
                     .withDescription("Parquet file supported compression");
 

diff --git a/...e/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/CompressFormat.java b/...e/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/CompressFormat.java
@@ -37,7 +37,8 @@ public enum CompressFormat implements Serializable {
     // only parquet support
     GZIP(".gz", CompressionKind.NONE, CompressionCodecName.GZIP),
     BROTLI(".br", CompressionKind.NONE, CompressionCodecName.BROTLI),
-    ZSTD(".zstd", CompressionKind.NONE, CompressionCodecName.ZSTD);
+    ZSTD(".zstd", CompressionKind.NONE, CompressionCodecName.ZSTD),
+    LZ4_RAW(".lz4raw", CompressionKind.NONE, CompressionCodecName.LZ4_RAW);
 
     private final String compressCodec;
     private final CompressionKind orcCompression;

diff --git a/...ava/org/apache/seatunnel/connectors/seatunnel/file/source/reader/ParquetReadStrategy.java b/...ava/org/apache/seatunnel/connectors/seatunnel/file/source/reader/ParquetReadStrategy.java
@@ -296,9 +296,14 @@ private SeaTunnelDataType<?> parquetType2SeaTunnelType(Type type, String name) {
                         case INT_8:
                             return BasicType.BYTE_TYPE;
                         case INT_16:
+                        case UINT_8:
                             return BasicType.SHORT_TYPE;
                         case DATE:
                             return LocalTimeType.LOCAL_DATE_TYPE;
+                        case UINT_16:
+                            return BasicType.INT_TYPE;
+                        case UINT_32:
+                            return BasicType.LONG_TYPE;
                         default:
                             throw CommonError.convertToSeaTunnelTypeError(
                                     PARQUET, type.toString(), name);