Skip to content

Commit

Permalink
[WIP][ISSUE-4346] Support Spark 3.5
Browse files Browse the repository at this point in the history
  • Loading branch information
@QiangCai
QiangCai committed Apr 10, 2024
1 parent 71abab0 commit 739a5b1
Show file tree
Hide file tree
Showing 73 changed files with 196 additions and 3,448 deletions.
2 changes: 1 addition & 1 deletion core/src/main/java/org/apache/carbondata/core/reader/CarbonIndexFileReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ public void openThriftReader(String filePath) throws IOException {
*
* @param fileData
*/
public void openThriftReader(byte[] fileData) {
public void openThriftReader(byte[] fileData) throws IOException {
thriftReader = new ThriftReader(fileData);
}

Expand Down
14 changes: 11 additions & 3 deletions core/src/main/java/org/apache/carbondata/core/reader/ThriftReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,9 +87,13 @@ public ThriftReader(String fileName, Configuration configuration) {
/**
* Constructor.
*/
public ThriftReader(byte[] fileData) {
public ThriftReader(byte[] fileData) throws IOException {
dataInputStream = new DataInputStream(new ByteArrayInputStream(fileData));
binaryIn = new TCompactProtocol(new TIOStreamTransport(dataInputStream));
try {
binaryIn = new TCompactProtocol(new TIOStreamTransport(dataInputStream));
} catch (TException e) {
throw new IOException(e);
}
}

/**
Expand All @@ -98,7 +102,11 @@ public ThriftReader(byte[] fileData) {
public void open() throws IOException {
Configuration conf = configuration != null ? configuration : FileFactory.getConfiguration();
dataInputStream = FileFactory.getDataInputStream(fileName, conf);
binaryIn = new TCompactProtocol(new TIOStreamTransport(dataInputStream));
try {
binaryIn = new TCompactProtocol(new TIOStreamTransport(dataInputStream));
} catch (TException e) {
throw new IOException(e);
}
}

/**
Expand Down
6 changes: 3 additions & 3 deletions core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1413,8 +1413,8 @@ public static String convertToString(List<Segment> values) {
public static byte[] getByteArray(TBase t) {
ByteArrayOutputStream stream = new ByteArrayOutputStream();
byte[] thriftByteArray = null;
TProtocol binaryOut = new TCompactProtocol(new TIOStreamTransport(stream));
try {
TProtocol binaryOut = new TCompactProtocol(new TIOStreamTransport(stream));
t.write(binaryOut);
stream.flush();
thriftByteArray = stream.toByteArray();
Expand All @@ -1439,9 +1439,9 @@ public static DataChunk3 readDataChunk3(ByteBuffer dataChunkBuffer, int offset,

public static DataChunk3 readDataChunk3(InputStream stream) throws IOException {
TBaseCreator creator = DataChunk3::new;
TProtocol binaryIn = new TCompactProtocol(new TIOStreamTransport(stream));
TBase t = creator.create();
try {
TProtocol binaryIn = new TCompactProtocol(new TIOStreamTransport(stream));
t.read(binaryIn);
} catch (TException e) {
throw new IOException(e);
Expand All @@ -1461,9 +1461,9 @@ public static DataChunk3 readDataChunk3(InputStream stream) throws IOException {
private static TBase read(byte[] data, TBaseCreator creator, int offset, int length)
throws IOException {
ByteArrayInputStream stream = new ByteArrayInputStream(data, offset, length);
TProtocol binaryIn = new TCompactProtocol(new TIOStreamTransport(stream));
TBase t = creator.create();
try {
TProtocol binaryIn = new TCompactProtocol(new TIOStreamTransport(stream));
t.read(binaryIn);
} catch (TException e) {
throw new IOException(e);
Expand Down
12 changes: 10 additions & 2 deletions core/src/main/java/org/apache/carbondata/core/writer/ThriftWriter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,11 @@ public ThriftWriter(String fileName, boolean append) {
*/
public void open() throws IOException {
dataOutputStream = FileFactory.getDataOutputStream(fileName, bufferSize, append);
binaryOut = new TCompactProtocol(new TIOStreamTransport(dataOutputStream));
try {
binaryOut = new TCompactProtocol(new TIOStreamTransport(dataOutputStream));
} catch (TException e) {
throw new IOException(e);
}
}

/**
Expand All @@ -92,7 +96,11 @@ public void open() throws IOException {
public void open(FileWriteOperation fileWriteOperation) throws IOException {
atomicFileOperationsWriter = AtomicFileOperationFactory.getAtomicFileOperations(fileName);
dataOutputStream = atomicFileOperationsWriter.openForWrite(fileWriteOperation);
binaryOut = new TCompactProtocol(new TIOStreamTransport(dataOutputStream));
try {
binaryOut = new TCompactProtocol(new TIOStreamTransport(dataOutputStream));
} catch (TException e) {
throw new IOException(e);
}
}

/**
Expand Down
22 changes: 0 additions & 22 deletions examples/spark/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -198,27 +198,5 @@
<maven.test.skip>true</maven.test.skip>
</properties>
</profile>
<profile>
<id>spark-2.3</id>
<properties>
<spark.binary.version>2.3</spark.binary.version>
</properties>
</profile>
<profile>
<id>spark-2.4</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<spark.binary.version>2.4</spark.binary.version>
</properties>
</profile>
<profile>
<id>spark-3.1</id>
<properties>
<spark.binary.version>3.1</spark.binary.version>
<dep.jackson.version>2.10.0</dep.jackson.version>
</properties>
</profile>
</profiles>
</project>
2 changes: 1 addition & 1 deletion format/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
<dependency>
<groupId>org.apache.thrift</groupId>
<artifactId>libthrift</artifactId>
<version>0.9.3</version>
<version>0.19.0</version>
</dependency>
</dependencies>

Expand Down
22 changes: 0 additions & 22 deletions index/examples/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,26 +77,4 @@
</plugin>
</plugins>
</build>

<profiles>
<profile>
<id>spark-2.3</id>
<properties>
<spark.binary.version>2.3</spark.binary.version>
</properties>
</profile>
<profile>
<id>spark-2.4</id>
<properties>
<spark.binary.version>2.4</spark.binary.version>
</properties>
</profile>
<profile>
<id>spark-3.1</id>
<properties>
<spark.binary.version>3.1</spark.binary.version>
</properties>
</profile>
</profiles>

</project>
21 changes: 0 additions & 21 deletions index/secondary-index/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,27 +156,6 @@
<maven.test.skip>true</maven.test.skip>
</properties>
</profile>
<profile>
<id>spark-2.3</id>
<properties>
<spark.binary.version>2.3</spark.binary.version>
</properties>
</profile>
<profile>
<id>spark-2.4</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<spark.binary.version>2.4</spark.binary.version>
</properties>
</profile>
<profile>
<id>spark-3.1</id>
<properties>
<spark.binary.version>3.1</spark.binary.version>
</properties>
</profile>
</profiles>

</project>
77 changes: 13 additions & 64 deletions integration/flink/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,12 @@
<dependencies>
<dependency>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-flink-proxy</artifactId>
<artifactId>carbondata-format</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-format</artifactId>
<artifactId>carbondata-flink-proxy</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
Expand Down Expand Up @@ -215,72 +215,21 @@
<version>4.1.17.Final</version>
<scope>test</scope>
</dependency>
</dependencies>

<profiles>
<profile>
<id>spark-2.3</id>
<properties>
<spark.binary.version>2.3</spark.binary.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-spark_${spark.binary.version}</artifactId>
<version>${project.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.apache.hive</groupId>
<artifactId>hive-exec</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
</profile>
<profile>
<id>spark-2.4</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<spark.binary.version>2.4</spark.binary.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-spark_${spark.binary.version}</artifactId>
<version>${project.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.apache.hive</groupId>
<artifactId>hive-exec</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
</profile>
<profile>
<id>spark-3.1</id>
<properties>
<spark.binary.version>3.1</spark.binary.version>
</properties>
<dependencies>
<dependency>
<dependency>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-spark_${spark.binary.version}</artifactId>
<version>${project.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>com.thoughtworks.paranamer</groupId>
<artifactId>paranamer</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
</profile>
<exclusions>
<exclusion>
<groupId>com.thoughtworks.paranamer</groupId>
<artifactId>paranamer</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>

<profiles>
<profile>
<id>sdvtest</id>
<properties>
Expand Down
Loading

0 comments on commit 739a5b1

Please sign in to comment.