From 4900ff35f5567d0ef1436af2edde2415a80022b2 Mon Sep 17 00:00:00 2001 From: "Ved, Kuldeep" Date: Thu, 30 Nov 2023 13:57:14 +0530 Subject: [PATCH 1/2] Signed-off-by: Ved, Kuldeep Kuldeep.Ved@dell.com Change log description Update Spark version to 3.5.0 Purpose of the change Fix #170 What the code does Update sparkVersion in gradle.properties How to verify it ./gradlew clean build should pass Signed-off-by: Ved, Kuldeep --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 1f8906d..9c10b8a 100644 --- a/gradle.properties +++ b/gradle.properties @@ -23,7 +23,7 @@ scalaArmVersion=2.0 scalaVersion=2.12.13 shadowGradlePlugin=6.1.0 slf4jApiVersion=1.7.25 -sparkVersion=3.4.0 +sparkVersion=3.5.0 # Version and base tags can be overridden at build time. connectorVersion=0.14.0-SNAPSHOT From 3a96f961fdb93004268963c2939408e7ed419346 Mon Sep 17 00:00:00 2001 From: "Ved, Kuldeep" Date: Mon, 4 Dec 2023 15:34:59 +0530 Subject: [PATCH 2/2] Signed-off-by: Ved, Kuldeep Kuldeep.Ved@dell.com Change log description Update Spark version to 3.5.0 Purpose of the change Fix pravega#170 What the code does Update sparkVersion in gradle.properties How to verify it ./gradlew clean build should pass Signed-off-by: Ved, Kuldeep --- .../pravega/connectors/spark/NonTransactionPravegaWriter.scala | 2 ++ .../io/pravega/connectors/spark/TransactionPravegaWriter.scala | 1 + 2 files changed, 3 insertions(+) diff --git a/src/main/scala/io/pravega/connectors/spark/NonTransactionPravegaWriter.scala b/src/main/scala/io/pravega/connectors/spark/NonTransactionPravegaWriter.scala index 8639357..09add0f 100755 --- a/src/main/scala/io/pravega/connectors/spark/NonTransactionPravegaWriter.scala +++ b/src/main/scala/io/pravega/connectors/spark/NonTransactionPravegaWriter.scala @@ -47,6 +47,8 @@ class NonTransactionPravegaWriter( schema: StructType) extends StreamingWrite with BatchWrite with Logging { + override def useCommitCoordinator() = true + override def createBatchWriterFactory(info: PhysicalWriteInfo): NonTransactionPravegaWriterFactory = NonTransactionPravegaWriterFactory(scopeName, streamName, clientConfig, schema) diff --git a/src/main/scala/io/pravega/connectors/spark/TransactionPravegaWriter.scala b/src/main/scala/io/pravega/connectors/spark/TransactionPravegaWriter.scala index a5b0b94..130b3b2 100755 --- a/src/main/scala/io/pravega/connectors/spark/TransactionPravegaWriter.scala +++ b/src/main/scala/io/pravega/connectors/spark/TransactionPravegaWriter.scala @@ -141,6 +141,7 @@ class TransactionPravegaWriter( log.debug(s"abort: END: epochId=$epochId, messages=${messages.mkString(",")}") } + override def useCommitCoordinator() = true override def createBatchWriterFactory(info: PhysicalWriteInfo): TransactionPravegaWriterFactory = TransactionPravegaWriterFactory(scopeName, streamName, clientConfig, transactionTimeoutMs, schema)