[Issue-170] Upgrade Spark Version to 3.5.0 (#174)

Signed-off-by: Ved, Kuldeep <[email protected]>
pravega · Dec 5, 2023 · 571146e · 571146e
1 parent fe5a25a
commit 571146e
Show file tree

Hide file tree

Showing 3 changed files with 4 additions and 1 deletion.
diff --git a/gradle.properties b/gradle.properties
@@ -23,7 +23,7 @@ scalaArmVersion=2.0
 scalaVersion=2.12.13
 shadowGradlePlugin=6.1.0
 slf4jApiVersion=1.7.25
-sparkVersion=3.4.0
+sparkVersion=3.5.0
 
 # Version and base tags can be overridden at build time.
 connectorVersion=0.14.0-SNAPSHOT

diff --git a/src/main/scala/io/pravega/connectors/spark/NonTransactionPravegaWriter.scala b/src/main/scala/io/pravega/connectors/spark/NonTransactionPravegaWriter.scala
@@ -47,6 +47,8 @@ class NonTransactionPravegaWriter(
                      schema: StructType)
   extends StreamingWrite with BatchWrite with Logging {
 
+  override def useCommitCoordinator() = true
+
   override def createBatchWriterFactory(info: PhysicalWriteInfo):  NonTransactionPravegaWriterFactory =
     NonTransactionPravegaWriterFactory(scopeName, streamName, clientConfig, schema)
 

diff --git a/src/main/scala/io/pravega/connectors/spark/TransactionPravegaWriter.scala b/src/main/scala/io/pravega/connectors/spark/TransactionPravegaWriter.scala
@@ -141,6 +141,7 @@ class TransactionPravegaWriter(
     log.debug(s"abort: END: epochId=$epochId, messages=${messages.mkString(",")}")
   }
 
+  override def useCommitCoordinator() = true
 
   override def createBatchWriterFactory(info: PhysicalWriteInfo): TransactionPravegaWriterFactory =
     TransactionPravegaWriterFactory(scopeName, streamName, clientConfig, transactionTimeoutMs, schema)