Bump delta to 2.4.0 (#15)

SharpData · May 26, 2023 · 3890d58 · 3890d58
1 parent ed4ddd6
commit 3890d58
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 4 deletions.
diff --git a/spark/build.gradle b/spark/build.gradle
@@ -78,7 +78,9 @@ dependencies {
  implementation "org.apache.spark:spark-streaming_$scalaVersion:$sparkVersion"
  implementation "org.apache.spark:spark-streaming-kafka-0-10_$scalaVersion:$sparkVersion"
  implementation "org.apache.kafka:kafka-clients:2.0.0"
- if (sparkVersion.startsWith("3.3") || sparkVersion.startsWith("3.4")) {
+ if (sparkVersion.startsWith("3.4")) {
+ implementation "io.delta:delta-core_$scalaVersion:2.4.0"
+ } else if (sparkVersion.startsWith("3.3")) {
  implementation "io.delta:delta-core_$scalaVersion:2.3.0"
  } else if (sparkVersion.startsWith("3.2")) {
  implementation "io.delta:delta-core_$scalaVersion:2.0.2"
@@ -362,7 +364,9 @@ shadowJar {
  include dependency('info.picocli:picocli:4.6.3')
  include dependency("net.liftweb:lift-json_$scalaVersion:3.4.3")
 
- if (sparkVersion.startsWith("3.3") || sparkVersion.startsWith("3.4")) {
+ if (sparkVersion.startsWith("3.4")) {
+ addDeps("io.delta:delta-core_$scalaVersion:2.4.0")
+ } else if (sparkVersion.startsWith("3.3")) {
  addDeps("io.delta:delta-core_$scalaVersion:2.3.0")
  } else if (sparkVersion.startsWith("3.2")) {
  addDeps("io.delta:delta-core_$scalaVersion:2.0.2")

diff --git a/spark/src/test/scala/com/github/sharpdata/sharpetl/spark/end2end/delta/DeltaLakeSpec.scala b/spark/src/test/scala/com/github/sharpdata/sharpetl/spark/end2end/delta/DeltaLakeSpec.scala
@@ -40,7 +40,7 @@ class DeltaLakeSpec extends DeltaSuit {
  it("delta should works") {
  if (spark.version.startsWith("2.3")) {
  ETLLogger.error("Delta Lake does NOT support Spark 2.3.x")
- } else if (spark.version.startsWith("2.4") || spark.version.startsWith("3.0") || spark.version.startsWith("3.1")|| spark.version.startsWith("3.4")) {
+ } else if (spark.version.startsWith("2.4") || spark.version.startsWith("3.0") || spark.version.startsWith("3.1")) {
  ETLLogger.error("Delta Lake does not works well on Spark 2.4.x, " +
  "CREATE TABLE USING delta is not supported by Spark before 3.0.0 and Delta Lake before 0.7.0.")
  } else {

diff --git a/spark/src/test/scala/com/github/sharpdata/sharpetl/spark/end2end/delta/FlyDeltaSpec.scala b/spark/src/test/scala/com/github/sharpdata/sharpetl/spark/end2end/delta/FlyDeltaSpec.scala
@@ -17,7 +17,7 @@ class FlyDeltaSpec extends AnyFunSpec
  it("should just run with delta") {
  if (spark.version.startsWith("2.3")) {
  ETLLogger.error("Delta Lake does NOT support Spark 2.3.x")
- } else if (spark.version.startsWith("2.4") || spark.version.startsWith("3.0") || spark.version.startsWith("3.1")|| spark.version.startsWith("3.4")) {
+ } else if (spark.version.startsWith("2.4") || spark.version.startsWith("3.0") || spark.version.startsWith("3.1")) {
  ETLLogger.error("Delta Lake does not works well on Spark 2.4.x, " +
  "CREATE TABLE USING delta is not supported by Spark before 3.0.0 and Delta Lake before 0.7.0.")
  } else {