diff --git a/.github/codecov.yml b/.github/codecov.yml index 70114e3a8..b6cfe0b03 100644 --- a/.github/codecov.yml +++ b/.github/codecov.yml @@ -3,7 +3,7 @@ codecov: branch: main notify: wait_for_ci: true - after_n_builds: 4 + after_n_builds: 5 coverage: precision: 2 @@ -37,4 +37,4 @@ comment: layout: "reach,diff,flags,files,footer" behavior: once # once: update, if exists. Otherwise, post new. Skip if deleted. require_changes: false # if true: only post the comment if coverage changes - after_n_builds: 4 + after_n_builds: 5 diff --git a/.github/labeler.yml b/.github/labeler.yml index c5f81374d..6a3f837f2 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -26,3 +26,7 @@ spark_3.2: spark_3.3: - core/src/main/spark*3.3*/**/* - core/src/main/scala/**/* + +spark_3.4: + - core/src/main/spark*3.4*/**/* + - core/src/main/scala/**/* diff --git a/.github/workflows/matrix_includes.json b/.github/workflows/matrix_includes.json index 5acb7e4da..4c6f406d2 100644 --- a/.github/workflows/matrix_includes.json +++ b/.github/workflows/matrix_includes.json @@ -113,5 +113,10 @@ "spark": "3.3", "scope": "test-uploadReport", "isRelease": "release" + }, + { + "spark": "3.4", + "scope": "test-uploadReport", + "isRelease": "release" } ] diff --git a/README.md b/README.md index f542c5166..b6256ecf0 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ Type-safe columns for spark DataFrames! | 3.1.x | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-1_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-1_2.12/0.0.6) | [![Codecov](https://img.shields.io/codecov/c/github/hablapps/doric?flag=spark-3.1.x&label=codecov&logo=codecov&token=N7ZXUXZX1I)](https://codecov.io/gh/hablapps/doric) | | 3.2.x | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-2_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-2_2.12/0.0.6) | [![Codecov](https://img.shields.io/codecov/c/github/hablapps/doric?flag=spark-3.2.x&label=codecov&logo=codecov&token=N7ZXUXZX1I)](https://codecov.io/gh/hablapps/doric) | | 3.3.x | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-3_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-3_2.12/0.0.6) | [![Codecov](https://img.shields.io/codecov/c/github/hablapps/doric?flag=spark-3.3.x&label=codecov&logo=codecov&token=N7ZXUXZX1I)](https://codecov.io/gh/hablapps/doric) | +| 3.4.x | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-4_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-4_2.12/0.0.6) | [![Codecov](https://img.shields.io/codecov/c/github/hablapps/doric?flag=spark-3.4.x&label=codecov&logo=codecov&token=N7ZXUXZX1I)](https://codecov.io/gh/hablapps/doric) | ---- Doric offers type-safety in DataFrame column expressions at a minimum @@ -85,7 +86,12 @@ following spark versions. | 3.2.0 | 2.12 | ✅ | - | | 3.2.1 | 2.12 | ✅ | - | | 3.2.2 | 2.12 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-2_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-2_2.12/0.0.6) | -| 3.3.0 | 2.12 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-3_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-3_2.12/0.0.6) | +| 3.3.0 | 2.12 | ✅ | - | +| 3.3.1 | 2.12 | ✅ | - | +| 3.3.2 | 2.12 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-3_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-3_2.12/0.0.6) | +| 3.4.0 | 2.12 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-4_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-4_2.12/0.0.6) | + + ## Contributing diff --git a/build.sbt b/build.sbt index 06c59c26b..2f9e5508d 100644 --- a/build.sbt +++ b/build.sbt @@ -4,12 +4,13 @@ import sbt.Compile val stableVersion = "0.0.6" -val sparkDefaultShortVersion = "3.3" +val sparkDefaultShortVersion = "3.4" val spark24Version = "2.4.8" val spark30Version = "3.0.3" val spark31Version = "3.1.3" val spark32Version = "3.2.3" val spark33Version = "3.3.2" +val spark34Version = "3.4.0" val versionRegex = """^(.*)\.(.*)\.(.*)$""".r val versionRegexShort = """^(.*)\.(.*)$""".r @@ -24,6 +25,7 @@ val parserSparkVersion: String => String = { case versionRegexShort("3", "1") => spark31Version case versionRegexShort("3", "2") => spark32Version case versionRegexShort("3", "3") => spark33Version + case versionRegexShort("3", "4") => spark34Version case versionRegex(a, b, c) => s"$a.$b.$c" } @@ -37,7 +39,7 @@ val scalaVersionSelect: String => List[String] = { case versionRegex("3", "1", _) => List(scala212) case versionRegex("3", "2", _) => List(scala212, scala213) case versionRegex("3", "3", _) => List(scala212, scala213) - + case versionRegex("3", "4", _) => List(scala212, scala213) } val catsVersion: String => String = { @@ -215,7 +217,7 @@ lazy val docs = project } } ) - .enablePlugins(plugins: _*) + .enablePlugins(plugins *) // Scoverage settings Global / coverageEnabled := false diff --git a/core/src/main/scala/doric/syntax/ArrayColumns.scala b/core/src/main/scala/doric/syntax/ArrayColumns.scala index 98f20d704..124550f49 100644 --- a/core/src/main/scala/doric/syntax/ArrayColumns.scala +++ b/core/src/main/scala/doric/syntax/ArrayColumns.scala @@ -1,16 +1,17 @@ package doric package syntax +import scala.jdk.CollectionConverters._ +import scala.language.higherKinds +import scala.reflect.ClassTag + import cats.data.Kleisli import cats.implicits._ import doric.types.{CollectionType, LiteralSparkType, SparkType} -import org.apache.spark.sql.catalyst.expressions.LambdaFunction.identity -import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.{Column, Dataset, Row, functions => f} -import scala.jdk.CollectionConverters._ -import scala.language.higherKinds -import scala.reflect.ClassTag +import org.apache.spark.sql.{Column, Dataset, Row, functions => f} +import org.apache.spark.sql.catalyst.expressions._ +import org.apache.spark.sql.catalyst.expressions.LambdaFunction.identity protected final case class Zipper[T1, T2, F[_]: CollectionType]( col: DoricColumn[F[T1]], @@ -392,7 +393,7 @@ protected trait ArrayColumns { * Null elements will be placed at the end of the returned array. * * @group Array Type - * @see [[org.apache.spark.sql.functions.array_sort]] + * @see [[https://spark.apache.org/docs/latest/api/scala/org/apache/spark/sql/functions$.html#array_sort(e:org.apache.spark.sql.Column):org.apache.spark.sql.Column]] */ def sortAscNullsLast: DoricColumn[F[T]] = col.elem.map(f.array_sort).toDC diff --git a/core/src/main/spark_3.0_3.1_3.2_3.3/scala/doric/syntax/ArrayColumns3x.scala b/core/src/main/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/ArrayColumns3x.scala similarity index 100% rename from core/src/main/spark_3.0_3.1_3.2_3.3/scala/doric/syntax/ArrayColumns3x.scala rename to core/src/main/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/ArrayColumns3x.scala diff --git a/core/src/main/spark_3.0_3.1_3.2_3.3/scala/doric/syntax/CommonColumns3x.scala b/core/src/main/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/CommonColumns3x.scala similarity index 100% rename from core/src/main/spark_3.0_3.1_3.2_3.3/scala/doric/syntax/CommonColumns3x.scala rename to core/src/main/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/CommonColumns3x.scala diff --git a/core/src/main/spark_3.0_3.1_3.2_3.3/scala/doric/syntax/DStructs3x.scala b/core/src/main/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/DStructs3x.scala similarity index 100% rename from core/src/main/spark_3.0_3.1_3.2_3.3/scala/doric/syntax/DStructs3x.scala rename to core/src/main/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/DStructs3x.scala diff --git a/core/src/main/spark_3.0_3.1_3.2_3.3/scala/doric/syntax/MapColumns3x.scala b/core/src/main/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/MapColumns3x.scala similarity index 100% rename from core/src/main/spark_3.0_3.1_3.2_3.3/scala/doric/syntax/MapColumns3x.scala rename to core/src/main/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/MapColumns3x.scala diff --git a/core/src/main/spark_3.0_3.1_3.2_3.3/scala/doric/syntax/StringColumn3x.scala b/core/src/main/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/StringColumn3x.scala similarity index 100% rename from core/src/main/spark_3.0_3.1_3.2_3.3/scala/doric/syntax/StringColumn3x.scala rename to core/src/main/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/StringColumn3x.scala diff --git a/core/src/main/spark_3.0_3.1_3.2_3.3/scala/org/apache/spark/sql/doric/RelationalGroupedDatasetDoricInterface.scala b/core/src/main/spark_3.0_3.1_3.2_3.3_3.4/scala/org/apache/spark/sql/doric/RelationalGroupedDatasetDoricInterface.scala similarity index 100% rename from core/src/main/spark_3.0_3.1_3.2_3.3/scala/org/apache/spark/sql/doric/RelationalGroupedDatasetDoricInterface.scala rename to core/src/main/spark_3.0_3.1_3.2_3.3_3.4/scala/org/apache/spark/sql/doric/RelationalGroupedDatasetDoricInterface.scala diff --git a/core/src/main/spark_3.1_3.2_3.3/scala/doric/syntax/AggregationColumns31.scala b/core/src/main/spark_3.1_3.2_3.3_3.4/scala/doric/syntax/AggregationColumns31.scala similarity index 100% rename from core/src/main/spark_3.1_3.2_3.3/scala/doric/syntax/AggregationColumns31.scala rename to core/src/main/spark_3.1_3.2_3.3_3.4/scala/doric/syntax/AggregationColumns31.scala diff --git a/core/src/main/spark_3.1_3.2_3.3/scala/doric/syntax/BooleanColumns31.scala b/core/src/main/spark_3.1_3.2_3.3_3.4/scala/doric/syntax/BooleanColumns31.scala similarity index 100% rename from core/src/main/spark_3.1_3.2_3.3/scala/doric/syntax/BooleanColumns31.scala rename to core/src/main/spark_3.1_3.2_3.3_3.4/scala/doric/syntax/BooleanColumns31.scala diff --git a/core/src/main/spark_3.1_3.2_3.3/scala/doric/syntax/NumericColumns31.scala b/core/src/main/spark_3.1_3.2_3.3_3.4/scala/doric/syntax/NumericColumns31.scala similarity index 99% rename from core/src/main/spark_3.1_3.2_3.3/scala/doric/syntax/NumericColumns31.scala rename to core/src/main/spark_3.1_3.2_3.3_3.4/scala/doric/syntax/NumericColumns31.scala index 1f458e31c..7769bd1ab 100644 --- a/core/src/main/spark_3.1_3.2_3.3/scala/doric/syntax/NumericColumns31.scala +++ b/core/src/main/spark_3.1_3.2_3.3_3.4/scala/doric/syntax/NumericColumns31.scala @@ -2,6 +2,7 @@ package doric package syntax import doric.types.NumericType + import org.apache.spark.sql.{functions => f} protected trait NumericColumns31 { diff --git a/core/src/main/spark_3.1_3.2_3.3/scala/doric/syntax/StringColumns31.scala b/core/src/main/spark_3.1_3.2_3.3_3.4/scala/doric/syntax/StringColumns31.scala similarity index 99% rename from core/src/main/spark_3.1_3.2_3.3/scala/doric/syntax/StringColumns31.scala rename to core/src/main/spark_3.1_3.2_3.3_3.4/scala/doric/syntax/StringColumns31.scala index 4968680b1..831224fe6 100644 --- a/core/src/main/spark_3.1_3.2_3.3/scala/doric/syntax/StringColumns31.scala +++ b/core/src/main/spark_3.1_3.2_3.3_3.4/scala/doric/syntax/StringColumns31.scala @@ -2,6 +2,7 @@ package doric package syntax import doric.sem.Location + import org.apache.spark.sql.{functions => f} protected trait StringColumns31 { diff --git a/core/src/main/spark_3.2_3.3/scala/doric/sqlExpressions/CustomAgg.scala b/core/src/main/spark_3.2_3.3_3.4/scala/doric/sqlExpressions/CustomAgg.scala similarity index 100% rename from core/src/main/spark_3.2_3.3/scala/doric/sqlExpressions/CustomAgg.scala rename to core/src/main/spark_3.2_3.3_3.4/scala/doric/sqlExpressions/CustomAgg.scala diff --git a/core/src/main/spark_3.2_3.3/scala/doric/syntax/AggregationColumns32.scala b/core/src/main/spark_3.2_3.3_3.4/scala/doric/syntax/AggregationColumns32.scala similarity index 100% rename from core/src/main/spark_3.2_3.3/scala/doric/syntax/AggregationColumns32.scala rename to core/src/main/spark_3.2_3.3_3.4/scala/doric/syntax/AggregationColumns32.scala diff --git a/core/src/main/spark_3.2_3.3/scala/doric/syntax/BinaryColumns32.scala b/core/src/main/spark_3.2_3.3_3.4/scala/doric/syntax/BinaryColumns32.scala similarity index 100% rename from core/src/main/spark_3.2_3.3/scala/doric/syntax/BinaryColumns32.scala rename to core/src/main/spark_3.2_3.3_3.4/scala/doric/syntax/BinaryColumns32.scala diff --git a/core/src/main/spark_3.2_3.3/scala/doric/syntax/NumericColumns32.scala b/core/src/main/spark_3.2_3.3_3.4/scala/doric/syntax/NumericColumns32.scala similarity index 95% rename from core/src/main/spark_3.2_3.3/scala/doric/syntax/NumericColumns32.scala rename to core/src/main/spark_3.2_3.3_3.4/scala/doric/syntax/NumericColumns32.scala index 2b25dbe23..a0b58a0ca 100644 --- a/core/src/main/spark_3.2_3.3/scala/doric/syntax/NumericColumns32.scala +++ b/core/src/main/spark_3.2_3.3_3.4/scala/doric/syntax/NumericColumns32.scala @@ -2,8 +2,8 @@ package doric package syntax import cats.implicits._ -import org.apache.spark.sql.Column -import org.apache.spark.sql.{functions => f} + +import org.apache.spark.sql.{Column, functions => f} import org.apache.spark.sql.catalyst.expressions.{ShiftLeft, ShiftRight, ShiftRightUnsigned} protected trait NumericColumns32 { diff --git a/core/src/main/spark_3.2_3.3/scala/doric/types/SparkTypeLPI_I_Specific.scala b/core/src/main/spark_3.2_3.3_3.4/scala/doric/types/SparkTypeLPI_I_Specific.scala similarity index 100% rename from core/src/main/spark_3.2_3.3/scala/doric/types/SparkTypeLPI_I_Specific.scala rename to core/src/main/spark_3.2_3.3_3.4/scala/doric/types/SparkTypeLPI_I_Specific.scala diff --git a/core/src/main/spark_3.4_mount/scala/doric/syntax/All.scala b/core/src/main/spark_3.4_mount/scala/doric/syntax/All.scala new file mode 100644 index 000000000..c006bcbbe --- /dev/null +++ b/core/src/main/spark_3.4_mount/scala/doric/syntax/All.scala @@ -0,0 +1,31 @@ +package doric.syntax + +private[doric] trait All + extends ArrayColumns + with TypeMatcher + with CommonColumns + with DStructs + with LiteralConversions + with MapColumns + with NumericColumns + with DateColumns + with TimestampColumns + with BooleanColumns + with StringColumns + with ControlStructures + with AggregationColumns + with CNameOps + with BinaryColumns + with Interpolators + with AggregationColumns31 + with BooleanColumns31 + with NumericColumns31 + with NumericColumns32 + with StringColumns31 + with BinaryColumns32 + with ArrayColumns3x + with CommonColumns3x + with MapColumns3x + with StringColumn3x + with AggregationColumns32 + with DStructs3x diff --git a/core/src/test/scala/doric/sem/ChildColumnNotFound.scala b/core/src/test/scala/doric/sem/ChildColumnNotFound.scala index b4ed95b1d..f4a006928 100644 --- a/core/src/test/scala/doric/sem/ChildColumnNotFound.scala +++ b/core/src/test/scala/doric/sem/ChildColumnNotFound.scala @@ -1,10 +1,19 @@ package doric.sem +import org.apache.spark.sql.SparkSession + object ChildColumnNotFound { - def apply(expectedCol: String, foundCols: List[String]): SparkErrorWrapper = { + def apply(expectedCol: String, foundCols: List[String])(implicit + location: Location, + sparkSession: SparkSession + ): SparkErrorWrapper = { SparkErrorWrapper( new Throwable( - s"No such struct field $expectedCol in ${foundCols.mkString(", ")}" + if (!sparkSession.version.startsWith("3.4")) + s"No such struct field $expectedCol in ${foundCols.mkString(", ")}" + else + s"[FIELD_NOT_FOUND] No such struct field `$expectedCol` in ${foundCols + .mkString("`", "`, `", "`")}." ) ) } diff --git a/core/src/test/scala/doric/sem/ColumnNotFound.scala b/core/src/test/scala/doric/sem/ColumnNotFound.scala index ece1e22d7..afa76fe1d 100644 --- a/core/src/test/scala/doric/sem/ColumnNotFound.scala +++ b/core/src/test/scala/doric/sem/ColumnNotFound.scala @@ -10,8 +10,12 @@ object ColumnNotFound { SparkErrorWrapper( new Throwable( - s"""Cannot resolve column name "$expectedCol" among (${foundCols - .mkString(", ")})""" + if (!sparkSession.version.startsWith("3.4")) + s"""Cannot resolve column name "$expectedCol" among (${foundCols + .mkString(", ")})""" + else + s"[UNRESOLVED_COLUMN.WITH_SUGGESTION] A column or function parameter with name `$expectedCol` cannot be resolved. Did you mean one of the following? [${foundCols + .mkString("`", "`, `", "`")}]." ) ) } diff --git a/core/src/test/scala/doric/sem/ErrorsSpec.scala b/core/src/test/scala/doric/sem/ErrorsSpec.scala index 4a9d44b0c..884b7ce1e 100644 --- a/core/src/test/scala/doric/sem/ErrorsSpec.scala +++ b/core/src/test/scala/doric/sem/ErrorsSpec.scala @@ -65,9 +65,7 @@ class ErrorsSpec val err = intercept[DoricMultiError] { Seq(1, 2, 3).toDF("value").select(colInt("notFound")) } - val err2 = SparkErrorWrapper( - new Exception("Cannot resolve column name \"notFound\" among (value)") - ) + val err2 = ColumnNotFound("notFound", List("value")) err.errors.head.equals(err2) shouldBe true } diff --git a/core/src/test/scala/doric/sem/JoinOpsSpec.scala b/core/src/test/scala/doric/sem/JoinOpsSpec.scala index 105b750cb..95e419898 100644 --- a/core/src/test/scala/doric/sem/JoinOpsSpec.scala +++ b/core/src/test/scala/doric/sem/JoinOpsSpec.scala @@ -2,6 +2,7 @@ package doric package sem import doric.implicitConversions._ + import org.apache.spark.sql.types.{LongType, StringType} class JoinOpsSpec extends DoricTestElements { @@ -58,7 +59,7 @@ class JoinOpsSpec extends DoricTestElements { val badJoinFunction: DoricJoinColumn = LeftDF.colString(id) === - RightDF.colString(id + "entifier") + RightDF.colString("identifier") intercept[DoricMultiError] { left.join(right, "inner", badJoinFunction) @@ -68,11 +69,7 @@ class JoinOpsSpec extends DoricTestElements { isLeft = true ), JoinDoricSingleError( - SparkErrorWrapper( - new Exception( - "Cannot resolve column name \"" + id + "entifier\" among (" + id + ", " + otherColumn + ")" - ) - ), + ColumnNotFound("identifier", List("id", "otherColumn")), isLeft = false ) ) @@ -94,11 +91,7 @@ class JoinOpsSpec extends DoricTestElements { isLeft = true ), JoinDoricSingleError( - SparkErrorWrapper( - new Exception( - "Cannot resolve column name \"" + id + "entifier\" among (" + id + ", " + otherColumn + ")" - ) - ), + ColumnNotFound("identifier", List("id", "otherColumn")), isLeft = false ) ) diff --git a/core/src/test/scala/doric/sem/TransformOpsSpec.scala b/core/src/test/scala/doric/sem/TransformOpsSpec.scala index 9e99dd80f..4dafdedd1 100644 --- a/core/src/test/scala/doric/sem/TransformOpsSpec.scala +++ b/core/src/test/scala/doric/sem/TransformOpsSpec.scala @@ -120,10 +120,15 @@ class TransformOpsSpec ) } error.getMessage should startWith( - "Found duplicate column(s) in given column names:" + if (!spark.version.startsWith("3.4")) + "Found duplicate column(s) in given column names:" + else + "[COLUMN_ALREADY_EXISTS] The column `a` already exists. Consider to choose another name or rename the existing column." ) error.getMessage should include("`a`") - error.getMessage should include("`b`") + if (!spark.version.startsWith("3.4")) { + error.getMessage should include("`b`") + } } it("should work with 'withNamedColumns' as with 'namedColumns'") { diff --git a/core/src/test/scala/doric/syntax/ArrayColumnsSpec.scala b/core/src/test/scala/doric/syntax/ArrayColumnsSpec.scala index 1ba73f16d..be073b7ab 100644 --- a/core/src/test/scala/doric/syntax/ArrayColumnsSpec.scala +++ b/core/src/test/scala/doric/syntax/ArrayColumnsSpec.scala @@ -1,15 +1,16 @@ package doric package syntax +import scala.jdk.CollectionConverters._ + import doric.SparkAuxFunctions.createLambda -import doric.sem.{ChildColumnNotFound, ColumnTypeError, DoricMultiError, SparkErrorWrapper} +import doric.sem.{ChildColumnNotFound, ColumnNotFound, ColumnTypeError, DoricMultiError} import doric.types.SparkType -import org.apache.spark.sql.catalyst.expressions.{ArrayExists, ZipWith} -import org.apache.spark.sql.types.{IntegerType, LongType, StringType, StructField, StructType} -import org.apache.spark.sql.{Column, Row, functions => f} - import java.sql.Timestamp -import scala.jdk.CollectionConverters._ + +import org.apache.spark.sql.{Column, Row, functions => f} +import org.apache.spark.sql.catalyst.expressions.{ArrayExists, ZipWith} +import org.apache.spark.sql.types._ class ArrayColumnsSpec extends DoricTestElements { @@ -52,11 +53,7 @@ class ArrayColumnsSpec extends DoricTestElements { .transform(_ => colString("something")) ) } should containAllErrors( - SparkErrorWrapper( - new Exception( - "Cannot resolve column name \"something2\" among (col, something)" - ) - ), + ColumnNotFound("something2", List("col", "something")), ColumnTypeError("something", StringType, IntegerType) ) @@ -72,11 +69,7 @@ class ArrayColumnsSpec extends DoricTestElements { ) } errors should containAllErrors( - SparkErrorWrapper( - new Exception( - "Cannot resolve column name \"something2\" among (col, something)" - ) - ), + ColumnNotFound("something2", List("col", "something")), ColumnTypeError("_1", LongType, IntegerType), ChildColumnNotFound("_3", List("_1", "_2")) ) @@ -179,11 +172,7 @@ class ArrayColumnsSpec extends DoricTestElements { .transformWithIndex(_ + _ + colInt("something2")) ) } should containAllErrors( - SparkErrorWrapper( - new Exception( - "Cannot resolve column name \"something2\" among (col, something)" - ) - ), + ColumnNotFound("something2", List("col", "something")), ColumnTypeError("something", IntegerType, StringType) ) } @@ -212,11 +201,7 @@ class ArrayColumnsSpec extends DoricTestElements { .aggregate(colInt("something2"))(_ + _ + colInt("something")) ) } should containAllErrors( - SparkErrorWrapper( - new Exception( - "Cannot resolve column name \"something2\" among (col, something)" - ) - ), + ColumnNotFound("something2", List("col", "something")), ColumnTypeError("something", IntegerType, StringType) ) } @@ -249,16 +234,8 @@ class ArrayColumnsSpec extends DoricTestElements { ) ) } should containAllErrors( - SparkErrorWrapper( - new Exception( - "Cannot resolve column name \"something2\" among (col, something)" - ) - ), - SparkErrorWrapper( - new Exception( - "Cannot resolve column name \"something3\" among (col, something)" - ) - ), + ColumnNotFound("something2", List("col", "something")), + ColumnNotFound("something3", List("col", "something")), ColumnTypeError("something", IntegerType, StringType) ) } diff --git a/core/src/test/scala/doric/syntax/AsSpec.scala b/core/src/test/scala/doric/syntax/AsSpec.scala index 186b9d85c..d085ef0b8 100644 --- a/core/src/test/scala/doric/syntax/AsSpec.scala +++ b/core/src/test/scala/doric/syntax/AsSpec.scala @@ -1,7 +1,7 @@ package doric package syntax -import doric.sem.{ColumnTypeError, DoricMultiError, SparkErrorWrapper} +import doric.sem.{ColumnNotFound, ColumnTypeError, DoricMultiError} import org.apache.spark.sql.functions.{col => sparkCol} import org.apache.spark.sql.types.{IntegerType, StringType} @@ -19,11 +19,7 @@ class AsSpec extends DoricTestElements { intercept[DoricMultiError] { df.select(originalColumn) } should containAllErrors( - SparkErrorWrapper( - new Exception( - """Cannot resolve column name "error" among (int, str)""" - ) - ) + ColumnNotFound("error", List("int", "str")) ) } diff --git a/core/src/test/scala/doric/syntax/StringColumnsSpec.scala b/core/src/test/scala/doric/syntax/StringColumnsSpec.scala index f6e91d48d..24ddd8022 100644 --- a/core/src/test/scala/doric/syntax/StringColumnsSpec.scala +++ b/core/src/test/scala/doric/syntax/StringColumnsSpec.scala @@ -480,11 +480,18 @@ class StringColumnsSpec extends DoricTestElements { df.testColumns2("col1", "")( (str, pattern) => colString(str).split(pattern.lit), (str, pattern) => f.split(f.col(str), pattern), - List( - Array("h", "e", "l", "l", "o", " ", "w", "o", "r", "l", "d", ""), - Array("1", "2", "3", "4", "5", ""), - null - ).map(Option(_)) + if (!spark.version.startsWith("3.4")) + List( + Array("h", "e", "l", "l", "o", " ", "w", "o", "r", "l", "d", ""), + Array("1", "2", "3", "4", "5", ""), + null + ).map(Option(_)) + else + List( + Array("h", "e", "l", "l", "o", " ", "w", "o", "r", "l", "d"), + Array("1", "2", "3", "4", "5"), + null + ).map(Option(_)) ) } } diff --git a/core/src/test/scala/doric/syntax/TypeMatcherSpec.scala b/core/src/test/scala/doric/syntax/TypeMatcherSpec.scala index 89959dfad..0b6bd541d 100644 --- a/core/src/test/scala/doric/syntax/TypeMatcherSpec.scala +++ b/core/src/test/scala/doric/syntax/TypeMatcherSpec.scala @@ -1,7 +1,8 @@ package doric package syntax -import doric.sem.{ColumnMultiTypeError, DoricMultiError, SparkErrorWrapper} +import doric.sem.{ColumnMultiTypeError, ColumnNotFound, DoricMultiError} + import org.apache.spark.sql.types.{IntegerType, StringType} class TypeMatcherSpec @@ -62,11 +63,7 @@ class TypeMatcherSpec intercept[DoricMultiError] { df.select(testColumn) } should containAllErrors( - SparkErrorWrapper( - new Exception( - "Cannot resolve column name \"int2\" among (colArr, int, str)" - ) - ) + ColumnNotFound("int2", List("colArr", "int", "str")) ) } @@ -81,11 +78,7 @@ class TypeMatcherSpec intercept[DoricMultiError] { df.select(testColumn) } should containAllErrors( - SparkErrorWrapper( - new Exception( - "Cannot resolve column name \"int3\" among (colArr, int, str)" - ) - ) + ColumnNotFound("int3", List("colArr", "int", "str")) ) } diff --git a/core/src/test/spark_3.0_3.1_3.2_3.3/scala/doric/DoricTestElementsCompanion_Specific.scala b/core/src/test/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/DoricTestElementsCompanion_Specific.scala similarity index 100% rename from core/src/test/spark_3.0_3.1_3.2_3.3/scala/doric/DoricTestElementsCompanion_Specific.scala rename to core/src/test/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/DoricTestElementsCompanion_Specific.scala diff --git a/core/src/test/spark_3.0_3.1_3.2_3.3/scala/doric/syntax/ArrayColumns3xSpec.scala b/core/src/test/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/ArrayColumns3xSpec.scala similarity index 100% rename from core/src/test/spark_3.0_3.1_3.2_3.3/scala/doric/syntax/ArrayColumns3xSpec.scala rename to core/src/test/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/ArrayColumns3xSpec.scala diff --git a/core/src/test/spark_3.0_3.1_3.2_3.3/scala/doric/syntax/CommonColumns3xSpec.scala b/core/src/test/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/CommonColumns3xSpec.scala similarity index 100% rename from core/src/test/spark_3.0_3.1_3.2_3.3/scala/doric/syntax/CommonColumns3xSpec.scala rename to core/src/test/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/CommonColumns3xSpec.scala diff --git a/core/src/test/spark_3.0_3.1_3.2_3.3/scala/doric/syntax/DStructs3xSpec.scala b/core/src/test/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/DStructs3xSpec.scala similarity index 100% rename from core/src/test/spark_3.0_3.1_3.2_3.3/scala/doric/syntax/DStructs3xSpec.scala rename to core/src/test/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/DStructs3xSpec.scala diff --git a/core/src/test/spark_3.0_3.1_3.2_3.3/scala/doric/syntax/DateColumns3xSpec.scala b/core/src/test/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/DateColumns3xSpec.scala similarity index 99% rename from core/src/test/spark_3.0_3.1_3.2_3.3/scala/doric/syntax/DateColumns3xSpec.scala rename to core/src/test/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/DateColumns3xSpec.scala index 35d110d4c..06179491f 100644 --- a/core/src/test/spark_3.0_3.1_3.2_3.3/scala/doric/syntax/DateColumns3xSpec.scala +++ b/core/src/test/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/DateColumns3xSpec.scala @@ -5,6 +5,7 @@ import java.sql.Date import java.time.{Instant, LocalDate} import org.scalatest.EitherValues import org.scalatest.matchers.should.Matchers + import org.apache.spark.sql.{DataFrame, functions => f} class DateColumns3xSpec diff --git a/core/src/test/spark_3.0_3.1_3.2_3.3/scala/doric/syntax/MapColumns3xSpec.scala b/core/src/test/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/MapColumns3xSpec.scala similarity index 100% rename from core/src/test/spark_3.0_3.1_3.2_3.3/scala/doric/syntax/MapColumns3xSpec.scala rename to core/src/test/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/MapColumns3xSpec.scala diff --git a/core/src/test/spark_3.0_3.1_3.2_3.3/scala/doric/syntax/StringColumns3xSpec.scala b/core/src/test/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/StringColumns3xSpec.scala similarity index 100% rename from core/src/test/spark_3.0_3.1_3.2_3.3/scala/doric/syntax/StringColumns3xSpec.scala rename to core/src/test/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/StringColumns3xSpec.scala diff --git a/core/src/test/spark_3.0_3.1_3.2_3.3/scala/doric/syntax/TimestampColumns3xSpec.scala b/core/src/test/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/TimestampColumns3xSpec.scala similarity index 100% rename from core/src/test/spark_3.0_3.1_3.2_3.3/scala/doric/syntax/TimestampColumns3xSpec.scala rename to core/src/test/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/TimestampColumns3xSpec.scala diff --git a/core/src/test/spark_3.1_3.2_3.3/scala/doric/syntax/AggregationColumns31Spec.scala b/core/src/test/spark_3.1_3.2_3.3_3.4/scala/doric/syntax/AggregationColumns31Spec.scala similarity index 100% rename from core/src/test/spark_3.1_3.2_3.3/scala/doric/syntax/AggregationColumns31Spec.scala rename to core/src/test/spark_3.1_3.2_3.3_3.4/scala/doric/syntax/AggregationColumns31Spec.scala diff --git a/core/src/test/spark_3.1_3.2_3.3/scala/doric/syntax/BooleanColumns31Spec.scala b/core/src/test/spark_3.1_3.2_3.3_3.4/scala/doric/syntax/BooleanColumns31Spec.scala similarity index 100% rename from core/src/test/spark_3.1_3.2_3.3/scala/doric/syntax/BooleanColumns31Spec.scala rename to core/src/test/spark_3.1_3.2_3.3_3.4/scala/doric/syntax/BooleanColumns31Spec.scala diff --git a/core/src/test/spark_3.1_3.2_3.3/scala/doric/syntax/Numeric31Spec.scala b/core/src/test/spark_3.1_3.2_3.3_3.4/scala/doric/syntax/Numeric31Spec.scala similarity index 99% rename from core/src/test/spark_3.1_3.2_3.3/scala/doric/syntax/Numeric31Spec.scala rename to core/src/test/spark_3.1_3.2_3.3_3.4/scala/doric/syntax/Numeric31Spec.scala index d281c3925..ff7e8c0e0 100644 --- a/core/src/test/spark_3.1_3.2_3.3/scala/doric/syntax/Numeric31Spec.scala +++ b/core/src/test/spark_3.1_3.2_3.3_3.4/scala/doric/syntax/Numeric31Spec.scala @@ -1,14 +1,15 @@ package doric package syntax -import Equalities._ +import scala.reflect.ClassTag + +import doric.Equalities._ import doric.types.NumericType import doric.types.SparkType.Primitive -import org.apache.spark.sql.{DataFrame, SparkSession, functions => f} +import java.sql.Timestamp import org.scalatest.funspec.AnyFunSpecLike -import java.sql.Timestamp -import scala.reflect.ClassTag +import org.apache.spark.sql.{DataFrame, SparkSession, functions => f} trait NumericOperations31Spec extends AnyFunSpecLike diff --git a/core/src/test/spark_3.1_3.2_3.3/scala/doric/syntax/StringColumns31Spec.scala b/core/src/test/spark_3.1_3.2_3.3_3.4/scala/doric/syntax/StringColumns31Spec.scala similarity index 99% rename from core/src/test/spark_3.1_3.2_3.3/scala/doric/syntax/StringColumns31Spec.scala rename to core/src/test/spark_3.1_3.2_3.3_3.4/scala/doric/syntax/StringColumns31Spec.scala index cbad9a91d..21884ed40 100644 --- a/core/src/test/spark_3.1_3.2_3.3/scala/doric/syntax/StringColumns31Spec.scala +++ b/core/src/test/spark_3.1_3.2_3.3_3.4/scala/doric/syntax/StringColumns31Spec.scala @@ -3,6 +3,7 @@ package syntax import org.scalatest.{Assertion, EitherValues} import org.scalatest.matchers.should.Matchers + import org.apache.spark.sql.{functions => f} import org.apache.spark.sql.types.NullType diff --git a/core/src/test/spark_3.2_3.3/scala/doric/syntax/AggregationColumns32Spec.scala b/core/src/test/spark_3.2_3.3_3.4/scala/doric/syntax/AggregationColumns32Spec.scala similarity index 100% rename from core/src/test/spark_3.2_3.3/scala/doric/syntax/AggregationColumns32Spec.scala rename to core/src/test/spark_3.2_3.3_3.4/scala/doric/syntax/AggregationColumns32Spec.scala diff --git a/core/src/test/spark_3.2_3.3/scala/doric/types/DeserializeSparkTypeSpec_Specific.scala b/core/src/test/spark_3.2_3.3_3.4/scala/doric/types/DeserializeSparkTypeSpec_Specific.scala similarity index 92% rename from core/src/test/spark_3.2_3.3/scala/doric/types/DeserializeSparkTypeSpec_Specific.scala rename to core/src/test/spark_3.2_3.3_3.4/scala/doric/types/DeserializeSparkTypeSpec_Specific.scala index c07cd0a8d..fb7553ad0 100644 --- a/core/src/test/spark_3.2_3.3/scala/doric/types/DeserializeSparkTypeSpec_Specific.scala +++ b/core/src/test/spark_3.2_3.3_3.4/scala/doric/types/DeserializeSparkTypeSpec_Specific.scala @@ -1,7 +1,6 @@ package doric package types -import org.apache.spark.sql.types.Decimal import org.apache.spark.unsafe.types.CalendarInterval trait DeserializeSparkTypeSpec_Specific extends DoricTestElements { diff --git a/core/src/test/spark_3.2_3.3/scala/doric/types/PrimitiveTypesSpec_Specific.scala b/core/src/test/spark_3.2_3.3_3.4/scala/doric/types/PrimitiveTypesSpec_Specific.scala similarity index 76% rename from core/src/test/spark_3.2_3.3/scala/doric/types/PrimitiveTypesSpec_Specific.scala rename to core/src/test/spark_3.2_3.3_3.4/scala/doric/types/PrimitiveTypesSpec_Specific.scala index 85a0b9da4..bcedac560 100644 --- a/core/src/test/spark_3.2_3.3/scala/doric/types/PrimitiveTypesSpec_Specific.scala +++ b/core/src/test/spark_3.2_3.3_3.4/scala/doric/types/PrimitiveTypesSpec_Specific.scala @@ -1,9 +1,6 @@ package doric package types -import org.apache.spark.sql.types.Decimal -import org.apache.spark.unsafe.types.CalendarInterval - trait PrimitiveTypesSpec_Specific extends DoricTestElements { describe("Simple Java/Scala types (since Spark 3.2)") { diff --git a/core/src/test/spark_3.2_3.3/scala/doric/types/SerializeSparkTypesSpec_Specific.scala b/core/src/test/spark_3.2_3.3_3.4/scala/doric/types/SerializeSparkTypeSpec_Specific.scala similarity index 100% rename from core/src/test/spark_3.2_3.3/scala/doric/types/SerializeSparkTypesSpec_Specific.scala rename to core/src/test/spark_3.2_3.3_3.4/scala/doric/types/SerializeSparkTypeSpec_Specific.scala diff --git a/docs/docs/docs/api.md b/docs/docs/docs/api.md index 77f242766..f4fa0f682 100644 --- a/docs/docs/docs/api.md +++ b/docs/docs/docs/api.md @@ -8,12 +8,14 @@ permalink: docs/api/ Here you can see the whole doric API for every supported spark version: -| Spark | Scala | API | doric | -|:-----:|:-----:|:----------------------------------------:|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------:| -| 2.4.8 | 2.11 | [doric_2-4_2.11](spark-2.4/scala-2.11/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_2-4_2.11)](https://mvnrepository.com/artifact/org.hablapps/doric_2-4_2.11/@STABLE_VERSION@) | -| 3.0.2 | 2.12 | [doric_3-0_2.12](spark-3.0/scala-2.12/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-0_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-0_2.12/@STABLE_VERSION@) | -| 3.1.3 | 2.12 | [doric_3-1_2.12](spark-3.1/scala-2.12/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-1_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-1_2.12/@STABLE_VERSION@) | -| 3.2.3 | 2.12 | [doric_3-2_2.12](spark-3.2/scala-2.12/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-2_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-2_2.12/@STABLE_VERSION@) | -| 3.2.3 | 2.13 | [doric_3-2_2.13](spark-3.2/scala-2.13/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-2_2.13)](https://mvnrepository.com/artifact/org.hablapps/doric_3-2_2.13/@STABLE_VERSION@) | -| 3.3.2 | 2.12 | [doric_3-3_2.12](spark-3.3/scala-2.12/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-3_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-3_2.12/@STABLE_VERSION@) | -| 3.3.2 | 2.13 | [doric_3-3_2.13](spark-3.3/scala-2.13/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-3_2.13)](https://mvnrepository.com/artifact/org.hablapps/doric_3-3_2.13/@STABLE_VERSION@) | +| Spark | Scala | API | doric | +|:-----:|:-----:|:---------------------------------------:|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------:| +| 2.4.8 | 2.11 | [doric_2-4_2.11](spark-2.4/scala-2.11/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_2-4_2.11)](https://mvnrepository.com/artifact/org.hablapps/doric_2-4_2.11/@STABLE_VERSION@) | +| 3.0.2 | 2.12 | [doric_3-0_2.12](spark-3.0/scala-2.12/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-0_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-0_2.12/@STABLE_VERSION@) | +| 3.1.3 | 2.12 | [doric_3-1_2.12](spark-3.1/scala-2.12/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-1_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-1_2.12/@STABLE_VERSION@) | +| 3.2.3 | 2.12 | [doric_3-2_2.12](spark-3.2/scala-2.12/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-2_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-2_2.12/@STABLE_VERSION@) | +| 3.2.3 | 2.13 | [doric_3-2_2.13](spark-3.2/scala-2.13/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-2_2.13)](https://mvnrepository.com/artifact/org.hablapps/doric_3-2_2.13/@STABLE_VERSION@) | +| 3.3.2 | 2.12 | [doric_3-3_2.12](spark-3.3/scala-2.12/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-3_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-3_2.12/@STABLE_VERSION@) | +| 3.3.2 | 2.13 | [doric_3-3_2.13](spark-3.3/scala-2.13/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-3_2.13)](https://mvnrepository.com/artifact/org.hablapps/doric_3-3_2.13/@STABLE_VERSION@) | +| 3.4.0 | 2.12 | [doric_3-3_2.12](spark-3.4/scala-2.12/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-4_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-4_2.12/@STABLE_VERSION@) | +| 3.4.0 | 2.13 | [doric_3-3_2.13](spark-3.4/scala-2.13/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-4_2.13)](https://mvnrepository.com/artifact/org.hablapps/doric_3-4_2.13/@STABLE_VERSION@) | diff --git a/docs/docs/docs/quickstart.md b/docs/docs/docs/quickstart.md index f79e12125..6f06f1a0d 100644 --- a/docs/docs/docs/quickstart.md +++ b/docs/docs/docs/quickstart.md @@ -61,6 +61,8 @@ Doric is committed to use the most modern APIs first. | 3.3.1 | 2.13 | ✅ | - | | 3.3.2 | 2.12 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-3_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-3_2.12/@STABLE_VERSION@) | | 3.3.2 | 2.13 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-3_2.13)](https://mvnrepository.com/artifact/org.hablapps/doric_3-3_2.13/@STABLE_VERSION@) | +| 3.4.0 | 2.12 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-4_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-4_2.12/@STABLE_VERSION@) | +| 3.4.0 | 2.13 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-4_2.13)](https://mvnrepository.com/artifact/org.hablapps/doric_3-4_2.13/@STABLE_VERSION@) | __Import statements__