diff --git a/.github/labeler.yml b/.github/labeler.yml index b45b9570e..661f376bb 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -32,4 +32,8 @@ spark_3.3: spark_3.4: - changed-files: - - any-glob-to-any-file: [ 'core/src/main/spark*3.4*/**/*', 'core/src/main/scala/**/*' ] + - any-glob-to-any-file: [ 'core/src/main/spark*3.4*/**/*', 'core/src/main/scala/**/*' ] + +spark_3.5: + - changed-files: + - any-glob-to-any-file: [ 'core/src/main/spark*3.5*/**/*', 'core/src/main/scala/**/*' ] diff --git a/.github/workflows/matrix_includes.json b/.github/workflows/matrix_includes.json index e4ef8a2a3..538764b0a 100644 --- a/.github/workflows/matrix_includes.json +++ b/.github/workflows/matrix_includes.json @@ -98,5 +98,15 @@ "spark": "3.4", "scope": "test-uploadReport", "isRelease": "release" + }, + { + "spark": "3.5.0", + "scope": "test", + "isRelease": "-" + }, + { + "spark": "3.5", + "scope": "test-uploadReport", + "isRelease": "release" } ] diff --git a/README.md b/README.md index 556a1035f..df7b88493 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,8 @@ following spark versions. | 3.4.0 | 2.12 / 2.13 | ✅ | - | | 3.4.1 | 2.12 / 2.13 | ✅ | - | | 3.4.2 | 2.12 / 2.13 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-4_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-4_2.12/0.0.7) | +| 3.5.0 | 2.12 / 2.13 | ✅ | - | +| 3.5.1 | 2.12 / 2.13 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-5_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-4_2.12/0.0.7) | diff --git a/build.sbt b/build.sbt index 5aaf9b827..9b913b9f3 100644 --- a/build.sbt +++ b/build.sbt @@ -4,12 +4,13 @@ import scala.language.postfixOps val stableVersion = "0.0.7" -val sparkDefaultShortVersion = "3.4" +val sparkDefaultShortVersion = "3.5" val spark30Version = "3.0.3" val spark31Version = "3.1.3" val spark32Version = "3.2.3" val spark33Version = "3.3.4" val spark34Version = "3.4.2" +val spark35Version = "3.5.1" val versionRegex = """^(.*)\.(.*)\.(.*)$""".r val versionRegexShort = """^(.*)\.(.*)$""".r @@ -23,7 +24,8 @@ val parserSparkVersion: String => String = { case versionRegexShort("3", "2") => spark32Version case versionRegexShort("3", "3") => spark33Version case versionRegexShort("3", "4") => spark34Version - case versionRegex("3", b, c) => s"3.$b.$c" + case versionRegexShort("3", "5") => spark35Version + case versionRegex("3", b, c) => s"3.$b.$c" } val long2ShortVersion: String => String = { case versionRegex(a, b, _) => @@ -36,6 +38,7 @@ val scalaVersionSelect: String => List[String] = { case versionRegex("3", "2", _) => List(scala212, scala213) case versionRegex("3", "3", _) => List(scala212, scala213) case versionRegex("3", "4", _) => List(scala212, scala213) + case versionRegex("3", "5", _) => List(scala212, scala213) } ThisBuild / organization := "org.hablapps" diff --git a/core/src/main/scala/doric/syntax/StringColumns.scala b/core/src/main/scala/doric/syntax/StringColumns.scala index 5db8bdeac..a75b08446 100644 --- a/core/src/main/scala/doric/syntax/StringColumns.scala +++ b/core/src/main/scala/doric/syntax/StringColumns.scala @@ -1,13 +1,14 @@ package doric package syntax +import scala.jdk.CollectionConverters._ + import cats.implicits._ import doric.DoricColumn.sparkFunction + +import org.apache.spark.sql.{Column, functions => f} import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.types.{DataType, StructType} -import org.apache.spark.sql.{Column, functions => f} - -import scala.jdk.CollectionConverters._ protected trait StringColumns { @@ -154,7 +155,7 @@ protected trait StringColumns { * Computes the Levenshtein distance of the two given string columns. * * @group String Type - * @see [[org.apache.spark.sql.functions.levenshtein]] + * @see [[https://spark.apache.org/docs/latest/api/scala/org/apache/spark/sql/functions$.html#levenshtein(l:org.apache.spark.sql.Column,r:org.apache.spark.sql.Column):org.apache.spark.sql.Column]] */ def levenshtein(dc: StringColumn): IntegerColumn = (s.elem, dc.elem).mapN(f.levenshtein).toDC diff --git a/core/src/main/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/ArrayColumns3x.scala b/core/src/main/spark_3.0_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/ArrayColumns3x.scala similarity index 99% rename from core/src/main/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/ArrayColumns3x.scala rename to core/src/main/spark_3.0_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/ArrayColumns3x.scala index a75562d61..fed561ca6 100644 --- a/core/src/main/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/ArrayColumns3x.scala +++ b/core/src/main/spark_3.0_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/ArrayColumns3x.scala @@ -3,7 +3,8 @@ package syntax import cats.implicits._ import doric.types.CollectionType -import org.apache.spark.sql.{Column, functions => f, Row} + +import org.apache.spark.sql.{Column, Row, functions => f} import org.apache.spark.sql.catalyst.expressions._ trait ArrayColumns3x { diff --git a/core/src/main/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/CommonColumns3x.scala b/core/src/main/spark_3.0_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/CommonColumns3x.scala similarity index 100% rename from core/src/main/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/CommonColumns3x.scala rename to core/src/main/spark_3.0_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/CommonColumns3x.scala diff --git a/core/src/main/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/DStructs3x.scala b/core/src/main/spark_3.0_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/DStructs3x.scala similarity index 99% rename from core/src/main/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/DStructs3x.scala rename to core/src/main/spark_3.0_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/DStructs3x.scala index 89b9f5b73..648e9fc1e 100644 --- a/core/src/main/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/DStructs3x.scala +++ b/core/src/main/spark_3.0_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/DStructs3x.scala @@ -1,10 +1,11 @@ package doric package syntax +import scala.jdk.CollectionConverters._ + import doric.types.SparkType -import org.apache.spark.sql.{Row, functions => f} -import scala.jdk.CollectionConverters._ +import org.apache.spark.sql.{Row, functions => f} private[syntax] trait DStructs3x { diff --git a/core/src/main/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/MapColumns3x.scala b/core/src/main/spark_3.0_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/MapColumns3x.scala similarity index 99% rename from core/src/main/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/MapColumns3x.scala rename to core/src/main/spark_3.0_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/MapColumns3x.scala index df24eb232..269c2883f 100644 --- a/core/src/main/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/MapColumns3x.scala +++ b/core/src/main/spark_3.0_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/MapColumns3x.scala @@ -3,8 +3,9 @@ package syntax import cats.implicits._ import doric.types.SparkType -import org.apache.spark.sql.catalyst.expressions.{MapFilter, MapZipWith, TransformKeys, TransformValues} + import org.apache.spark.sql.{Column, Row, functions => f} +import org.apache.spark.sql.catalyst.expressions.{MapFilter, MapZipWith, TransformKeys, TransformValues} trait MapColumns3x { diff --git a/core/src/main/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/StringColumn3x.scala b/core/src/main/spark_3.0_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/StringColumn3x.scala similarity index 99% rename from core/src/main/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/StringColumn3x.scala rename to core/src/main/spark_3.0_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/StringColumn3x.scala index 195a57b7a..5540f461d 100644 --- a/core/src/main/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/StringColumn3x.scala +++ b/core/src/main/spark_3.0_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/StringColumn3x.scala @@ -1,12 +1,13 @@ package doric package syntax +import scala.jdk.CollectionConverters._ + import cats.implicits._ + +import org.apache.spark.sql.{Column, functions => f} import org.apache.spark.sql.catalyst.expressions.StringSplit import org.apache.spark.sql.types.StructType -import org.apache.spark.sql.{Column, functions => f} - -import scala.jdk.CollectionConverters._ trait StringColumn3x { implicit class StringOperationsSyntax3x(s: DoricColumn[String]) { diff --git a/core/src/main/spark_3.0_3.1_3.2_3.3_3.4/scala/org/apache/spark/sql/doric/RelationalGroupedDatasetDoricInterface.scala b/core/src/main/spark_3.0_3.1_3.2_3.3_3.4_3.5/scala/org/apache/spark/sql/doric/RelationalGroupedDatasetDoricInterface.scala similarity index 100% rename from core/src/main/spark_3.0_3.1_3.2_3.3_3.4/scala/org/apache/spark/sql/doric/RelationalGroupedDatasetDoricInterface.scala rename to core/src/main/spark_3.0_3.1_3.2_3.3_3.4_3.5/scala/org/apache/spark/sql/doric/RelationalGroupedDatasetDoricInterface.scala diff --git a/core/src/main/spark_3.1_3.2_3.3_3.4/scala/doric/syntax/AggregationColumns31.scala b/core/src/main/spark_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/AggregationColumns31.scala similarity index 100% rename from core/src/main/spark_3.1_3.2_3.3_3.4/scala/doric/syntax/AggregationColumns31.scala rename to core/src/main/spark_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/AggregationColumns31.scala diff --git a/core/src/main/spark_3.1_3.2_3.3_3.4/scala/doric/syntax/BooleanColumns31.scala b/core/src/main/spark_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/BooleanColumns31.scala similarity index 100% rename from core/src/main/spark_3.1_3.2_3.3_3.4/scala/doric/syntax/BooleanColumns31.scala rename to core/src/main/spark_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/BooleanColumns31.scala diff --git a/core/src/main/spark_3.1_3.2_3.3_3.4/scala/doric/syntax/NumericColumns31.scala b/core/src/main/spark_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/NumericColumns31.scala similarity index 100% rename from core/src/main/spark_3.1_3.2_3.3_3.4/scala/doric/syntax/NumericColumns31.scala rename to core/src/main/spark_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/NumericColumns31.scala diff --git a/core/src/main/spark_3.1_3.2_3.3_3.4/scala/doric/syntax/StringColumns31.scala b/core/src/main/spark_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/StringColumns31.scala similarity index 100% rename from core/src/main/spark_3.1_3.2_3.3_3.4/scala/doric/syntax/StringColumns31.scala rename to core/src/main/spark_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/StringColumns31.scala diff --git a/core/src/main/spark_3.2_3.3_3.4/scala/doric/sqlExpressions/CustomAgg.scala b/core/src/main/spark_3.2_3.3_3.4_3.5/scala/doric/sqlExpressions/CustomAgg.scala similarity index 100% rename from core/src/main/spark_3.2_3.3_3.4/scala/doric/sqlExpressions/CustomAgg.scala rename to core/src/main/spark_3.2_3.3_3.4_3.5/scala/doric/sqlExpressions/CustomAgg.scala diff --git a/core/src/main/spark_3.2_3.3_3.4/scala/doric/syntax/AggregationColumns32.scala b/core/src/main/spark_3.2_3.3_3.4_3.5/scala/doric/syntax/AggregationColumns32.scala similarity index 100% rename from core/src/main/spark_3.2_3.3_3.4/scala/doric/syntax/AggregationColumns32.scala rename to core/src/main/spark_3.2_3.3_3.4_3.5/scala/doric/syntax/AggregationColumns32.scala diff --git a/core/src/main/spark_3.2_3.3_3.4/scala/doric/syntax/BinaryColumns32.scala b/core/src/main/spark_3.2_3.3_3.4_3.5/scala/doric/syntax/BinaryColumns32.scala similarity index 100% rename from core/src/main/spark_3.2_3.3_3.4/scala/doric/syntax/BinaryColumns32.scala rename to core/src/main/spark_3.2_3.3_3.4_3.5/scala/doric/syntax/BinaryColumns32.scala diff --git a/core/src/main/spark_3.2_3.3_3.4/scala/doric/syntax/NumericColumns32.scala b/core/src/main/spark_3.2_3.3_3.4_3.5/scala/doric/syntax/NumericColumns32.scala similarity index 100% rename from core/src/main/spark_3.2_3.3_3.4/scala/doric/syntax/NumericColumns32.scala rename to core/src/main/spark_3.2_3.3_3.4_3.5/scala/doric/syntax/NumericColumns32.scala diff --git a/core/src/main/spark_3.2_3.3_3.4/scala/doric/types/SparkTypeLPI_I_Specific.scala b/core/src/main/spark_3.2_3.3_3.4_3.5/scala/doric/types/SparkTypeLPI_I_Specific.scala similarity index 100% rename from core/src/main/spark_3.2_3.3_3.4/scala/doric/types/SparkTypeLPI_I_Specific.scala rename to core/src/main/spark_3.2_3.3_3.4_3.5/scala/doric/types/SparkTypeLPI_I_Specific.scala diff --git a/core/src/main/spark_3.5_mount/scala/doric/syntax/All.scala b/core/src/main/spark_3.5_mount/scala/doric/syntax/All.scala new file mode 100644 index 000000000..c006bcbbe --- /dev/null +++ b/core/src/main/spark_3.5_mount/scala/doric/syntax/All.scala @@ -0,0 +1,31 @@ +package doric.syntax + +private[doric] trait All + extends ArrayColumns + with TypeMatcher + with CommonColumns + with DStructs + with LiteralConversions + with MapColumns + with NumericColumns + with DateColumns + with TimestampColumns + with BooleanColumns + with StringColumns + with ControlStructures + with AggregationColumns + with CNameOps + with BinaryColumns + with Interpolators + with AggregationColumns31 + with BooleanColumns31 + with NumericColumns31 + with NumericColumns32 + with StringColumns31 + with BinaryColumns32 + with ArrayColumns3x + with CommonColumns3x + with MapColumns3x + with StringColumn3x + with AggregationColumns32 + with DStructs3x diff --git a/core/src/test/scala/doric/sem/ChildColumnNotFound.scala b/core/src/test/scala/doric/sem/ChildColumnNotFound.scala index f4a006928..59758e32d 100644 --- a/core/src/test/scala/doric/sem/ChildColumnNotFound.scala +++ b/core/src/test/scala/doric/sem/ChildColumnNotFound.scala @@ -9,7 +9,10 @@ object ChildColumnNotFound { ): SparkErrorWrapper = { SparkErrorWrapper( new Throwable( - if (!sparkSession.version.startsWith("3.4")) + if ( + !(sparkSession.version.startsWith("3.4") || sparkSession.version + .startsWith("3.5")) + ) s"No such struct field $expectedCol in ${foundCols.mkString(", ")}" else s"[FIELD_NOT_FOUND] No such struct field `$expectedCol` in ${foundCols diff --git a/core/src/test/scala/doric/sem/ColumnNotFound.scala b/core/src/test/scala/doric/sem/ColumnNotFound.scala index afa76fe1d..dcf1a9bc5 100644 --- a/core/src/test/scala/doric/sem/ColumnNotFound.scala +++ b/core/src/test/scala/doric/sem/ColumnNotFound.scala @@ -10,7 +10,10 @@ object ColumnNotFound { SparkErrorWrapper( new Throwable( - if (!sparkSession.version.startsWith("3.4")) + if ( + !(sparkSession.version.startsWith("3.4") || sparkSession.version + .startsWith("3.5")) + ) s"""Cannot resolve column name "$expectedCol" among (${foundCols .mkString(", ")})""" else diff --git a/core/src/test/scala/doric/sem/TransformOpsSpec.scala b/core/src/test/scala/doric/sem/TransformOpsSpec.scala index 348d81de2..acde45c9d 100644 --- a/core/src/test/scala/doric/sem/TransformOpsSpec.scala +++ b/core/src/test/scala/doric/sem/TransformOpsSpec.scala @@ -2,11 +2,12 @@ package doric package sem import doric.implicitConversions._ -import org.apache.spark.sql.Row -import org.apache.spark.sql.types.{DoubleType, IntegerType, StringType, TimestampType} import org.scalatest.matchers.should.Matchers import org.scalatest.EitherValues +import org.apache.spark.sql.Row +import org.apache.spark.sql.types.{DoubleType, IntegerType, StringType, TimestampType} + class TransformOpsSpec extends DoricTestElements with Matchers @@ -122,13 +123,17 @@ class TransformOpsSpec ) } error.getMessage should startWith( - if (!spark.version.startsWith("3.4")) + if ( + !(spark.version.startsWith("3.4") || spark.version.startsWith("3.5")) + ) "Found duplicate column(s) in given column names:" else "[COLUMN_ALREADY_EXISTS] The column `a` already exists. Consider to choose another name or rename the existing column." ) error.getMessage should include("`a`") - if (!spark.version.startsWith("3.4")) { + if ( + !(spark.version.startsWith("3.4") || spark.version.startsWith("3.5")) + ) { error.getMessage should include("`b`") } } diff --git a/core/src/test/scala/doric/syntax/StringColumnsSpec.scala b/core/src/test/scala/doric/syntax/StringColumnsSpec.scala index 89685723f..9ac671120 100644 --- a/core/src/test/scala/doric/syntax/StringColumnsSpec.scala +++ b/core/src/test/scala/doric/syntax/StringColumnsSpec.scala @@ -322,7 +322,7 @@ class StringColumnsSpec extends DoricTestElements { } it("should raise an error if group > regex group result") { - intercept[java.lang.IllegalArgumentException] { + intercept[java.lang.RuntimeException] { df.withColumn( "res", colString("col1").regexpExtract("(\\d+)-(\\d+)".lit, 4.lit) @@ -481,7 +481,9 @@ class StringColumnsSpec extends DoricTestElements { df.testColumns2("col1", "")( (str, pattern) => colString(str).split(pattern.lit), (str, pattern) => f.split(f.col(str), pattern), - if (!spark.version.startsWith("3.4")) + if ( + !(spark.version.startsWith("3.4") || spark.version.startsWith("3.5")) + ) List( Array("h", "e", "l", "l", "o", " ", "w", "o", "r", "l", "d", ""), Array("1", "2", "3", "4", "5", ""), diff --git a/core/src/test/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/DoricTestElementsCompanion_Specific.scala b/core/src/test/spark_3.0_3.1_3.2_3.3_3.4_3.5/scala/doric/DoricTestElementsCompanion_Specific.scala similarity index 100% rename from core/src/test/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/DoricTestElementsCompanion_Specific.scala rename to core/src/test/spark_3.0_3.1_3.2_3.3_3.4_3.5/scala/doric/DoricTestElementsCompanion_Specific.scala diff --git a/core/src/test/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/ArrayColumns3xSpec.scala b/core/src/test/spark_3.0_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/ArrayColumns3xSpec.scala similarity index 99% rename from core/src/test/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/ArrayColumns3xSpec.scala rename to core/src/test/spark_3.0_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/ArrayColumns3xSpec.scala index 6cc689cc2..23f104717 100644 --- a/core/src/test/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/ArrayColumns3xSpec.scala +++ b/core/src/test/spark_3.0_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/ArrayColumns3xSpec.scala @@ -2,11 +2,12 @@ package doric package syntax import doric.testUtilities.data.User -import org.apache.spark.sql.catalyst.expressions.ArraySort -import org.apache.spark.sql.{Column, Row, functions => f} import org.scalatest.EitherValues import org.scalatest.matchers.should.Matchers +import org.apache.spark.sql.{Column, Row, functions => f} +import org.apache.spark.sql.catalyst.expressions.ArraySort + class ArrayColumns3xSpec extends DoricTestElements with EitherValues diff --git a/core/src/test/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/CommonColumns3xSpec.scala b/core/src/test/spark_3.0_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/CommonColumns3xSpec.scala similarity index 100% rename from core/src/test/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/CommonColumns3xSpec.scala rename to core/src/test/spark_3.0_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/CommonColumns3xSpec.scala diff --git a/core/src/test/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/DStructs3xSpec.scala b/core/src/test/spark_3.0_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/DStructs3xSpec.scala similarity index 97% rename from core/src/test/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/DStructs3xSpec.scala rename to core/src/test/spark_3.0_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/DStructs3xSpec.scala index 4794dd378..f45cf0d1d 100644 --- a/core/src/test/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/DStructs3xSpec.scala +++ b/core/src/test/spark_3.0_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/DStructs3xSpec.scala @@ -1,10 +1,11 @@ package doric.syntax -import doric.{DoricTestElements, colStruct} -import org.apache.spark.sql.{functions => f} +import scala.jdk.CollectionConverters._ +import doric.{colStruct, DoricTestElements} import java.sql.Timestamp -import scala.jdk.CollectionConverters._ + +import org.apache.spark.sql.{functions => f} class DStructs3xSpec extends DoricTestElements { diff --git a/core/src/test/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/DateColumns3xSpec.scala b/core/src/test/spark_3.0_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/DateColumns3xSpec.scala similarity index 100% rename from core/src/test/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/DateColumns3xSpec.scala rename to core/src/test/spark_3.0_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/DateColumns3xSpec.scala diff --git a/core/src/test/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/MapColumns3xSpec.scala b/core/src/test/spark_3.0_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/MapColumns3xSpec.scala similarity index 100% rename from core/src/test/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/MapColumns3xSpec.scala rename to core/src/test/spark_3.0_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/MapColumns3xSpec.scala diff --git a/core/src/test/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/StringColumns3xSpec.scala b/core/src/test/spark_3.0_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/StringColumns3xSpec.scala similarity index 99% rename from core/src/test/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/StringColumns3xSpec.scala rename to core/src/test/spark_3.0_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/StringColumns3xSpec.scala index 9a87e9b8d..c1659e062 100644 --- a/core/src/test/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/StringColumns3xSpec.scala +++ b/core/src/test/spark_3.0_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/StringColumns3xSpec.scala @@ -1,13 +1,14 @@ package doric package syntax -import org.apache.spark.sql.types.StructType -import org.apache.spark.sql.{Row, functions => f} +import scala.jdk.CollectionConverters._ + +import java.sql.Timestamp import org.scalatest.EitherValues import org.scalatest.matchers.should.Matchers -import java.sql.Timestamp -import scala.jdk.CollectionConverters._ +import org.apache.spark.sql.{Row, functions => f} +import org.apache.spark.sql.types.StructType class StringColumns3xSpec extends DoricTestElements diff --git a/core/src/test/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/TimestampColumns3xSpec.scala b/core/src/test/spark_3.0_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/TimestampColumns3xSpec.scala similarity index 100% rename from core/src/test/spark_3.0_3.1_3.2_3.3_3.4/scala/doric/syntax/TimestampColumns3xSpec.scala rename to core/src/test/spark_3.0_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/TimestampColumns3xSpec.scala diff --git a/core/src/test/spark_3.1_3.2_3.3_3.4/scala/doric/syntax/AggregationColumns31Spec.scala b/core/src/test/spark_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/AggregationColumns31Spec.scala similarity index 100% rename from core/src/test/spark_3.1_3.2_3.3_3.4/scala/doric/syntax/AggregationColumns31Spec.scala rename to core/src/test/spark_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/AggregationColumns31Spec.scala diff --git a/core/src/test/spark_3.1_3.2_3.3_3.4/scala/doric/syntax/BooleanColumns31Spec.scala b/core/src/test/spark_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/BooleanColumns31Spec.scala similarity index 100% rename from core/src/test/spark_3.1_3.2_3.3_3.4/scala/doric/syntax/BooleanColumns31Spec.scala rename to core/src/test/spark_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/BooleanColumns31Spec.scala diff --git a/core/src/test/spark_3.1_3.2_3.3_3.4/scala/doric/syntax/Numeric31Spec.scala b/core/src/test/spark_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/Numeric31Spec.scala similarity index 100% rename from core/src/test/spark_3.1_3.2_3.3_3.4/scala/doric/syntax/Numeric31Spec.scala rename to core/src/test/spark_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/Numeric31Spec.scala diff --git a/core/src/test/spark_3.1_3.2_3.3_3.4/scala/doric/syntax/StringColumns31Spec.scala b/core/src/test/spark_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/StringColumns31Spec.scala similarity index 100% rename from core/src/test/spark_3.1_3.2_3.3_3.4/scala/doric/syntax/StringColumns31Spec.scala rename to core/src/test/spark_3.1_3.2_3.3_3.4_3.5/scala/doric/syntax/StringColumns31Spec.scala diff --git a/core/src/test/spark_3.2_3.3_3.4/scala/doric/syntax/AggregationColumns32Spec.scala b/core/src/test/spark_3.2_3.3_3.4_3.5/scala/doric/syntax/AggregationColumns32Spec.scala similarity index 100% rename from core/src/test/spark_3.2_3.3_3.4/scala/doric/syntax/AggregationColumns32Spec.scala rename to core/src/test/spark_3.2_3.3_3.4_3.5/scala/doric/syntax/AggregationColumns32Spec.scala diff --git a/core/src/test/spark_3.2_3.3_3.4/scala/doric/types/DeserializeSparkTypeSpec_Specific.scala b/core/src/test/spark_3.2_3.3_3.4_3.5/scala/doric/types/DeserializeSparkTypeSpec_Specific.scala similarity index 100% rename from core/src/test/spark_3.2_3.3_3.4/scala/doric/types/DeserializeSparkTypeSpec_Specific.scala rename to core/src/test/spark_3.2_3.3_3.4_3.5/scala/doric/types/DeserializeSparkTypeSpec_Specific.scala diff --git a/core/src/test/spark_3.2_3.3_3.4/scala/doric/types/PrimitiveTypesSpec_Specific.scala b/core/src/test/spark_3.2_3.3_3.4_3.5/scala/doric/types/PrimitiveTypesSpec_Specific.scala similarity index 100% rename from core/src/test/spark_3.2_3.3_3.4/scala/doric/types/PrimitiveTypesSpec_Specific.scala rename to core/src/test/spark_3.2_3.3_3.4_3.5/scala/doric/types/PrimitiveTypesSpec_Specific.scala diff --git a/core/src/test/spark_3.2_3.3_3.4/scala/doric/types/SerializeSparkTypeSpec_Specific.scala b/core/src/test/spark_3.2_3.3_3.4_3.5/scala/doric/types/SerializeSparkTypeSpec_Specific.scala similarity index 100% rename from core/src/test/spark_3.2_3.3_3.4/scala/doric/types/SerializeSparkTypeSpec_Specific.scala rename to core/src/test/spark_3.2_3.3_3.4_3.5/scala/doric/types/SerializeSparkTypeSpec_Specific.scala diff --git a/docs/docs/docs/api.md b/docs/docs/docs/api.md index f7a85b0a7..7aa6bb10c 100644 --- a/docs/docs/docs/api.md +++ b/docs/docs/docs/api.md @@ -10,7 +10,7 @@ Here you can see the whole doric API for every supported spark version: | Spark | Scala | API | doric | |:-----:|:-----:|:--------------------------------------------------:|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------:| -| 2.4.8 | 2.11 | Deprecated [doric_2-4_2.11](spark-2.4/scala-2.11/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_2-4_2.11)](https://mvnrepository.com/artifact/org.hablapps/doric_2-4_2.11/0.0.7) | +| 2.4.8 | 2.11 | Deprecated [doric_2-4_2.11](spark-2.4/scala-2.11/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_2-4_2.11)](https://mvnrepository.com/artifact/org.hablapps/doric_2-4_2.11/0.0.7) | | 3.0.2 | 2.12 | [doric_3-0_2.12](spark-3.0/scala-2.12/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-0_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-0_2.12/@STABLE_VERSION@) | | 3.1.3 | 2.12 | [doric_3-1_2.12](spark-3.1/scala-2.12/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-1_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-1_2.12/@STABLE_VERSION@) | | 3.2.3 | 2.12 | [doric_3-2_2.12](spark-3.2/scala-2.12/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-2_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-2_2.12/@STABLE_VERSION@) | @@ -19,3 +19,5 @@ Here you can see the whole doric API for every supported spark version: | 3.3.4 | 2.13 | [doric_3-3_2.13](spark-3.3/scala-2.13/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-3_2.13)](https://mvnrepository.com/artifact/org.hablapps/doric_3-3_2.13/@STABLE_VERSION@) | | 3.4.2 | 2.12 | [doric_3-3_2.12](spark-3.4/scala-2.12/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-4_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-4_2.12/@STABLE_VERSION@) | | 3.4.2 | 2.13 | [doric_3-3_2.13](spark-3.4/scala-2.13/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-4_2.13)](https://mvnrepository.com/artifact/org.hablapps/doric_3-4_2.13/@STABLE_VERSION@) | +| 3.5.1 | 2.12 | [doric_3-3_2.12](spark-3.5/scala-2.12/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-5_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-5_2.12/@STABLE_VERSION@) | +| 3.5.1 | 2.13 | [doric_3-3_2.13](spark-3.5/scala-2.13/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-5_2.13)](https://mvnrepository.com/artifact/org.hablapps/doric_3-5_2.13/@STABLE_VERSION@) | diff --git a/docs/docs/docs/quickstart.md b/docs/docs/docs/quickstart.md index 11921a766..f734ecb6a 100644 --- a/docs/docs/docs/quickstart.md +++ b/docs/docs/docs/quickstart.md @@ -31,39 +31,41 @@ Doric is committed to use the most modern APIs first. * The latest experimental version of doric is @VERSION@. * Doric is compatible with the following Spark versions: -| Spark | Scala | Tested | doric | -|:------------------:|:-----:|:------:|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:| -| 2.4.x (Deprecated) | 2.11 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_2-4_2.11)](https://mvnrepository.com/artifact/org.hablapps/doric_2-4_2.11/0.0.7) | -| 3.0.0 | 2.12 | ✅ | - | -| 3.0.1 | 2.12 | ✅ | - | -| 3.0.2 | 2.12 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-0_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-0_2.12/@STABLE_VERSION@) | -| 3.1.0 | 2.12 | ✅ | - | -| 3.1.1 | 2.12 | ✅ | - | -| 3.1.2 | 2.12 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-1_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-1_2.12/@STABLE_VERSION@) | -| 3.2.0 | 2.12 | ✅ | - | -| 3.2.0 | 2.13 | ✅ | - | -| 3.2.1 | 2.12 | ✅ | - | -| 3.2.1 | 2.13 | ✅ | - | -| 3.2.2 | 2.12 | ✅ | - | -| 3.2.2 | 2.13 | ✅ | - | -| 3.2.3 | 2.12 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-2_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-2_2.12/@STABLE_VERSION@) | -| 3.2.3 | 2.13 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-2_2.13)](https://mvnrepository.com/artifact/org.hablapps/doric_3-2_2.13/@STABLE_VERSION@) | -| 3.3.0 | 2.12 | ✅ | - | -| 3.3.0 | 2.13 | ✅ | - | -| 3.3.1 | 2.12 | ✅ | - | -| 3.3.1 | 2.13 | ✅ | - | -| 3.3.2 | 2.12 | ✅ | - | -| 3.3.2 | 2.13 | ✅ | - | -| 3.3.3 | 2.12 | ✅ | - | -| 3.3.3 | 2.13 | ✅ | - | -| 3.3.4 | 2.12 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-3_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-3_2.12/@STABLE_VERSION@) | -| 3.3.4 | 2.13 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-3_2.13)](https://mvnrepository.com/artifact/org.hablapps/doric_3-3_2.13/@STABLE_VERSION@) | -| 3.4.0 | 2.12 | ✅ | - | -| 3.4.0 | 2.13 | ✅ | - | -| 3.4.1 | 2.12 | ✅ | - | -| 3.4.1 | 2.13 | ✅ | - | -| 3.4.2 | 2.12 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-4_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-4_2.12/@STABLE_VERSION@) | -| 3.4.2 | 2.13 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-4_2.13)](https://mvnrepository.com/artifact/org.hablapps/doric_3-4_2.13/@STABLE_VERSION@) | +| Spark | Scala | Tested | doric | +|:------------------:|:-----:|:------:|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------:| +| 2.4.x (Deprecated) | 2.11 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_2-4_2.11)](https://mvnrepository.com/artifact/org.hablapps/doric_2-4_2.11/0.0.7) | +| 3.0.0 | 2.12 | ✅ | - | +| 3.0.1 | 2.12 | ✅ | - | +| 3.0.2 | 2.12 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-0_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-0_2.12/@STABLE_VERSION@) | +| 3.1.0 | 2.12 | ✅ | - | +| 3.1.1 | 2.12 | ✅ | - | +| 3.1.2 | 2.12 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-1_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-1_2.12/@STABLE_VERSION@) | +| 3.2.0 | 2.12 | ✅ | - | +| 3.2.0 | 2.13 | ✅ | - | +| 3.2.1 | 2.12 | ✅ | - | +| 3.2.1 | 2.13 | ✅ | - | +| 3.2.2 | 2.12 | ✅ | - | +| 3.2.2 | 2.13 | ✅ | - | +| 3.2.3 | 2.12 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-2_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-2_2.12/@STABLE_VERSION@) | +| 3.2.3 | 2.13 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-2_2.13)](https://mvnrepository.com/artifact/org.hablapps/doric_3-2_2.13/@STABLE_VERSION@) | +| 3.3.0 | 2.12 | ✅ | - | +| 3.3.0 | 2.13 | ✅ | - | +| 3.3.1 | 2.12 | ✅ | - | +| 3.3.1 | 2.13 | ✅ | - | +| 3.3.2 | 2.12 | ✅ | - | +| 3.3.2 | 2.13 | ✅ | - | +| 3.3.3 | 2.12 | ✅ | - | +| 3.3.3 | 2.13 | ✅ | - | +| 3.3.4 | 2.12 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-3_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-3_2.12/@STABLE_VERSION@) | +| 3.3.4 | 2.13 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-3_2.13)](https://mvnrepository.com/artifact/org.hablapps/doric_3-3_2.13/@STABLE_VERSION@) | +| 3.4.0 | 2.12 | ✅ | - | +| 3.4.0 | 2.13 | ✅ | - | +| 3.4.1 | 2.12 | ✅ | - | +| 3.4.1 | 2.13 | ✅ | - | +| 3.4.2 | 2.12 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-4_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-4_2.12/@STABLE_VERSION@) | +| 3.4.2 | 2.13 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-4_2.13)](https://mvnrepository.com/artifact/org.hablapps/doric_3-4_2.13/@STABLE_VERSION@) | +| 3.5.1 | 2.12 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-5_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-5_2.12/@STABLE_VERSION@) | +| 3.5.1 | 2.13 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-5_2.13)](https://mvnrepository.com/artifact/org.hablapps/doric_3-5_2.13/@STABLE_VERSION@) | __Import statements__