diff --git a/.github/actions/build_and_coverage/action.yml b/.github/actions/build_and_coverage/action.yml deleted file mode 100644 index 130b6130c..000000000 --- a/.github/actions/build_and_coverage/action.yml +++ /dev/null @@ -1,76 +0,0 @@ -name: "Build & coverage" -description: "Check the code, build, testing & generate project coverage" - -runs: - using: "composite" - steps: - - # SETUP & CACHE - - uses: coursier/setup-action@v1.3.0 - with: - jvm: adopt:8 - apps: sbt scala scalafmt # https://github.com/coursier/apps - - - uses: coursier/cache-action@v6 - with: - extraKey: "${{ env.SPARK_VERSION }}" - - # CHECK CODE FORMAT - - name: Code Format - id: code_format - if: contains(env.SCOPE, 'test') - run: sbt -DsparkVersion=${{ env.SPARK_VERSION }} scalafmtCheckAll - shell: bash - - # TESTING & COVERAGE - - name: Test & coverage 📋 - id: test_coverage - if: steps.code_format.conclusion == 'success' - run: sbt -DsparkVersion=${{ env.SPARK_VERSION }} coverage +core/test coverageAggregate - shell: bash - - - name: Test Summary - id: test_summary - if: ${{ success() && steps.test_coverage.conclusion == 'success' || failure() && steps.test_coverage.conclusion == 'failure' }} - uses: test-summary/action@v2.0 - with: - paths: "./core/target/test-reports/**/TEST-*.xml" - output: "test-summary.md" - - - name: Add 2 GitHub step summary - if: ${{ (success() || failure()) && steps.test_summary.conclusion == 'success' }} - shell: bash - run: cat test-summary.md >> $GITHUB_STEP_SUMMARY - - - name: Add summary link - id: add_summary_link - if: ${{ (success() || failure()) && steps.test_summary.conclusion == 'success' }} - shell: bash - run: | - img=$(head -1 test-summary.md | perl -pe 's/(<.*?.>).*/$1/') - url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" - echo "| ${{ env.SPARK_VERSION }} | [${img}](${url}) |" > test-summary-${{ env.SPARK_VERSION }}.md - cat -e test-summary-${{ env.SPARK_VERSION }}.md - - - name: Upload test summary - if: ${{ github.event_name == 'pull_request' && ((success() || failure()) && steps.add_summary_link.conclusion == 'success') }} - uses: actions/upload-artifact@v3 - with: - name: "test-summary-${{ env.SPARK_VERSION }}.md" - path: "test-summary-${{ env.SPARK_VERSION }}.md" - if-no-files-found: error - retention-days: 1 - - - name: Publish coverage to codecov 📊 - if: contains(env.SCOPE, 'uploadReport') - uses: codecov/codecov-action@v3 - with: - files: ./target/scala-2.11/scoverage-report/scoverage.xml,./target/scala-2.12/scoverage-report/scoverage.xml,./target/scala-2.13/scoverage-report/scoverage.xml - fail_ci_if_error: true - verbose: false - flags: 'spark-${{ env.SPARK_VERSION }}.x' - - # CLEAN PROJECT BEFORE CACHE - - name: Cleaning before cache 🚯 - if: ${{ always() }} - uses: ./.github/actions/clean_cache diff --git a/.github/actions/publish_coverage/action.yml b/.github/actions/publish_coverage/action.yml new file mode 100644 index 000000000..2b0a26dff --- /dev/null +++ b/.github/actions/publish_coverage/action.yml @@ -0,0 +1,36 @@ +name: "Publish coverage to codecov 📊" +description: "Publish coverage to codecov 📊" + +runs: + using: "composite" + steps: + - name: Attempt 1 + id: attempt_1 + continue-on-error: true + uses: codecov/codecov-action@v3.1.1 + with: + files: ./target/scala-2.11/scoverage-report/scoverage.xml,./target/scala-2.12/scoverage-report/scoverage.xml,./target/scala-2.13/scoverage-report/scoverage.xml + fail_ci_if_error: true + verbose: false + flags: 'spark-${{ env.SPARK_VERSION }}.x' + + - name: Attempt 2 + id: attempt_2 + if: steps.attempt_1.conclusion == 'failure' + continue-on-error: true + uses: codecov/codecov-action@v3.1.1 + with: + files: ./target/scala-2.11/scoverage-report/scoverage.xml,./target/scala-2.12/scoverage-report/scoverage.xml,./target/scala-2.13/scoverage-report/scoverage.xml + fail_ci_if_error: true + verbose: false + flags: 'spark-${{ env.SPARK_VERSION }}.x' + + - name: Attempt 3 + if: steps.attempt_2.conclusion == 'failure' + continue-on-error: false + uses: codecov/codecov-action@v3.1.1 + with: + files: ./target/scala-2.11/scoverage-report/scoverage.xml,./target/scala-2.12/scoverage-report/scoverage.xml,./target/scala-2.13/scoverage-report/scoverage.xml + fail_ci_if_error: true + verbose: false + flags: 'spark-${{ env.SPARK_VERSION }}.x' diff --git a/.github/codecov.yml b/.github/codecov.yml index 70114e3a8..b6cfe0b03 100644 --- a/.github/codecov.yml +++ b/.github/codecov.yml @@ -3,7 +3,7 @@ codecov: branch: main notify: wait_for_ci: true - after_n_builds: 4 + after_n_builds: 5 coverage: precision: 2 @@ -37,4 +37,4 @@ comment: layout: "reach,diff,flags,files,footer" behavior: once # once: update, if exists. Otherwise, post new. Skip if deleted. require_changes: false # if true: only post the comment if coverage changes - after_n_builds: 4 + after_n_builds: 5 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 122431faf..f7db88554 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,7 +51,7 @@ jobs: }" > pr_context.json - name: Upload PR context - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v3.1.2 with: name: "pr_context.json" path: "pr_context.json" @@ -69,7 +69,7 @@ jobs: check_mdoc: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v3.3.0 # BUILD MDOC, sbt should take the latest version - name: Build the microsite 💻 diff --git a/.github/workflows/dependency-graph.yml b/.github/workflows/dependency-graph.yml index 84d7f6fcd..36aca89a4 100644 --- a/.github/workflows/dependency-graph.yml +++ b/.github/workflows/dependency-graph.yml @@ -11,5 +11,5 @@ jobs: name: Update Dependency Graph runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: scalacenter/sbt-dependency-submission@v2 + - uses: actions/checkout@v3.3.0 + - uses: scalacenter/sbt-dependency-submission@v2.1.0 diff --git a/.github/workflows/pr_comment.yml b/.github/workflows/pr_comment.yml index 7f2c28757..906edd686 100644 --- a/.github/workflows/pr_comment.yml +++ b/.github/workflows/pr_comment.yml @@ -52,7 +52,7 @@ jobs: issue-number: ${{ fromJson(steps.output_pr_context.outputs.pr_context).pr_number }} body-includes: "This is an auto-generated comment" -# - uses: actions/checkout@v3 +# - uses: actions/checkout@v3.3.0 # # - name: Render template # id: template diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bbfa8e8e3..54e94714c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -29,7 +29,7 @@ jobs: outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v3.3.0 - id: set-matrix uses: JoshuaTheMiller/conditional-build-matrix@v1.0.1 @@ -45,7 +45,7 @@ jobs: matrix: ${{fromJson(needs.matrix_prep.outputs.matrix)}} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v3.3.0 # SETUP & CACHE - uses: coursier/setup-action@v1.3.0 diff --git a/.github/workflows/wc_build.yml b/.github/workflows/wc_build.yml index ce7c8371b..9b8b7b043 100644 --- a/.github/workflows/wc_build.yml +++ b/.github/workflows/wc_build.yml @@ -13,7 +13,7 @@ jobs: outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v3.3.0 - id: set-matrix uses: JoshuaTheMiller/conditional-build-matrix@v1.0.1 @@ -27,9 +27,69 @@ jobs: matrix: ${{fromJson(needs.matrix_prep.outputs.matrix)}} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v3.3.0 - - uses: ./.github/actions/build_and_coverage - env: - SPARK_VERSION: ${{ matrix.spark }} - SCOPE: ${{ matrix.scope }} + # SETUP & CACHE + - uses: coursier/setup-action@v1.3.0 + with: + jvm: adopt:8 + apps: sbt scala scalafmt # https://github.com/coursier/apps + + - uses: coursier/cache-action@v6 + with: + extraKey: "${{ matrix.spark }}" + + # CHECK CODE FORMAT + - name: Code Format + id: code_format + if: contains(matrix.scope, 'test') + run: sbt -DsparkVersion=${{ matrix.spark }} scalafmtCheckAll + shell: bash + + # TESTING & COVERAGE + - name: Test & coverage 📋 + id: test_coverage + if: steps.code_format.conclusion == 'success' + run: sbt -DsparkVersion=${{ matrix.spark }} coverage +core/test coverageAggregate + shell: bash + + - name: Test Summary + id: test_summary + if: ${{ success() && steps.test_coverage.conclusion == 'success' || failure() && steps.test_coverage.conclusion == 'failure' }} + uses: test-summary/action@v2.1 + with: + paths: "./core/target/test-reports/**/TEST-*.xml" + output: "test-summary.md" + + - name: Add 2 GitHub step summary + if: ${{ (success() || failure()) && steps.test_summary.conclusion == 'success' }} + shell: bash + run: cat test-summary.md >> $GITHUB_STEP_SUMMARY + + - name: Add summary link + id: add_summary_link + if: ${{ (success() || failure()) && steps.test_summary.conclusion == 'success' }} + shell: bash + run: | + img=$(head -1 test-summary.md | perl -pe 's/(<.*?.>).*/$1/') + url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + echo "| ${{ matrix.spark }} | [${img}](${url}) |" > test-summary-${{ matrix.spark }}.md + cat -e test-summary-${{ matrix.spark }}.md + + - name: Upload test summary + if: ${{ github.event_name == 'pull_request' && ((success() || failure()) && steps.add_summary_link.conclusion == 'success') }} + uses: actions/upload-artifact@v3.1.2 + with: + name: "test-summary-${{ matrix.spark }}.md" + path: "test-summary-${{ matrix.spark }}.md" + if-no-files-found: error + retention-days: 1 + + - name: Publish coverage to codecov 📊 + if: contains(matrix.scope, 'uploadReport') + uses: ./.github/actions/publish_coverage + + # CLEAN PROJECT BEFORE CACHE + - name: Cleaning before cache 🚯 + if: ${{ always() }} + uses: ./.github/actions/clean_cache diff --git a/.github/workflows/wc_publishSite.yml b/.github/workflows/wc_publishSite.yml index 07e1a64c0..967c6b150 100644 --- a/.github/workflows/wc_publishSite.yml +++ b/.github/workflows/wc_publishSite.yml @@ -14,7 +14,7 @@ jobs: outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v3.3.0 - id: set-matrix uses: JoshuaTheMiller/conditional-build-matrix@v1.0.1 @@ -29,7 +29,7 @@ jobs: matrix: ${{fromJson(needs.matrix_prep.outputs.matrix)}} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v3.3.0 # SETUP & CACHE - uses: coursier/setup-action@v1.3.0 @@ -61,7 +61,7 @@ jobs: - name: Upload docs ⬆️ if: steps.prepare_scaladocs.conclusion == 'success' - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v3.1.2 with: name: "spark-${{ matrix.spark }}" path: "spark-${{ matrix.spark }}" @@ -79,7 +79,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v3.3.0 # BUILD MDOC, sbt should take the latest version - name: Build the microsite 💻 @@ -88,7 +88,7 @@ jobs: # DOWNLOAD DOCS, this will download all the artifacts because no name provided - name: Download docs ⬇️ - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v3.0.2 with: path: "docs/target/mdoc/docs/api/" diff --git a/core/src/main/scala/doric/syntax/ArrayColumns.scala b/core/src/main/scala/doric/syntax/ArrayColumns.scala index eaf9bd17a..40602a79c 100644 --- a/core/src/main/scala/doric/syntax/ArrayColumns.scala +++ b/core/src/main/scala/doric/syntax/ArrayColumns.scala @@ -92,7 +92,7 @@ private[syntax] trait ArrayColumns { * * @group Array Type * @param n - * the index of the element to retreave. + * the index of the element to retrieve. * @return * the DoricColumn with the selected element. */ diff --git a/core/src/main/scala/doric/syntax/LiteralConversions.scala b/core/src/main/scala/doric/syntax/LiteralConversions.scala index 1908626ce..140232a87 100644 --- a/core/src/main/scala/doric/syntax/LiteralConversions.scala +++ b/core/src/main/scala/doric/syntax/LiteralConversions.scala @@ -29,6 +29,8 @@ private[syntax] trait LiteralConversions { /** * Transforms the original value to a literal. * + * @group All Types + * * @return * a literal with the same type. */ diff --git a/core/src/main/spark_3.0_3.1_3.2_3.3/scala/doric/syntax/ArrayColumns3x.scala b/core/src/main/spark_3.0_3.1_3.2_3.3/scala/doric/syntax/ArrayColumns3x.scala index c35190519..fdc566b7f 100644 --- a/core/src/main/spark_3.0_3.1_3.2_3.3/scala/doric/syntax/ArrayColumns3x.scala +++ b/core/src/main/spark_3.0_3.1_3.2_3.3/scala/doric/syntax/ArrayColumns3x.scala @@ -9,6 +9,11 @@ import org.apache.spark.sql.catalyst.expressions._ trait ArrayColumns3x { + /** + * Extension methods for arrays + * + * @group Array Type + */ implicit class ArrayColumnSyntax3x[T, F[_]: CollectionType]( private val col: DoricColumn[F[T]] ) { diff --git a/docs/docs/docs/api.md b/docs/docs/docs/api.md index 9ad268c6f..7fad0c5d7 100644 --- a/docs/docs/docs/api.md +++ b/docs/docs/docs/api.md @@ -15,5 +15,5 @@ Here you can see the whole doric API for every supported spark version: | 3.1.3 | 2.12 | [doric_3-1_2.12](spark-3.1/scala-2.12/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-1_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-1_2.12/@STABLE_VERSION@) | | 3.2.3 | 2.12 | [doric_3-2_2.12](spark-3.2/scala-2.12/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-2_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-2_2.12/@STABLE_VERSION@) | | 3.2.3 | 2.13 | [doric_3-2_2.13](spark-3.2/scala-2.13/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-2_2.13)](https://mvnrepository.com/artifact/org.hablapps/doric_3-2_2.13/@STABLE_VERSION@) | -| 3.3.0 | 2.12 | [doric_3-3_2.12](spark-3.3/scala-2.12/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-3_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-3_2.12/@STABLE_VERSION@) | +| 3.3.1 | 2.12 | [doric_3-3_2.12](spark-3.3/scala-2.12/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-3_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-3_2.12/@STABLE_VERSION@) | | 3.3.1 | 2.13 | [doric_3-3_2.13](spark-3.3/scala-2.13/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-3_2.13)](https://mvnrepository.com/artifact/org.hablapps/doric_3-3_2.13/@STABLE_VERSION@) | diff --git a/docs/docs/docs/quickstart.md b/docs/docs/docs/quickstart.md index 671be9585..80e8000d5 100644 --- a/docs/docs/docs/quickstart.md +++ b/docs/docs/docs/quickstart.md @@ -31,34 +31,34 @@ Doric is committed to use the most modern APIs first. * The latest experimental version of doric is @VERSION@. * Doric is compatible with the following Spark versions: -| Spark | Scala | Tested | doric | -|:-----:|:-----:|:------:|:------------------------------------------------------------------------------------------------------------------------------------------------------------:| -| 2.4.1 | 2.11 | ✅ | - | -| 2.4.2 | 2.11 | ✅ | - | -| 2.4.3 | 2.11 | ✅ | - | -| 2.4.4 | 2.11 | ✅ | - | -| 2.4.5 | 2.11 | ✅ | - | -| 2.4.6 | 2.11 | ✅ | - | -| 2.4.7 | 2.11 | ✅ | - | +| Spark | Scala | Tested | doric | +|:-----:|:-----:|:------:|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------:| +| 2.4.1 | 2.11 | ✅ | - | +| 2.4.2 | 2.11 | ✅ | - | +| 2.4.3 | 2.11 | ✅ | - | +| 2.4.4 | 2.11 | ✅ | - | +| 2.4.5 | 2.11 | ✅ | - | +| 2.4.6 | 2.11 | ✅ | - | +| 2.4.7 | 2.11 | ✅ | - | | 2.4.8 | 2.11 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_2-4_2.11)](https://mvnrepository.com/artifact/org.hablapps/doric_2-4_2.11/@STABLE_VERSION@) | -| 3.0.0 | 2.12 | ✅ | - | -| 3.0.1 | 2.12 | ✅ | - | -| 3.0.2 | 2.12 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-0_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-0_2.12/0.0.6) | -| 3.1.0 | 2.12 | ✅ | - | -| 3.1.1 | 2.12 | ✅ | - | -| 3.1.2 | 2.12 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-1_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-1_2.12/0.0.6) | -| 3.2.0 | 2.12 | ✅ | - | -| 3.2.0 | 2.13 | ✅ | - | -| 3.2.1 | 2.12 | ✅ | - | -| 3.2.1 | 2.13 | ✅ | - | -| 3.2.2 | 2.12 | ✅ | - | -| 3.2.2 | 2.13 | ✅ | - | -| 3.2.3 | 2.12 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-2_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-2_2.12/0.0.6) | -| 3.2.3 | 2.13 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-2_2.13)](https://mvnrepository.com/artifact/org.hablapps/doric_3-2_2.13/0.0.6) | -| 3.3.0 | 2.12 | ✅ | - | -| 3.3.0 | 2.13 | ✅ | - | -| 3.3.1 | 2.12 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-3_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-3_2.12/0.0.6) | -| 3.3.1 | 2.13 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-3_2.13)](https://mvnrepository.com/artifact/org.hablapps/doric_3-3_2.13/0.0.6) | +| 3.0.0 | 2.12 | ✅ | - | +| 3.0.1 | 2.12 | ✅ | - | +| 3.0.2 | 2.12 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-0_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-0_2.12/@STABLE_VERSION@) | +| 3.1.0 | 2.12 | ✅ | - | +| 3.1.1 | 2.12 | ✅ | - | +| 3.1.2 | 2.12 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-1_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-1_2.12/@STABLE_VERSION@) | +| 3.2.0 | 2.12 | ✅ | - | +| 3.2.0 | 2.13 | ✅ | - | +| 3.2.1 | 2.12 | ✅ | - | +| 3.2.1 | 2.13 | ✅ | - | +| 3.2.2 | 2.12 | ✅ | - | +| 3.2.2 | 2.13 | ✅ | - | +| 3.2.3 | 2.12 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-2_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-2_2.12/@STABLE_VERSION@) | +| 3.2.3 | 2.13 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-2_2.13)](https://mvnrepository.com/artifact/org.hablapps/doric_3-2_2.13/@STABLE_VERSION@) | +| 3.3.0 | 2.12 | ✅ | - | +| 3.3.0 | 2.13 | ✅ | - | +| 3.3.1 | 2.12 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-3_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-3_2.12/@STABLE_VERSION@) | +| 3.3.1 | 2.13 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-3_2.13)](https://mvnrepository.com/artifact/org.hablapps/doric_3-3_2.13/@STABLE_VERSION@) | __Import statements__