[TEST] Ci doc/refactor

.github/actions/publish_coverage/action.yml

@@ -0,0 +1,37 @@
+name: "Publish coverage to codecov 📊"
+description: "Publish coverage to codecov 📊"
+
+runs:
+  using: "composite"
+  steps:
+
+    - name: Codecov attempt 1
+      id: attempt_1
+      continue-on-error: true
+      uses: codecov/[email protected]
+      with:
+        files: ./target/scala-2.11/scoverage-report/scoverage.xml,./target/scala-2.12/scoverage-report/scoverage.xml,./target/scala-2.13/scoverage-report/scoverage.xml
+        fail_ci_if_error: true
+        verbose: false
+        flags: 'spark-${{ env.SPARK_VERSION }}.x'
+
+    - name: Codecov attempt 2
+      id: attempt_2
+      if: steps.attempt_1.conclusion == 'failure'
+      continue-on-error: true
+      uses: codecov/[email protected]
+      with:
+        files: ./target/scala-2.11/scoverage-report/scoverage.xml,./target/scala-2.12/scoverage-report/scoverage.xml,./target/scala-2.13/scoverage-report/scoverage.xml
+        fail_ci_if_error: true
+        verbose: false
+        flags: 'spark-${{ env.SPARK_VERSION }}.x'
+
+    - name: Codecov attempt 3
+      if: steps.attempt_2.conclusion == 'failure'
+      continue-on-error: false
+      uses: codecov/[email protected]
+      with:
+        files: ./target/scala-2.11/scoverage-report/scoverage.xml,./target/scala-2.12/scoverage-report/scoverage.xml,./target/scala-2.13/scoverage-report/scoverage.xml
+        fail_ci_if_error: true
+        verbose: false
+        flags: 'spark-${{ env.SPARK_VERSION }}.x'

.github/codecov.yml

@@ -3,7 +3,7 @@ codecov:
   branch: main
   notify:
     wait_for_ci: true
-    after_n_builds: 4
+    after_n_builds: 5
 
 coverage:
   precision: 2
@@ -37,4 +37,4 @@ comment:
   layout: "reach,diff,flags,files,footer"
   behavior: once  # once: update, if exists. Otherwise, post new. Skip if deleted.
   require_changes: false  # if true: only post the comment if coverage changes
-  after_n_builds: 4
+  after_n_builds: 5

.github/workflows/ci.yml

@@ -51,7 +51,7 @@ jobs:
           }" > pr_context.json
 
       - name: Upload PR context
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v3.1.2
         with:
           name: "pr_context.json"
           path: "pr_context.json"
@@ -69,7 +69,7 @@ jobs:
   check_mdoc:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v3.3.0
 
       # BUILD MDOC, sbt should take the latest version
       - name: Build the microsite 💻

.github/workflows/dependency-graph.yml

@@ -11,5 +11,5 @@ jobs:
     name: Update Dependency Graph
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: scalacenter/sbt-dependency-submission@v2
+      - uses: actions/checkout@v3.3.0
+      - uses: scalacenter/sbt-dependency-submission@v2.1.0

.github/workflows/pr_comment.yml

@@ -52,7 +52,7 @@ jobs:
           issue-number: ${{ fromJson(steps.output_pr_context.outputs.pr_context).pr_number }}
           body-includes: "This is an auto-generated comment"
 
-#      - uses: actions/checkout@v3
+#      - uses: actions/checkout@v3.3.0
 #
 #      - name: Render template
 #        id: template

.github/workflows/release.yml

@@ -29,7 +29,7 @@ jobs:
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v3.3.0
 
       - id: set-matrix
         uses: JoshuaTheMiller/[email protected]
@@ -45,7 +45,7 @@ jobs:
       matrix: ${{fromJson(needs.matrix_prep.outputs.matrix)}}
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v3.3.0
 
       # SETUP & CACHE
       - uses: coursier/[email protected]

.github/workflows/wc_build.yml

@@ -13,7 +13,28 @@ jobs:
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/[email protected]
+
+      - name: Read codecov file
+        id: read_codecov_yml
+        uses: pietrobolcato/[email protected]
+        with:
+          config: .github/codecov.yml
+
+      - name: Validate releases & codecov file
+        shell: bash
+        env:
+          notify_after_n_builds: steps.read_codecov_yml.outputs.codecov.notify.after_n_builds
+          comment_after_n_builds: steps.read_codecov_yml.outputs.comment.after_n_builds
+        run: |
+          numReleases=$(grep "\"release\"" .github/workflows/matrix_includes.json | wc -l)
+          echo "Number of matrix releases      : $numReleases"
+          echo "Codecov will wait until notify : ${{ env.notify_after_n_builds }}"
+          echo "Codecov will wait until comment: ${{ env.comment_after_n_builds }}"
+          if [[ $numReleases -ne ${{ env.notify_after_n_builds }} ]] || [[ $numReleases -ne ${{ env.comment_after_n_builds }} ]]
+            echo "::error::Codecov config file does not have the same releases as the matrix config file"
+            exit 1
+          fi
 
       - id: set-matrix
         uses: JoshuaTheMiller/[email protected]
@@ -27,9 +48,69 @@ jobs:
       matrix: ${{fromJson(needs.matrix_prep.outputs.matrix)}}
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v3.3.0
 
-      - uses: ./.github/actions/build_and_coverage
-        env:
-          SPARK_VERSION: ${{ matrix.spark }}
-          SCOPE: ${{ matrix.scope }}
+      # SETUP & CACHE
+      - uses: coursier/[email protected]
+        with:
+          jvm: adopt:8
+          apps: sbt scala scalafmt  # https://github.com/coursier/apps
+
+      - uses: coursier/cache-action@v6
+        with:
+          extraKey: "${{ matrix.spark }}"
+
+        # CHECK CODE FORMAT
+      - name: Code Format
+        id: code_format
+        if: contains(matrix.scope, 'test')
+        run: sbt -DsparkVersion=${{ matrix.spark }} scalafmtCheckAll
+        shell: bash
+
+        # TESTING & COVERAGE
+      - name: Test & coverage 📋
+        id: test_coverage
+        if: steps.code_format.conclusion == 'success'
+        run: sbt -DsparkVersion=${{ matrix.spark }} coverage +core/test coverageAggregate
+        shell: bash
+
+      - name: Test Summary
+        id: test_summary
+        if: ${{ success() && steps.test_coverage.conclusion == 'success' || failure() && steps.test_coverage.conclusion == 'failure' }}
+        uses: test-summary/[email protected]
+        with:
+          paths: "./core/target/test-reports/**/TEST-*.xml"
+          output: "test-summary.md"
+
+      - name: Add 2 GitHub step summary
+        if: ${{ (success() || failure()) && steps.test_summary.conclusion == 'success' }}
+        shell: bash
+        run: cat test-summary.md >> $GITHUB_STEP_SUMMARY
+
+      - name: Add summary link
+        id: add_summary_link
+        if: ${{ (success() || failure()) && steps.test_summary.conclusion == 'success' }}
+        shell: bash
+        run: |
+          img=$(head -1 test-summary.md | perl -pe 's/(<.*?.>).*/$1/')
+          url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          echo "| ${{ matrix.spark }} | [${img}](${url}) |" > test-summary-${{ matrix.spark }}.md
+          cat -e test-summary-${{ matrix.spark }}.md
+
+      - name: Upload test summary
+        if: ${{ github.event_name == 'pull_request' && ((success() || failure()) && steps.add_summary_link.conclusion == 'success') }}
+        uses: actions/[email protected]
+        with:
+          name: "test-summary-${{ matrix.spark }}.md"
+          path: "test-summary-${{ matrix.spark }}.md"
+          if-no-files-found: error
+          retention-days: 1
+
+      - name: Publish coverage to codecov 📊
+        if: contains(matrix.scope, 'uploadReport')
+        uses: ./.github/actions/publish_coverage
+
+        # CLEAN PROJECT BEFORE CACHE
+      - name: Cleaning before cache 🚯
+        if: ${{ always() }}
+        uses: ./.github/actions/clean_cache

.github/workflows/wc_publishSite.yml

@@ -14,7 +14,7 @@ jobs:
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v3.3.0
 
       - id: set-matrix
         uses: JoshuaTheMiller/[email protected]
@@ -29,7 +29,7 @@ jobs:
       matrix: ${{fromJson(needs.matrix_prep.outputs.matrix)}}
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v3.3.0
 
       # SETUP & CACHE
       - uses: coursier/[email protected]
@@ -61,7 +61,7 @@ jobs:
 
       - name: Upload docs ⬆️
         if: steps.prepare_scaladocs.conclusion == 'success'
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v3.1.2
         with:
           name: "spark-${{ matrix.spark }}"
           path: "spark-${{ matrix.spark }}"
@@ -79,7 +79,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v3.3.0
 
       # BUILD MDOC, sbt should take the latest version
       - name: Build the microsite 💻
@@ -88,7 +88,7 @@ jobs:
 
       # DOWNLOAD DOCS, this will download all the artifacts because no name provided
       - name: Download docs ⬇️
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v3.0.2
         with:
           path: "docs/target/mdoc/docs/api/"
 

core/src/main/scala/doric/syntax/ArrayColumns.scala

@@ -92,7 +92,7 @@ private[syntax] trait ArrayColumns {
       *
       * @group Array Type
       * @param n
-      * the index of the element to retreave.
+      * the index of the element to retrieve.
       * @return
       * the DoricColumn with the selected element.
       */

core/src/main/scala/doric/syntax/LiteralConversions.scala

@@ -29,6 +29,8 @@ private[syntax] trait LiteralConversions {
     /**
       * Transforms the original value to a literal.
       *
+      * @group All Types
+      *
       * @return
       * a literal with the same type.
       */

core/src/main/spark_3.0_3.1_3.2_3.3/scala/doric/syntax/ArrayColumns3x.scala

@@ -9,6 +9,11 @@ import org.apache.spark.sql.catalyst.expressions._
 
 trait ArrayColumns3x {
 
+  /**
+    * Extension methods for arrays
+    *
+    * @group Array Type
+    */
   implicit class ArrayColumnSyntax3x[T, F[_]: CollectionType](
       private val col: DoricColumn[F[T]]
   ) {