From 1ee44a040e6c729048e90700a9e6cefe81d14b60 Mon Sep 17 00:00:00 2001 From: Kunal Bhattacharya Date: Fri, 23 Feb 2024 13:53:48 +0530 Subject: [PATCH 1/6] v1 of CI workflow to run all tests on latest version of Spark after PR creation --- .github/workflows/pr.yml | 60 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 .github/workflows/pr.yml diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml new file mode 100644 index 00000000..5bb7440b --- /dev/null +++ b/.github/workflows/pr.yml @@ -0,0 +1,60 @@ +name: CI checks on PR + +on: + workflow_dispatch: + pull_request: + branches: + - main + +jobs: + + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - pyspark-version: 3.4.0 + pip-packages: "pyspark==3.4.0" + + steps: + - uses: actions/checkout@v1 + with: + fetch-depth: 1 + + - name: Setup Java + uses: actions/setup-java@v3 + with: + distribution: 'zulu' + java-version: '8' # Supported by Spark 2.x & 3.x + + - name: Set up Python 3.9 + uses: actions/setup-python@v2 + with: + python-version: 3.9 + + - name: Install Poetry + uses: snok/install-poetry@v1 + with: + version: 1.6.1 + + - name: Cache Poetry virtualenv + uses: actions/cache@v1 + id: cache + with: + path: ~/.virtualenvs + key: poetry-${{ hashFiles('**/poetry.lock') }} + restore-keys: | + poetry-${{ hashFiles('**/poetry.lock') }} + + - name: Install dependencies + run: make install_test + if: steps.cache.outputs.cache-hit != 'true' + + - name: Change PySpark to version ${{ matrix.pyspark-version }} + env: + PIP_PACKAGES: ${{ matrix.pip-packages }} + run: poetry run pip install $PIP_PACKAGES # Using pip shouldn't mess up poetry cache + + - name: Run tests with pytest against PySpark ${{ matrix.pyspark-version }} + run: make test From 45ffc033f9cbb49c9ff4a595b93a16a738abb59f Mon Sep 17 00:00:00 2001 From: Kunal Bhattacharya Date: Fri, 23 Feb 2024 16:31:40 +0530 Subject: [PATCH 2/6] Removed matrix strategy and change Spark version jobs --- .github/workflows/pr.yml | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 5bb7440b..1668682e 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -10,15 +10,8 @@ jobs: test: runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - - pyspark-version: 3.4.0 - pip-packages: "pyspark==3.4.0" - steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v3 with: fetch-depth: 1 @@ -51,10 +44,5 @@ jobs: run: make install_test if: steps.cache.outputs.cache-hit != 'true' - - name: Change PySpark to version ${{ matrix.pyspark-version }} - env: - PIP_PACKAGES: ${{ matrix.pip-packages }} - run: poetry run pip install $PIP_PACKAGES # Using pip shouldn't mess up poetry cache - - - name: Run tests with pytest against PySpark ${{ matrix.pyspark-version }} + - name: Run tests with pytest against PySpark 3.4.2 run: make test From e30a568259d6e32bd778f8f880e6c261a5da67b0 Mon Sep 17 00:00:00 2001 From: Kunal Bhattacharya Date: Fri, 23 Feb 2024 21:39:34 +0530 Subject: [PATCH 3/6] Accept commit suggestion Co-authored-by: Semyon --- .github/workflows/pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 1668682e..97cdcd5c 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -44,5 +44,5 @@ jobs: run: make install_test if: steps.cache.outputs.cache-hit != 'true' - - name: Run tests with pytest against PySpark 3.4.2 + - name: Run tests with pytest run: make test From 2ea3fa65de10e73f32659f9ab330b781baa741a3 Mon Sep 17 00:00:00 2001 From: Kunal Bhattacharya Date: Fri, 23 Feb 2024 21:39:51 +0530 Subject: [PATCH 4/6] Accept commit suggestion Co-authored-by: Semyon --- .github/workflows/pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 97cdcd5c..8d0085a1 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -1,4 +1,4 @@ -name: CI checks on PR +name: Testing against single PySpark version on: workflow_dispatch: From f44398cf484bef389f6217727e8db426f4f3a713 Mon Sep 17 00:00:00 2001 From: Kunal Bhattacharya Date: Sun, 25 Feb 2024 20:59:27 +0530 Subject: [PATCH 5/6] Added GHA job to only run PR workflow in case code has been changed --- .github/workflows/pr.yml | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 8d0085a1..001075fd 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -8,31 +8,54 @@ on: jobs: + detect_code_changes: + runs-on: ubuntu-latest + outputs: + code_changes: ${{ steps.filter.outputs.code_changes }} + steps: + - uses: dorny/paths-filter@v3 + id: changes + with: + filters: | + code_changes: + - 'quinn/**' + - 'tests/**' + - 'benchmarks/**' + - '.github/**' + - 'poetry.lock' + - 'pyproject.toml' + test: runs-on: ubuntu-latest + needs: [detect_code_changes] steps: - uses: actions/checkout@v3 + if: needs.detect_code_changes.outputs.code_changes == 'true' with: fetch-depth: 1 - name: Setup Java uses: actions/setup-java@v3 + if: needs.detect_code_changes.outputs.code_changes == 'true' with: distribution: 'zulu' java-version: '8' # Supported by Spark 2.x & 3.x - name: Set up Python 3.9 - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 + if: needs.detect_code_changes.outputs.code_changes == 'true' with: python-version: 3.9 - name: Install Poetry uses: snok/install-poetry@v1 + if: needs.detect_code_changes.outputs.code_changes == 'true' with: version: 1.6.1 - name: Cache Poetry virtualenv uses: actions/cache@v1 + if: needs.detect_code_changes.outputs.code_changes == 'true' id: cache with: path: ~/.virtualenvs @@ -41,8 +64,12 @@ jobs: poetry-${{ hashFiles('**/poetry.lock') }} - name: Install dependencies + if: | + needs.detect_code_changes.outputs.code_changes == 'true' && + steps.cache.outputs.cache-hit != 'true' run: make install_test - if: steps.cache.outputs.cache-hit != 'true' + # if: steps.cache.outputs.cache-hit != 'true' - name: Run tests with pytest + if: needs.detect_code_changes.outputs.code_changes == 'true' run: make test From b19ccc58b1dd7124e4229a74747d1798444a67c3 Mon Sep 17 00:00:00 2001 From: Kunal Bhattacharya Date: Sun, 25 Feb 2024 21:05:01 +0530 Subject: [PATCH 6/6] Typo fix --- .github/workflows/pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 001075fd..b38851c8 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -11,7 +11,7 @@ jobs: detect_code_changes: runs-on: ubuntu-latest outputs: - code_changes: ${{ steps.filter.outputs.code_changes }} + code_changes: ${{ steps.changes.outputs.code_changes }} steps: - uses: dorny/paths-filter@v3 id: changes