diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f267e847..6d02a5e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,8 +51,18 @@ jobs: with: python-version: ${{ steps.python_version.outputs.value }} + - name: Get supported Poetry version + uses: haya14busa/action-cond@v1 + id: poetry_version + with: + cond: ${{ startsWith(matrix.pyspark-version, '2.') }} + if_true: '1.5' # latest supported version for PySpark 2.x + if_false: '1.6' # PySpark 3+ + - name: Install Poetry uses: snok/install-poetry@v1 + with: + version: ${{ steps.poetry_version.outputs.value }} - name: Cache Poetry virtualenv uses: actions/cache@v1 diff --git a/README.md b/README.md index 18c63a9c..59bb3a71 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,7 @@ # Quinn ![![image](https://github.com/MrPowers/quinn/workflows/build/badge.svg)](https://github.com/MrPowers/quinn/actions/workflows/ci.yml/badge.svg) - -[![Linter: Ruff](https://img.shields.io/badge/Linter-Ruff-brightgreen?style=flat-square)](https://github.com/MrPowers/quinn/actions/workflows/ruff.yml/badge.svg) +[![image](https://github.com/MrPowers/quinn/workflows/build/badge.svg)](https://github.com/MrPowers/quinn/actions/workflows/ruff.yml/badge.svg) ![PyPI - Downloads](https://img.shields.io/pypi/dm/quinn) [![PyPI version](https://badge.fury.io/py/quinn.svg)](https://badge.fury.io/py/quinn) diff --git a/tests/test_dataframe_validator.py b/tests/test_dataframe_validator.py index b56fb2eb..b30f3f3a 100644 --- a/tests/test_dataframe_validator.py +++ b/tests/test_dataframe_validator.py @@ -37,10 +37,10 @@ def it_raises_when_struct_field_is_missing1(spark): with pytest.raises(quinn.DataFrameMissingStructFieldError) as excinfo: quinn.validate_schema(source_df, required_schema) - if semver.compare(spark.version, "3.2.0") >= 0: # Spark 3.3+ + if semver.compare(spark.version, "3.3.0") >= 0: # Spark 3.3+ expected_error_message = "The [StructField('city', StringType(), True)] StructFields are not included in the DataFrame with the following StructFields StructType([StructField('name', StringType(), True), StructField('age', LongType(), True)])" # noqa else: - expected_error_message = "The [StructField(city,StringType,true)] StructFields are not included in the DataFrame with the following StructFields StructType(List(StructField(name,StringType,true), StructField(age,LongType,true)))" # noqa + expected_error_message = "The [StructField(city,StringType,true)] StructFields are not included in the DataFrame with the following StructFields StructType(List(StructField(name,StringType,true),StructField(age,LongType,true)))" # noqa assert excinfo.value.args[0] == expected_error_message def it_does_nothing_when_the_schema_matches(spark):