diff --git a/kedro-datasets/tests/spark/test_spark_streaming_dataset.py b/kedro-datasets/tests/spark/test_spark_streaming_dataset.py index 660e43a82..ca252773d 100644 --- a/kedro-datasets/tests/spark/test_spark_streaming_dataset.py +++ b/kedro-datasets/tests/spark/test_spark_streaming_dataset.py @@ -4,9 +4,11 @@ import pytest from kedro.io.core import DataSetError from moto import mock_s3 +from pyspark import __version__ from pyspark.sql import SparkSession from pyspark.sql.types import IntegerType, StringType, StructField, StructType from pyspark.sql.utils import AnalysisException +from semver import VersionInfo from kedro_datasets.spark.spark_dataset import SparkDataSet from kedro_datasets.spark.spark_streaming_dataset import SparkStreamingDataSet @@ -15,6 +17,7 @@ BUCKET_NAME = "test_bucket" AWS_CREDENTIALS = {"key": "FAKE_ACCESS_KEY", "secret": "FAKE_SECRET_KEY"} +SPARK_VERSION = VersionInfo.parse(__version__) def sample_schema(schema_path): """read the schema file from json path"""