Merge branch 'main' into release/kedro-datasets/5.0.0

.github/workflows/kedro-airflow.yml

@@ -22,7 +22,7 @@ jobs:
     strategy:
       matrix:
         os: [ ubuntu-latest, windows-latest ]
-        python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
+        python-version: [ "3.9", "3.10", "3.11", "3.12" ]
     uses: ./.github/workflows/unit-tests.yml
     with:
       plugin: kedro-airflow
@@ -40,7 +40,7 @@ jobs:
     strategy:
       matrix:
         os: [ ubuntu-latest ]
-        python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
+        python-version: [ "3.9", "3.10", "3.11", "3.12" ]
     uses: ./.github/workflows/e2e-tests.yml
     with:
       plugin: kedro-airflow

.github/workflows/kedro-datasets.yml

@@ -22,7 +22,7 @@ jobs:
     strategy:
       matrix:
         os: [ ubuntu-latest, windows-latest ]
-        python-version: [ "3.9", "3.10", "3.11", "3.12" ]
+        python-version: [ "3.10", "3.11", "3.12" ]
     uses: ./.github/workflows/unit-tests.yml
     with:
       plugin: kedro-datasets

.github/workflows/kedro-docker.yml

@@ -22,7 +22,7 @@ jobs:
     strategy:
       matrix:
         os: [ ubuntu-latest, windows-latest ]
-        python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
+        python-version: [ "3.9", "3.10", "3.11", "3.12" ]
     uses: ./.github/workflows/unit-tests.yml
     with:
       plugin: kedro-docker
@@ -40,7 +40,7 @@ jobs:
     strategy:
       matrix:
         os: [ ubuntu-latest ]
-        python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
+        python-version: [ "3.9", "3.10", "3.11", "3.12" ]
     uses: ./.github/workflows/e2e-tests.yml
     with:
       plugin: kedro-docker

.github/workflows/kedro-telemetry.yml

@@ -22,7 +22,7 @@ jobs:
     strategy:
       matrix:
         os: [ ubuntu-latest ]
-        python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
+        python-version: [ "3.9", "3.10", "3.11", "3.12" ]
     uses: ./.github/workflows/unit-tests.yml
     with:
       plugin: kedro-telemetry

.pre-commit-config.yaml

@@ -42,7 +42,7 @@ repos:
         exclude: ^(?!kedro-airflow/kedro_airflow/).*\.py$
         pass_filenames: false
         stages: [ manual ]
-        entry: ruff kedro-airflow --fix --exit-non-zero-on-fix
+        entry: ruff check kedro-airflow --fix --exit-non-zero-on-fix
 
       - id: ruff-kedro-docker
         name: "Ruff on kedro_docker/*"
@@ -51,7 +51,7 @@ repos:
         exclude: ^(?!kedro-docker/kedro_docker/).*\.py$
         pass_filenames: false
         stages: [ manual ]
-        entry: ruff kedro-docker --fix --exit-non-zero-on-fix
+        entry: ruff check kedro-docker --fix --exit-non-zero-on-fix
 
       - id: ruff-kedro-telemetry
         name: "Ruff on kedro_telemetry/*"
@@ -60,7 +60,7 @@ repos:
         exclude: ^(?!kedro-telemetry/kedro_telemetry/).*\.py$
         pass_filenames: false
         stages: [manual]
-        entry: ruff kedro-telemetry --fix --exit-non-zero-on-fix
+        entry: ruff check kedro-telemetry --fix --exit-non-zero-on-fix
 
       - id: black-kedro-datasets
         name: "Black"

kedro-airflow/README.md

@@ -1,7 +1,7 @@
 # Kedro-Airflow
 
 [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
-[![Python Version](https://img.shields.io/badge/python-3.8%20%7C%203.9%20%7C%203.10%20%7C%203.11%20%7C%203.12-blue.svg)](https://pypi.org/project/kedro-airflow/)
+[![Python Version](https://img.shields.io/badge/python-3.9%20%7C%203.10%20%7C%203.11%20%7C%203.12-blue.svg)](https://pypi.org/project/kedro-airflow/)
 [![PyPI Version](https://badge.fury.io/py/kedro-airflow.svg)](https://pypi.org/project/kedro-airflow/)
 [![Code Style: Black](https://img.shields.io/badge/code%20style-black-black.svg)](https://github.com/ambv/black)
 

kedro-airflow/RELEASE.md

@@ -1,5 +1,8 @@
 # Upcoming Release
 
+# Release 0.10.0
+* Removed support for Python 3.8
+
 # Release 0.9.1
 * Added support to specify `--conf-source` which would point to the runtime configuration directory to be used for running the DAG in airflow. This configuration path is added to the generated DAG.
 

kedro-airflow/features/steps/sh_run.py

@@ -1,10 +1,9 @@
 import shlex
 import subprocess
-from typing import Dict
 
 
 def run(
-    cmd: str, split: bool = True, print_output: bool = False, **kwargs: Dict
+    cmd: str, split: bool = True, print_output: bool = False, **kwargs: dict
 ) -> int:
     """
     Args:

kedro-airflow/pyproject.toml

@@ -8,7 +8,7 @@ authors = [
     {name = "Kedro"}
 ]
 description = "Kedro-Airflow makes it easy to deploy Kedro projects to Airflow"
-requires-python = ">=3.8"
+requires-python = ">=3.9"
 license = {text = "Apache Software License (Apache 2.0)"}
 dependencies = [
     "kedro>=0.19.0",

kedro-datasets/CONTRIBUTING.md

@@ -41,7 +41,7 @@ Core datasets are maintained by the [Kedro Technical Steering Committee (TSC)](h
 3. Must have working doctests (unless complex cloud/DB setup required, which can be discussed in the review).
 4. Must run as part of the regular CI/CD jobs.
 5. Must have 100% test coverage.
-6. Should support all Python versions under NEP 29 (3.9+ currently).
+6. Should support all Python versions under NEP 29 (3.10+ currently).
 7. Should work on Linux, macOS, and Windows.
 
 #### Experimental datasets

kedro-datasets/README.md

@@ -3,7 +3,7 @@
 <!-- Note that the contents of this file are also used in the documentation, see docs/source/index.md -->
 
 [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
-[![Python Version](https://img.shields.io/badge/python-3.9%20%7C%203.10%20%7C%203.11%20%7C%203.12-blue.svg)](https://pypi.org/project/kedro-datasets/)
+[![Python Version](https://img.shields.io/badge/python-3.10%20%7C%203.11%20%7C%203.12-blue.svg)](https://pypi.org/project/kedro-datasets/)
 [![PyPI Version](https://badge.fury.io/py/kedro-datasets.svg)](https://pypi.org/project/kedro-datasets/)
 [![Code Style: Black](https://img.shields.io/badge/code%20style-black-black.svg)](https://github.com/ambv/black)
 

kedro-datasets/RELEASE.md

@@ -3,6 +3,7 @@
 
 # Release 5.0.0
 ## Major features and improvements
+* Removed support for Python 3.9
 * Added the following new **experimental** datasets:
 
 | Type                                | Description                                               | Location                                |
@@ -34,6 +35,7 @@ Many thanks to the following Kedroids for contributing PRs to this release:
 * [janickspirig](https://github.com/janickspirig)
 * [Galen Seilis](https://github.com/galenseilis)
 * [Mariusz Wojakowski](https://github.com/mariusz89016)
+* [Felix Scherz](https://github.com/felixscherz)
 
 
 # Release 4.1.0

kedro-datasets/docs/source/conf.py

@@ -97,7 +97,7 @@
 
 intersphinx_mapping = {
     "kedro": ("https://docs.kedro.org/en/stable/", None),
-    "python": ("https://docs.python.org/3.9/", None),
+    "python": ("https://docs.python.org/3.10/", None),
 }
 
 type_targets = {
@@ -145,7 +145,6 @@
     ),
     "py:data": (
         "typing.Any",
-        "typing.Union",
         "typing.Optional",
         "typing.Tuple",
     ),

kedro-datasets/kedro_datasets/dask/csv_dataset.py

@@ -1,4 +1,4 @@
-"""``CSVDataset`` is a data set used to load and save data to CSV files using Dask
+"""``CSVDataset`` is a dataset used to load and save data to CSV files using Dask
 dataframe"""
 from __future__ import annotations
 

kedro-datasets/kedro_datasets/dask/parquet_dataset.py

@@ -1,4 +1,4 @@
-"""``ParquetDataset`` is a data set used to load and save data to parquet files using Dask
+"""``ParquetDataset`` is a dataset used to load and save data to parquet files using Dask
 dataframe"""
 from __future__ import annotations
 

kedro-datasets/kedro_datasets/databricks/managed_table_dataset.py

@@ -242,7 +242,7 @@ def __init__(  # noqa: PLR0913
             database: the name of the database.
                 (also referred to as schema). Defaults to "default".
             write_mode: the mode to write the data into the table. If not
-                present, the data set is read-only.
+                present, the dataset is read-only.
                 Options are:["overwrite", "append", "upsert"].
                 "upsert" mode requires primary_key field to be populated.
                 Defaults to None.
@@ -297,7 +297,7 @@ def load(self) -> DataFrame | pd.DataFrame:
                 the init doesn't exist
 
         Returns:
-            Union[DataFrame, pd.DataFrame]: Returns a dataframe
+            DataFrame | pd.DataFrame: Returns a dataframe
                 in the format defined in the init
         """
         if self._version and self._version.load >= 0:

kedro-datasets/kedro_datasets/geopandas/geojson_dataset.py

@@ -6,7 +6,7 @@
 
 import copy
 from pathlib import PurePosixPath
-from typing import Any, Union
+from typing import Any
 
 import fsspec
 import geopandas as gpd
@@ -21,7 +21,7 @@
 
 class GeoJSONDataset(
     AbstractVersionedDataset[
-        gpd.GeoDataFrame, Union[gpd.GeoDataFrame, dict[str, gpd.GeoDataFrame]]
+        gpd.GeoDataFrame, gpd.GeoDataFrame | dict[str, gpd.GeoDataFrame]
     ]
 ):
     """``GeoJSONDataset`` loads/saves data to a GeoJSON file using an underlying filesystem

kedro-datasets/kedro_datasets/matplotlib/matplotlib_writer.py

@@ -6,7 +6,7 @@
 import io
 from copy import deepcopy
 from pathlib import PurePosixPath
-from typing import Any, NoReturn, Union
+from typing import Any, NoReturn
 from warnings import warn
 
 import fsspec
@@ -24,7 +24,7 @@
 
 
 class MatplotlibWriter(
-    AbstractVersionedDataset[Union[Figure, list[Figure], dict[str, Figure]], NoReturn]
+    AbstractVersionedDataset[Figure | list[Figure] | dict[str, Figure], NoReturn]
 ):
     """``MatplotlibWriter`` saves one or more Matplotlib objects as
     image files to an underlying filesystem (e.g. local, S3, GCS).
@@ -203,7 +203,7 @@ def load(self) -> NoReturn:
     def save(self, data: Figure | (list[Figure] | dict[str, Figure])) -> None:
         save_path = self._get_save_path()
 
-        if isinstance(data, (list, dict)) and self._overwrite and self._exists():
+        if isinstance(data, list | dict) and self._overwrite and self._exists():
             self._fs.rm(get_filepath_str(save_path, self._protocol), recursive=True)
 
         if isinstance(data, list):

kedro-datasets/kedro_datasets/pandas/excel_dataset.py

@@ -6,7 +6,7 @@
 import logging
 from copy import deepcopy
 from pathlib import PurePosixPath
-from typing import Any, Union
+from typing import Any
 
 import fsspec
 import pandas as pd
@@ -26,8 +26,8 @@
 
 class ExcelDataset(
     AbstractVersionedDataset[
-        Union[pd.DataFrame, dict[str, pd.DataFrame]],
-        Union[pd.DataFrame, dict[str, pd.DataFrame]],
+        pd.DataFrame | dict[str, pd.DataFrame],
+        pd.DataFrame | dict[str, pd.DataFrame],
     ]
 ):
     """``ExcelDataset`` loads/saves data from/to a Excel file using an underlying

kedro-datasets/kedro_datasets/pandas/feather_dataset.py

@@ -1,4 +1,4 @@
-"""``FeatherDataset`` is a data set used to load and save data to feather files
+"""``FeatherDataset`` is a dataset used to load and save data to feather files
 using an underlying filesystem (e.g.: local, S3, GCS). The underlying functionality
 is supported by pandas, so it supports all operations the pandas supports.
 """

kedro-datasets/kedro_datasets/pandas/sql_dataset.py

@@ -309,7 +309,7 @@ class SQLQueryDataset(AbstractDataset[None, pd.DataFrame]):
     by SQLAlchemy can be found here:
     https://docs.sqlalchemy.org/core/engines.html#database-urls
 
-    It does not support save method so it is a read only data set.
+    It does not support save method so it is a read only dataset.
     To save data to a SQL server use ``SQLTableDataset``.
 
     Example usage for the

kedro-datasets/kedro_datasets/partitions/incremental_dataset.py

@@ -9,8 +9,9 @@
 from __future__ import annotations
 
 import operator
+from collections.abc import Callable
 from copy import deepcopy
-from typing import Any, Callable
+from typing import Any
 
 from cachetools import cachedmethod
 from kedro.io.core import (

kedro-datasets/kedro_datasets/partitions/partitioned_dataset.py

@@ -5,9 +5,10 @@
 from __future__ import annotations
 
 import operator
+from collections.abc import Callable
 from copy import deepcopy
 from pathlib import PurePosixPath
-from typing import Any, Callable
+from typing import Any
 from urllib.parse import urlparse
 from warnings import warn
 

kedro-datasets/kedro_datasets/plotly/html_dataset.py

@@ -5,7 +5,7 @@
 
 from copy import deepcopy
 from pathlib import PurePosixPath
-from typing import Any, NoReturn, Union
+from typing import Any, NoReturn
 
 import fsspec
 from kedro.io.core import (
@@ -18,9 +18,7 @@
 from plotly import graph_objects as go
 
 
-class HTMLDataset(
-    AbstractVersionedDataset[go.Figure, Union[go.Figure, go.FigureWidget]]
-):
+class HTMLDataset(AbstractVersionedDataset[go.Figure, go.Figure | go.FigureWidget]):
     """``HTMLDataset`` saves a plotly figure to an HTML file using an
     underlying filesystem (e.g.: local, S3, GCS).
 

kedro-datasets/kedro_datasets/plotly/json_dataset.py

@@ -6,7 +6,7 @@
 import json
 from copy import deepcopy
 from pathlib import PurePosixPath
-from typing import Any, Union
+from typing import Any
 
 import fsspec
 import plotly.io as pio
@@ -21,9 +21,7 @@
 from kedro_datasets._typing import PlotlyPreview
 
 
-class JSONDataset(
-    AbstractVersionedDataset[go.Figure, Union[go.Figure, go.FigureWidget]]
-):
+class JSONDataset(AbstractVersionedDataset[go.Figure, go.Figure | go.FigureWidget]):
     """``JSONDataset`` loads/saves a plotly figure from/to a JSON file using an
     underlying filesystem (e.g.: local, S3, GCS).
 

kedro-datasets/kedro_datasets/polars/lazy_polars_dataset.py

@@ -7,7 +7,7 @@
 import logging
 from copy import deepcopy
 from pathlib import PurePosixPath
-from typing import Any, ClassVar, Union
+from typing import Any, ClassVar
 
 import fsspec
 import polars as pl
@@ -22,7 +22,7 @@
 
 ACCEPTED_FILE_FORMATS = ["csv", "parquet"]
 
-PolarsFrame = Union[pl.LazyFrame, pl.DataFrame]
+PolarsFrame = pl.LazyFrame | pl.DataFrame
 
 logger = logging.getLogger(__name__)
 

kedro-datasets/kedro_datasets/spark/spark_hive_dataset.py

@@ -16,7 +16,7 @@
 
 class SparkHiveDataset(AbstractDataset[DataFrame, DataFrame]):
     """``SparkHiveDataset`` loads and saves Spark dataframes stored on Hive.
-    This data set also handles some incompatible file types such as using partitioned parquet on
+    This dataset also handles some incompatible file types such as using partitioned parquet on
     hive which will not normally allow upserts to existing data without a complete replacement
     of the existing file/partition.
 

kedro-datasets/kedro_datasets/svmlight/svmlight_dataset.py

@@ -6,7 +6,7 @@
 
 from copy import deepcopy
 from pathlib import PurePosixPath
-from typing import Any, Union
+from typing import Any
 
 import fsspec
 from kedro.io.core import (
@@ -25,7 +25,7 @@
 # in kedro-plugins (https://github.com/kedro-org/kedro-plugins)
 
 # Type of data input
-_DI = tuple[Union[ndarray, csr_matrix], ndarray]
+_DI = tuple[ndarray | csr_matrix, ndarray]
 # Type of data output
 _DO = tuple[csr_matrix, ndarray]
 

kedro-datasets/kedro_datasets_experimental/langchain/_anthropic.py

@@ -68,7 +68,7 @@ def _describe(self) -> dict[str, Any]:
         return {**self.kwargs}
 
     def save(self, data: None) -> NoReturn:
-        raise DatasetError(f"{self.__class__.__name__} is a read only data set type")
+        raise DatasetError(f"{self.__class__.__name__} is a read only dataset type")
 
     def load(self) -> ChatAnthropic:
         return ChatAnthropic(