From 378389135d89f6c55c79324d0055a8a2891587b2 Mon Sep 17 00:00:00 2001 From: Sajid Alam <90610031+SajidAlamQB@users.noreply.github.com> Date: Thu, 18 Jan 2024 18:49:57 +0000 Subject: [PATCH] Move tools e2e tests to kedro-starters (#209) * add tools e2e tests * add tools.feature * run example_pipeline with "y" * fix run * update tools e2e tests to shortnames * update check_created_project_structure_from_tools for shortnames * add shortname changes to e2e run_steps * fix example_pipeline again --- .github/workflows/all-checks.yml | 2 +- features/steps/run_steps.py | 82 ++++++++++++++++++++++++++++++++ features/tools.feature | 41 ++++++++++++++++ 3 files changed, 124 insertions(+), 1 deletion(-) create mode 100644 features/tools.feature diff --git a/.github/workflows/all-checks.yml b/.github/workflows/all-checks.yml index 4b6895ab..5365b094 100644 --- a/.github/workflows/all-checks.yml +++ b/.github/workflows/all-checks.yml @@ -58,7 +58,7 @@ jobs: echo "PATH=$env:HADOOP_HOME\bin;$env:PATH" | Out-File -Append -Encoding ascii -FilePath $env:GITHUB_ENV - name: Run `kedro run` end to end tests for all starters run: | - behave features/run.feature + behave features/run.feature features/tools.feature lint: strategy: diff --git a/features/steps/run_steps.py b/features/steps/run_steps.py index bb39fa4e..c0853e48 100644 --- a/features/steps/run_steps.py +++ b/features/steps/run_steps.py @@ -1,4 +1,5 @@ import subprocess +from pathlib import Path import yaml from behave import given, then, when @@ -47,6 +48,87 @@ def create_project_from_config_file(context, starter_name): telemetry_file.write_text("consent: false", encoding="utf-8") +@given("I have run a non-interactive kedro new without starter") +@when("I run a non-interactive kedro new without starter") +def create_project_without_starter(context): + """Behave step to run kedro new given the config I previously created.""" + res = subprocess.run( + [context.kedro, "new", "-c", str(context.config_file)], + env=context.env, + cwd=context.temp_dir, + ) + assert res.returncode == OK_EXIT_CODE, res + # prevent telemetry from prompting for input during e2e tests + telemetry_file = context.root_project_dir / ".telemetry" + telemetry_file.write_text("consent: false", encoding="utf-8") + + +@given('I have prepared a config file with tools "{tools}"') +def create_config_file_with_tools(context, tools): + """Behave step to create a temporary config file + (given the existing temp directory) and store it in the context. + It takes a custom tools list and sets example prompt to `n`. + """ + + context.config_file = context.temp_dir / "config.yml" + context.project_name = "project-dummy" + context.root_project_dir = context.temp_dir / context.project_name + context.package_name = context.project_name.replace("-", "_") + config = { + "tools": tools, + "example_pipeline": "y", + "project_name": context.project_name, + "repo_name": context.project_name, + "output_dir": str(context.temp_dir), + "python_package": context.package_name, + } + with context.config_file.open("w") as config_file: + yaml.dump(config, config_file, default_flow_style=False) + + +@then('the expected tool directories and files should be created with "{tools}"') +def check_created_project_structure_from_tools(context, tools): + """Behave step to check the subdirectories created by kedro new with tools.""" + + def is_created(name): + """Check if path exists.""" + return (context.root_project_dir / name).exists() + + # Base checks for any project + for path in ["README.md", "src", "pyproject.toml", "requirements.txt"]: + assert is_created(path), f"{path} does not exist" + + tools_list = ( + tools.split(",") if tools != "all" else ["lint", "test", "log", "docs", "data", "pyspark", "viz"] + ) + + if "lint" in tools_list: # lint tool + pass # No files are added + + if "test" in tools_list: # test tool + assert is_created("tests"), "tests directory does not exist" + + if "log" in tools_list: # log tool + assert is_created("conf/logging.yml"), "logging configuration does not exist" + + if "docs" in tools_list: # docs tool + assert is_created("docs"), "docs directory does not exist" + + if "data" in tools_list: # data tool + assert is_created("data"), "data directory does not exist" + + if "pyspark" in tools_list: # PySpark tool + assert is_created("conf/base/spark.yml"), "spark.yml does not exist" + + if "viz" in tools_list: # viz tool + expected_reporting_path = Path( + f"src/{context.package_name}/pipelines/reporting" + ) + assert is_created( + expected_reporting_path + ), "reporting pipeline directory does not exist" + + @given("I have installed the Kedro project's dependencies") def install_project_dependencies(context): reqs_path = "requirements.txt" diff --git a/features/tools.feature b/features/tools.feature new file mode 100644 index 00000000..c43d96e8 --- /dev/null +++ b/features/tools.feature @@ -0,0 +1,41 @@ +Feature: New Kedro project with tools + + Scenario: Create a new Kedro project without any tools + Given I have prepared a config file with tools "none" + When I run a non-interactive kedro new without starter + Then the expected tool directories and files should be created with "none" + Given I have installed the Kedro project's dependencies + When I run the Kedro pipeline + Then I should get a successful exit code + + Scenario: Create a new Kedro project with all tools except 'viz' and 'pyspark' + Given I have prepared a config file with tools "lint, test, log, docs, data" + When I run a non-interactive kedro new without starter + Then the expected tool directories and files should be created with "lint, test, log, docs, data" + Given I have installed the Kedro project's dependencies + When I run the Kedro pipeline + Then I should get a successful exit code + + Scenario: Create a new Kedro project with all tools + Given I have prepared a config file with tools "all" + When I run a non-interactive kedro new without starter + Then the expected tool directories and files should be created with "all" + Given I have installed the Kedro project's dependencies + When I run the Kedro pipeline + Then I should get a successful exit code + + Scenario: Create a new Kedro project with only 'pyspark' tool + Given I have prepared a config file with tools "pyspark" + When I run a non-interactive kedro new without starter + Then the expected tool directories and files should be created with "pyspark" + Given I have installed the Kedro project's dependencies + When I run the Kedro pipeline + Then I should get a successful exit code + + Scenario: Create a new Kedro project with only 'viz' tool + Given I have prepared a config file with tools "viz" + When I run a non-interactive kedro new without starter + Then the expected tool directories and files should be created with "viz" + Given I have installed the Kedro project's dependencies + When I run the Kedro pipeline + Then I should get a successful exit code