Move tools e2e tests to kedro-starters (#209)

* add tools e2e tests

* add tools.feature

* run example_pipeline with "y"

* fix run

* update tools e2e tests to shortnames

* update check_created_project_structure_from_tools for shortnames

* add shortname changes to e2e run_steps

* fix example_pipeline again

.github/workflows/all-checks.yml

@@ -58,7 +58,7 @@ jobs:
           echo "PATH=$env:HADOOP_HOME\bin;$env:PATH" | Out-File -Append -Encoding ascii -FilePath $env:GITHUB_ENV
       - name: Run `kedro run` end to end tests for all starters
         run: |
-          behave features/run.feature 
+          behave features/run.feature features/tools.feature
 
   lint:
     strategy:

features/steps/run_steps.py

@@ -1,4 +1,5 @@
 import subprocess
+from pathlib import Path
 
 import yaml
 from behave import given, then, when
@@ -47,6 +48,87 @@ def create_project_from_config_file(context, starter_name):
     telemetry_file.write_text("consent: false", encoding="utf-8")
 
 
+@given("I have run a non-interactive kedro new without starter")
+@when("I run a non-interactive kedro new without starter")
+def create_project_without_starter(context):
+    """Behave step to run kedro new given the config I previously created."""
+    res = subprocess.run(
+        [context.kedro, "new", "-c", str(context.config_file)],
+        env=context.env,
+        cwd=context.temp_dir,
+    )
+    assert res.returncode == OK_EXIT_CODE, res
+    # prevent telemetry from prompting for input during e2e tests
+    telemetry_file = context.root_project_dir / ".telemetry"
+    telemetry_file.write_text("consent: false", encoding="utf-8")
+
+
+@given('I have prepared a config file with tools "{tools}"')
+def create_config_file_with_tools(context, tools):
+    """Behave step to create a temporary config file
+    (given the existing temp directory) and store it in the context.
+    It takes a custom tools list and sets example prompt to `n`.
+    """
+
+    context.config_file = context.temp_dir / "config.yml"
+    context.project_name = "project-dummy"
+    context.root_project_dir = context.temp_dir / context.project_name
+    context.package_name = context.project_name.replace("-", "_")
+    config = {
+        "tools": tools,
+        "example_pipeline": "y",
+        "project_name": context.project_name,
+        "repo_name": context.project_name,
+        "output_dir": str(context.temp_dir),
+        "python_package": context.package_name,
+    }
+    with context.config_file.open("w") as config_file:
+        yaml.dump(config, config_file, default_flow_style=False)
+
+
+@then('the expected tool directories and files should be created with "{tools}"')
+def check_created_project_structure_from_tools(context, tools):
+    """Behave step to check the subdirectories created by kedro new with tools."""
+
+    def is_created(name):
+        """Check if path exists."""
+        return (context.root_project_dir / name).exists()
+
+    # Base checks for any project
+    for path in ["README.md", "src", "pyproject.toml", "requirements.txt"]:
+        assert is_created(path), f"{path} does not exist"
+
+    tools_list = (
+        tools.split(",") if tools != "all" else ["lint", "test", "log", "docs", "data", "pyspark", "viz"]
+    )
+
+    if "lint" in tools_list:  # lint tool
+        pass  # No files are added
+
+    if "test" in tools_list:  # test tool
+        assert is_created("tests"), "tests directory does not exist"
+
+    if "log" in tools_list:  # log tool
+        assert is_created("conf/logging.yml"), "logging configuration does not exist"
+
+    if "docs" in tools_list:  # docs tool
+        assert is_created("docs"), "docs directory does not exist"
+
+    if "data" in tools_list:  # data tool
+        assert is_created("data"), "data directory does not exist"
+
+    if "pyspark" in tools_list:  # PySpark tool
+        assert is_created("conf/base/spark.yml"), "spark.yml does not exist"
+
+    if "viz" in tools_list:  # viz tool
+        expected_reporting_path = Path(
+            f"src/{context.package_name}/pipelines/reporting"
+        )
+        assert is_created(
+            expected_reporting_path
+        ), "reporting pipeline directory does not exist"
+
+
 @given("I have installed the Kedro project's dependencies")
 def install_project_dependencies(context):
     reqs_path = "requirements.txt"

features/tools.feature

@@ -0,0 +1,41 @@
+Feature: New Kedro project with tools
+
+  Scenario: Create a new Kedro project without any tools
+    Given I have prepared a config file with tools "none"
+    When I run a non-interactive kedro new without starter
+    Then the expected tool directories and files should be created with "none"
+    Given I have installed the Kedro project's dependencies
+    When I run the Kedro pipeline
+    Then I should get a successful exit code
+
+  Scenario: Create a new Kedro project with all tools except 'viz' and 'pyspark'
+    Given I have prepared a config file with tools "lint, test, log, docs, data"
+    When I run a non-interactive kedro new without starter
+    Then the expected tool directories and files should be created with "lint, test, log, docs, data"
+    Given I have installed the Kedro project's dependencies
+    When I run the Kedro pipeline
+    Then I should get a successful exit code
+
+  Scenario: Create a new Kedro project with all tools
+    Given I have prepared a config file with tools "all"
+    When I run a non-interactive kedro new without starter
+    Then the expected tool directories and files should be created with "all"
+    Given I have installed the Kedro project's dependencies
+    When I run the Kedro pipeline
+    Then I should get a successful exit code
+
+  Scenario: Create a new Kedro project with only 'pyspark' tool
+    Given I have prepared a config file with tools "pyspark"
+    When I run a non-interactive kedro new without starter
+    Then the expected tool directories and files should be created with "pyspark"
+    Given I have installed the Kedro project's dependencies
+    When I run the Kedro pipeline
+    Then I should get a successful exit code
+
+  Scenario: Create a new Kedro project with only 'viz' tool
+    Given I have prepared a config file with tools "viz"
+    When I run a non-interactive kedro new without starter
+    Then the expected tool directories and files should be created with "viz"
+    Given I have installed the Kedro project's dependencies
+    When I run the Kedro pipeline
+    Then I should get a successful exit code