From 927b6142f876292d7a426d3a4ae48dcbefc0888c Mon Sep 17 00:00:00 2001 From: Benjamin Forehand Jr Date: Wed, 25 Sep 2024 11:36:58 -0500 Subject: [PATCH] Fix #160, add slug and branch reporting to HTML reports. --- .github/workflows/android_manual.yml | 4 +-- .github/workflows/ios_manual.yml | 4 +-- tests/android/conftest.py | 53 +++++++++++++++++++++++----- tests/conftest.py | 29 +++++++++++++-- tests/ios/conftest.py | 48 ++++++++++++++++++++----- 5 files changed, 116 insertions(+), 22 deletions(-) diff --git a/.github/workflows/android_manual.yml b/.github/workflows/android_manual.yml index 4bb0563..6897262 100644 --- a/.github/workflows/android_manual.yml +++ b/.github/workflows/android_manual.yml @@ -160,8 +160,8 @@ jobs: pipenv install pipenv install pytest-rerunfailures pipenv run python generate_smoke_tests.py - echo "PIPENV COMMAND: 'pipenv run pytest --experiment ${{ inputs.slug }} --experiment-branch ${{ matrix.branch }} --experiment-server ${{ inputs.experiment-server }} --experiment-feature ${{ inputs.feature-name }} ${{ inputs.extra-arguments }} --reruns 1'" - pipenv run pytest --experiment ${{ inputs.slug }} --experiment-branch ${{ matrix.branch }} --experiment-server ${{ inputs.experiment-server }} --experiment-feature ${{ inputs.feature-name }} ${{ inputs.extra-arguments }} --reruns 1 + echo "TEST COMMAND: pipenv run pytest --experiment-slug ${{ inputs.slug }} --experiment-branch ${{ matrix.branch }} --experiment-server ${{ inputs.experiment-server }} --experiment-feature ${{ inputs.feature-name }} --firefox-version ${{ matrix.firefox }} ${{ inputs.extra-arguments }} --reruns 1" + pipenv run pytest --experiment-slug ${{ inputs.slug }} --experiment-branch ${{ matrix.branch }} --experiment-server ${{ inputs.experiment-server }} --experiment-feature ${{ inputs.feature-name }} --firefox-version ${{ matrix.firefox }} ${{ inputs.extra-arguments }} --reruns 1 - uses: actions/upload-artifact@v4 if: ${{ always() }} with: diff --git a/.github/workflows/ios_manual.yml b/.github/workflows/ios_manual.yml index 76a6002..cdd0c9c 100644 --- a/.github/workflows/ios_manual.yml +++ b/.github/workflows/ios_manual.yml @@ -101,8 +101,8 @@ jobs: xcrun simctl shutdown all export SIMULATOR_UDID=$(python get_specific_device_udid.py) poetry run python generate_smoke_tests.py - echo "TEST COMMAND: poetry run pytest --experiment ${{ inputs.slug }} --experiment-branch ${{ matrix.branch }} --experiment-server ${{ inputs.experiment-server }} --experiment-feature ${{ inputs.feature-name }} ${{ inputs.extra-arguments }} --reruns 1" - poetry run pytest --experiment ${{ inputs.slug }} --experiment-branch ${{ matrix.branch }} --experiment-server ${{ inputs.experiment-server }} --experiment-feature ${{ inputs.feature-name }} ${{ inputs.extra-arguments }} --reruns 1 + echo "TEST COMMAND: poetry run pytest --experiment-slug ${{ inputs.slug }} --experiment-branch ${{ matrix.branch }} --experiment-server ${{ inputs.experiment-server }} --experiment-feature ${{ inputs.feature-name }} --firefox-version ${{ matrix.firefox }} ${{ inputs.extra-arguments }} --reruns 1" + poetry run pytest --experiment-slug ${{ inputs.slug }} --experiment-branch ${{ matrix.branch }} --experiment-server ${{ inputs.experiment-server }} --experiment-feature ${{ inputs.feature-name }} --firefox-version ${{ matrix.firefox }} ${{ inputs.extra-arguments }} --reruns 1 - name: Archive Results if: ${{ always() }} run: zip -r results.zip /Users/runner/Library/Developer/Xcode/DerivedData/**/Logs/Test/*.xcresult diff --git a/tests/android/conftest.py b/tests/android/conftest.py index 48f3e93..98f17fa 100644 --- a/tests/android/conftest.py +++ b/tests/android/conftest.py @@ -11,6 +11,7 @@ import pytest import requests +from pytest_metadata.plugin import metadata_key from .gradlewbuild import GradlewBuild from .models.models import TelemetryModel @@ -27,7 +28,6 @@ def pytest_addoption(parser): action="store", help="Feature name you want to test against", ) - parser.addoption("--experiment", action="store", help="Feature name you want to test against") parser.addoption( "--experiment-branch", action="store", @@ -40,6 +40,18 @@ def pytest_addoption(parser): default="prod", help="Experiment Server the experiment is hosted on", ) + parser.addoption( + "--experiment-slug", + action="store", + default=None, + help="Experiment slug from Experimenter", + ) + parser.addoption( + "--firefox-version", + action="store", + default=None, + help="The Firefox Version you are testing on. This is just use for reporting", + ) def pytest_runtest_setup(item): @@ -75,9 +87,25 @@ def fixture_nimbus_cli_args(): return "" -@pytest.fixture(name="experiment_branch") +@pytest.fixture(name="experiment_branch", scope="session", autouse=True) def fixture_experiment_branch(request): - return request.config.getoption("--experiment-branch") + branch = request.config.getoption("--experiment-branch") + os.environ["EXPERIMENT_BRANCH"] = branch + return branch + + +@pytest.fixture(name="experiment_slug", scope="session", autouse=True) +def fixture_experiment_slug(request): + slug = request.config.getoption("--experiment-slug") + os.environ["EXPERIMENT_SLUG"] = slug + return slug + + +@pytest.fixture(name="firefox_version", scope="session", autouse=True) +def fixture_firefox_version(request): + ff_version = request.config.getoption("--firefox-version") + os.environ["FIREFOX_VERSION"] = ff_version + return ff_version @pytest.fixture(name="experiment_server") @@ -157,11 +185,6 @@ def fixture_experiment_url(request, variables): pass -@pytest.fixture(name="experiment_slug") -def fixture_experiment_slug(request): - return request.config.getoption("--experiment") - - @pytest.fixture(name="ping_server", autouse=True, scope="session") def fixture_ping_server(): process = start_process("ping_server", ["python", "ping_server.py"]) @@ -271,6 +294,20 @@ def fixture_set_experiment_test_name(experiment_data): os.environ["EXP_NAME"] = exp_name[random.randint(0, len(exp_name) - 1)] +@pytest.hookimpl(tryfirst=True) +def pytest_sessionfinish(session, exitstatus): + # Add data to html report + session.config.stash[metadata_key]["Experiment Slug"] = os.environ.get( + "EXPERIMENT_SLUG", "N/A" + ) + session.config.stash[metadata_key]["Experiment Branch"] = os.environ.get( + "EXPERIMENT_BRANCH", "N/A" + ) + session.config.stash[metadata_key]["Firefox Version"] = os.environ.get( + "FIREFOX_VERSION", "N/A" + ) + + @pytest.fixture(name="setup_experiment") def fixture_setup_experiment( experiment_slug, diff --git a/tests/conftest.py b/tests/conftest.py index 877d726..f42ec7b 100755 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,6 +15,7 @@ import pytest import requests from pytest_bdd import given, then +from pytest_metadata.plugin import metadata_key from selenium.common.exceptions import JavascriptException from selenium.webdriver import ActionChains, Keys from selenium.webdriver.common.by import By @@ -148,9 +149,18 @@ def fixture_enroll_experiment( logging.info("Experiment loaded successfully!") -@pytest.fixture(name="experiment_slug") +@pytest.fixture(name="experiment_slug", scope="session", autouse=True) def fixture_experiment_slug(request) -> typing.Any: - return request.config.getoption("--experiment-slug") + slug = request.config.getoption("--experiment-slug") + os.environ["EXPERIMENT_SLUG"] = slug + return slug + + +@pytest.fixture(name="experiment_branch", scope="session", autouse=True) +def fixture_experiment_branch(request): + branch = request.config.getoption("--experiment-branch") + os.environ["EXPERIMENT_BRANCH"] = branch + return branch @pytest.fixture @@ -512,9 +522,24 @@ def fixture_firefox_version(selenium): version = selenium.execute_script(script) version = [item for item in version.split() if "Firefox" in item][0] logging.info(f"Firefox version {version}") + os.environ["FIREFOX_VERSION"] = version return version +@pytest.hookimpl(tryfirst=True) +def pytest_sessionfinish(session, exitstatus): + # Add data to html report + session.config.stash[metadata_key]["Experiment Slug"] = os.environ.get( + "EXPERIMENT_SLUG", "N/A" + ) + session.config.stash[metadata_key]["Experiment Branch"] = os.environ.get( + "EXPERIMENT_BRANCH", "N/A" + ) + session.config.stash[metadata_key]["Firefox Version"] = os.environ.get( + "FIREFOX_VERSION", "N/A" + ) + + @then("Firefox should be allowed to open a new tab") def open_a_new_tab(selenium): with selenium.context(selenium.CONTEXT_CHROME): diff --git a/tests/ios/conftest.py b/tests/ios/conftest.py index a7db369..4a5e122 100644 --- a/tests/ios/conftest.py +++ b/tests/ios/conftest.py @@ -10,6 +10,7 @@ import pytest import requests +from pytest_metadata.plugin import metadata_key from .models.models import TelemetryModel from .xcodebuild import XCodeBuild @@ -22,7 +23,7 @@ def pytest_addoption(parser): - parser.addoption("--experiment", action="store", help="The experiments experimenter URL") + parser.addoption("--experiment-slug", action="store", help="The experiments experimenter URL") parser.addoption("--stage", action="store_true", default=None, help="Use the stage server") parser.addoption( "--build-dev", @@ -45,6 +46,12 @@ def pytest_addoption(parser): default="prod", help="Experiment Server the experiment is hosted on", ) + parser.addoption( + "--firefox-version", + action="store", + default=None, + help="The Firefox Version you are testing on. This is just use for reporting", + ) def pytest_runtest_setup(item): @@ -59,9 +66,25 @@ def fixture_nimbus_cli_args(): return "FIREFOX_SKIP_INTRO FIREFOX_TEST DISABLE_ANIMATIONS 'GCDWEBSERVER_PORT:7777'" -@pytest.fixture(name="experiment_branch") +@pytest.fixture(name="experiment_slug", scope="session", autouse=True) +def fixture_experiment_slug(request): + slug = request.config.getoption("--experiment-slug") + os.environ["EXPERIMENT_SLUG"] = slug + return slug + + +@pytest.fixture(name="experiment_branch", scope="session", autouse=True) def fixture_experiment_branch(request): - return request.config.getoption("--experiment-branch") + branch = request.config.getoption("--experiment-branch") + os.environ["EXPERIMENT_BRANCH"] = branch + return branch + + +@pytest.fixture(name="firefox_version", scope="session", autouse=True) +def fixture_firefox_version(request): + ff_version = request.config.getoption("--firefox-version") + os.environ["FIREFOX_VERSION"] = ff_version + return ff_version @pytest.fixture(name="experiment_server") @@ -195,11 +218,6 @@ def fixture_experiment_url(request, variables): pass -@pytest.fixture(name="experiment_slug") -def fixture_experiment_slug(request): - return request.config.getoption("--experiment") - - @pytest.fixture(name="send_test_results", scope="session") def fixture_send_test_results(xcrun): yield @@ -289,3 +307,17 @@ def _setup_experiment(): run_nimbus_cli_command(" ".join(command)) return _setup_experiment + + +@pytest.hookimpl(tryfirst=True) +def pytest_sessionfinish(session, exitstatus): + # Add data to html report + session.config.stash[metadata_key]["Experiment Slug"] = os.environ.get( + "EXPERIMENT_SLUG", "N/A" + ) + session.config.stash[metadata_key]["Experiment Branch"] = os.environ.get( + "EXPERIMENT_BRANCH", "N/A" + ) + session.config.stash[metadata_key]["Firefox Version"] = os.environ.get( + "FIREFOX_VERSION", "N/A" + )