Fix #160, add slug and branch reporting to HTML reports.

mozilla · Sep 25, 2024 · 5ab4280 · 5ab4280
1 parent 809c056
commit 5ab4280
Show file tree

Hide file tree

Showing 5 changed files with 116 additions and 22 deletions.
diff --git a/.github/workflows/android_manual.yml b/.github/workflows/android_manual.yml
@@ -160,8 +160,8 @@ jobs:
           pipenv install
           pipenv install pytest-rerunfailures
           pipenv run python generate_smoke_tests.py
-          echo "PIPENV COMMAND: 'pipenv run pytest --experiment  ${{ inputs.slug }} --experiment-branch ${{ matrix.branch }} --experiment-server ${{ inputs.experiment-server }} --experiment-feature ${{ inputs.feature-name }} ${{ inputs.extra-arguments }} --reruns 1'"
-          pipenv run pytest --experiment ${{ inputs.slug }} --experiment-branch ${{ matrix.branch }} --experiment-server ${{ inputs.experiment-server }} --experiment-feature ${{ inputs.feature-name }} ${{ inputs.extra-arguments }} --reruns 1
+          echo "TEST COMMAND: pipenv run pytest --experiment-slug ${{ inputs.slug }} --experiment-branch ${{ matrix.branch }} --experiment-server ${{ inputs.experiment-server }} --experiment-feature ${{ inputs.feature-name }} --firefox-version ${{ matrix.firefox }} ${{ inputs.extra-arguments }} --reruns 1"
+          pipenv run pytest --experiment-slug ${{ inputs.slug }} --experiment-branch ${{ matrix.branch }} --experiment-server ${{ inputs.experiment-server }} --experiment-feature ${{ inputs.feature-name }} --firefox-version ${{ matrix.firefox }} ${{ inputs.extra-arguments }} --reruns 1
       - uses: actions/upload-artifact@v4
         if: ${{ always() }}
         with:

diff --git a/.github/workflows/ios_manual.yml b/.github/workflows/ios_manual.yml
@@ -101,8 +101,8 @@ jobs:
           xcrun simctl shutdown all
           export SIMULATOR_UDID=$(python get_specific_device_udid.py)
           poetry run python generate_smoke_tests.py
-          echo "TEST COMMAND: poetry run pytest --experiment ${{ inputs.slug }} --experiment-branch ${{ matrix.branch }} --experiment-server ${{ inputs.experiment-server }} --experiment-feature ${{ inputs.feature-name }} ${{ inputs.extra-arguments }} --reruns 1"
-          poetry run pytest --experiment ${{ inputs.slug }} --experiment-branch ${{ matrix.branch }} --experiment-server ${{ inputs.experiment-server }} --experiment-feature ${{ inputs.feature-name }} ${{ inputs.extra-arguments }} --reruns 1
+          echo "TEST COMMAND: poetry run pytest --experiment-slug ${{ inputs.slug }} --experiment-branch ${{ matrix.branch }} --experiment-server ${{ inputs.experiment-server }} --experiment-feature ${{ inputs.feature-name }} --firefox-version ${{ matrix.firefox }}  ${{ inputs.extra-arguments }} --reruns 1"
+          poetry run pytest --experiment-slug ${{ inputs.slug }} --experiment-branch ${{ matrix.branch }} --experiment-server ${{ inputs.experiment-server }} --experiment-feature ${{ inputs.feature-name }} --firefox-version ${{ matrix.firefox }}  ${{ inputs.extra-arguments }} --reruns 1
       - name: Archive Results
         if: ${{ always() }}
         run: zip -r results.zip /Users/runner/Library/Developer/Xcode/DerivedData/**/Logs/Test/*.xcresult

diff --git a/tests/android/conftest.py b/tests/android/conftest.py
@@ -11,6 +11,7 @@
 
 import pytest
 import requests
+from pytest_metadata.plugin import metadata_key
 
 from .gradlewbuild import GradlewBuild
 from .models.models import TelemetryModel
@@ -27,7 +28,6 @@ def pytest_addoption(parser):
         action="store",
         help="Feature name you want to test against",
     )
-    parser.addoption("--experiment", action="store", help="Feature name you want to test against")
     parser.addoption(
         "--experiment-branch",
         action="store",
@@ -40,6 +40,18 @@ def pytest_addoption(parser):
         default="prod",
         help="Experiment Server the experiment is hosted on",
     )
+    parser.addoption(
+        "--experiment-slug",
+        action="store",
+        default=None,
+        help="Experiment slug from Experimenter",
+    )
+    parser.addoption(
+        "--firefox-version",
+        action="store",
+        default=None,
+        help="The Firefox Version you are testing on. This is just use for reporting",
+    )
 
 
 def pytest_runtest_setup(item):
@@ -75,9 +87,25 @@ def fixture_nimbus_cli_args():
     return ""
 
 
-@pytest.fixture(name="experiment_branch")
+@pytest.fixture(name="experiment_branch", scope="session", autouse=True)
 def fixture_experiment_branch(request):
-    return request.config.getoption("--experiment-branch")
+    branch = request.config.getoption("--experiment-branch")
+    os.environ["EXPERIMENT_BRANCH"] = branch
+    return branch
+
+
+@pytest.fixture(name="experiment_slug", scope="session", autouse=True)
+def fixture_experiment_slug(request):
+    slug = request.config.getoption("--experiment-slug")
+    os.environ["EXPERIMENT_SLUG"] = slug
+    return slug
+
+
+@pytest.fixture(name="firefox_version", scope="session", autouse=True)
+def fixture_firefox_version(request):
+    ff_version = request.config.getoption("--firefox-version")
+    os.environ["FIREFOX_VERSION"] = ff_version
+    return ff_version
 
 
 @pytest.fixture(name="experiment_server")
@@ -157,11 +185,6 @@ def fixture_experiment_url(request, variables):
         pass
 
 
-@pytest.fixture(name="experiment_slug")
-def fixture_experiment_slug(request):
-    return request.config.getoption("--experiment")
-
-
 @pytest.fixture(name="ping_server", autouse=True, scope="session")
 def fixture_ping_server():
     process = start_process("ping_server", ["python", "ping_server.py"])
@@ -271,6 +294,20 @@ def fixture_set_experiment_test_name(experiment_data):
     os.environ["EXP_NAME"] = exp_name[random.randint(0, len(exp_name) - 1)]
 
 
+@pytest.hookimpl(tryfirst=True)
+def pytest_sessionfinish(session, exitstatus):
+    # Add data to html report
+    session.config.stash[metadata_key]["Experiment Slug"] = os.environ.get(
+        "EXPERIMENT_SLUG", "N/A"
+    )
+    session.config.stash[metadata_key]["Experiment Branch"] = os.environ.get(
+        "EXPERIMENT_BRANCH", "N/A"
+    )
+    session.config.stash[metadata_key]["Firefox Version"] = os.environ.get(
+        "FIREFOX_VERSION", "N/A"
+    )
+
+
 @pytest.fixture(name="setup_experiment")
 def fixture_setup_experiment(
     experiment_slug,

diff --git a/tests/conftest.py b/tests/conftest.py
@@ -15,6 +15,7 @@
 import pytest
 import requests
 from pytest_bdd import given, then
+from pytest_metadata.plugin import metadata_key
 from selenium.common.exceptions import JavascriptException
 from selenium.webdriver import ActionChains, Keys
 from selenium.webdriver.common.by import By
@@ -148,9 +149,18 @@ def fixture_enroll_experiment(
         logging.info("Experiment loaded successfully!")
 
 
-@pytest.fixture(name="experiment_slug")
+@pytest.fixture(name="experiment_slug", scope="session", autouse=True)
 def fixture_experiment_slug(request) -> typing.Any:
-    return request.config.getoption("--experiment-slug")
+    slug = request.config.getoption("--experiment-slug")
+    os.environ["EXPERIMENT_SLUG"] = slug
+    return slug
+
+
+@pytest.fixture(name="experiment_branch", scope="session", autouse=True)
+def fixture_experiment_branch(request):
+    branch = request.config.getoption("--experiment-branch")
+    os.environ["EXPERIMENT_BRANCH"] = branch
+    return branch
 
 
 @pytest.fixture
@@ -512,9 +522,24 @@ def fixture_firefox_version(selenium):
     version = selenium.execute_script(script)
     version = [item for item in version.split() if "Firefox" in item][0]
     logging.info(f"Firefox version {version}")
+    os.environ["FIREFOX_VERSION"] = version.split('/')[-1]
     return version
 
 
+@pytest.hookimpl(tryfirst=True)
+def pytest_sessionfinish(session, exitstatus):
+    # Add data to html report
+    session.config.stash[metadata_key]["Experiment Slug"] = os.environ.get(
+        "EXPERIMENT_SLUG", "N/A"
+    )
+    session.config.stash[metadata_key]["Experiment Branch"] = os.environ.get(
+        "EXPERIMENT_BRANCH", "N/A"
+    )
+    session.config.stash[metadata_key]["Firefox Version"] = os.environ.get(
+        "FIREFOX_VERSION", "N/A"
+    )
+
+
 @then("Firefox should be allowed to open a new tab")
 def open_a_new_tab(selenium):
     with selenium.context(selenium.CONTEXT_CHROME):

diff --git a/tests/ios/conftest.py b/tests/ios/conftest.py
@@ -10,6 +10,7 @@
 
 import pytest
 import requests
+from pytest_metadata.plugin import metadata_key
 
 from .models.models import TelemetryModel
 from .xcodebuild import XCodeBuild
@@ -22,7 +23,7 @@
 
 
 def pytest_addoption(parser):
-    parser.addoption("--experiment", action="store", help="The experiments experimenter URL")
+    parser.addoption("--experiment-slug", action="store", help="The experiments experimenter URL")
     parser.addoption("--stage", action="store_true", default=None, help="Use the stage server")
     parser.addoption(
         "--build-dev",
@@ -45,6 +46,12 @@ def pytest_addoption(parser):
         default="prod",
         help="Experiment Server the experiment is hosted on",
     )
+    parser.addoption(
+        "--firefox-version",
+        action="store",
+        default=None,
+        help="The Firefox Version you are testing on. This is just use for reporting",
+    )
 
 
 def pytest_runtest_setup(item):
@@ -59,9 +66,25 @@ def fixture_nimbus_cli_args():
     return "FIREFOX_SKIP_INTRO FIREFOX_TEST DISABLE_ANIMATIONS 'GCDWEBSERVER_PORT:7777'"
 
 
-@pytest.fixture(name="experiment_branch")
+@pytest.fixture(name="experiment_slug", scope="session", autouse=True)
+def fixture_experiment_slug(request):
+    slug = request.config.getoption("--experiment-slug")
+    os.environ["EXPERIMENT_SLUG"] = slug
+    return slug
+
+
+@pytest.fixture(name="experiment_branch", scope="session", autouse=True)
 def fixture_experiment_branch(request):
-    return request.config.getoption("--experiment-branch")
+    branch = request.config.getoption("--experiment-branch")
+    os.environ["EXPERIMENT_BRANCH"] = branch
+    return branch
+
+
+@pytest.fixture(name="firefox_version", scope="session", autouse=True)
+def fixture_firefox_version(request):
+    ff_version = request.config.getoption("--firefox-version")
+    os.environ["FIREFOX_VERSION"] = ff_version
+    return ff_version
 
 
 @pytest.fixture(name="experiment_server")
@@ -195,11 +218,6 @@ def fixture_experiment_url(request, variables):
         pass
 
 
-@pytest.fixture(name="experiment_slug")
-def fixture_experiment_slug(request):
-    return request.config.getoption("--experiment")
-
-
 @pytest.fixture(name="send_test_results", scope="session")
 def fixture_send_test_results(xcrun):
     yield
@@ -289,3 +307,17 @@ def _setup_experiment():
         run_nimbus_cli_command(" ".join(command))
 
     return _setup_experiment
+
+
+@pytest.hookimpl(tryfirst=True)
+def pytest_sessionfinish(session, exitstatus):
+    # Add data to html report
+    session.config.stash[metadata_key]["Experiment Slug"] = os.environ.get(
+        "EXPERIMENT_SLUG", "N/A"
+    )
+    session.config.stash[metadata_key]["Experiment Branch"] = os.environ.get(
+        "EXPERIMENT_BRANCH", "N/A"
+    )
+    session.config.stash[metadata_key]["Firefox Version"] = os.environ.get(
+        "FIREFOX_VERSION", "N/A"
+    )