From 32247e3276007e1bfa51feae6db707f3c3568605 Mon Sep 17 00:00:00 2001 From: Andrei Matveyeu Date: Wed, 10 Apr 2024 07:41:14 +0200 Subject: [PATCH 01/10] Add OpenTelemetry instrumentation --- Dockerfile | 12 ++- projects/etos_suite_runner/requirements.txt | 8 ++ projects/etos_suite_runner/setup.cfg | 7 +- .../src/etos_suite_runner/__init__.py | 29 ++++++ .../src/etos_suite_runner/__main__.py | 3 +- .../src/etos_suite_runner/esr.py | 69 +++++++------- .../src/etos_suite_runner/lib/executor.py | 32 ++++--- .../src/etos_suite_runner/lib/runner.py | 18 ++-- .../src/etos_suite_runner/lib/suite.py | 90 +++++++++++-------- .../src/etos_suite_runner/otel_tracing.py | 34 +++++++ 10 files changed, 210 insertions(+), 92 deletions(-) create mode 100644 projects/etos_suite_runner/src/etos_suite_runner/otel_tracing.py diff --git a/Dockerfile b/Dockerfile index ea9067a..90c9f9d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,20 @@ -FROM python:3.9.0-buster AS build +FROM python:3.9-buster AS build COPY . /src WORKDIR /src/projects/etos_suite_runner RUN python3 setup.py bdist_wheel -FROM python:3.9.0-slim-buster +FROM python:3.9-slim-buster COPY --from=build /src/projects/etos_suite_runner/dist/*.whl /tmp # hadolint ignore=DL3013 -RUN pip install --no-cache-dir /tmp/*.whl && groupadd -r etos && useradd -r -m -s /bin/false -g etos etos +RUN apt-get update && \ + apt-get install -y gcc libc-dev --no-install-recommends && \ + pip install --no-cache-dir /tmp/*.whl && \ + apt-get purge -y --auto-remove gcc libc-dev && \ + rm -rf /var/lib/apt/lists/* && \ + groupadd -r etos && useradd -r -m -s /bin/false -g etos etos + USER etos LABEL org.opencontainers.image.source=https://github.com/eiffel-community/etos-suite-runner diff --git a/projects/etos_suite_runner/requirements.txt b/projects/etos_suite_runner/requirements.txt index 6ea27e2..9926fc8 100644 --- a/projects/etos_suite_runner/requirements.txt +++ b/projects/etos_suite_runner/requirements.txt @@ -18,5 +18,13 @@ PyScaffold==3.2.3 packageurl-python~=0.11 cryptography>=42.0.4,<43.0.0 +<<<<<<< HEAD etos_lib==4.0.0 etos_environment_provider~=4.1 +======= +etos_lib==4.1.1 +#etos_environment_provider~=3.2 +opentelemetry-api~=1.21 +opentelemetry-exporter-otlp~=1.21 +opentelemetry-sdk~=1.21 +>>>>>>> e187c17 (Add OpenTelemetry instrumentation) diff --git a/projects/etos_suite_runner/setup.cfg b/projects/etos_suite_runner/setup.cfg index c0fb1ba..ed6d447 100644 --- a/projects/etos_suite_runner/setup.cfg +++ b/projects/etos_suite_runner/setup.cfg @@ -28,8 +28,11 @@ install_requires = PyScaffold==3.2.3 packageurl-python~=0.11 cryptography>=42.0.4,<43.0.0 - etos_lib==4.0.0 - etos_environment_provider~=4.1 + etos_lib==4.1.1 + #etos_environment_provider~=4.0 + opentelemetry-api~=1.21 + opentelemetry-exporter-otlp~=1.21 + opentelemetry-sdk~=1.21 python_requires = >=3.4 diff --git a/projects/etos_suite_runner/src/etos_suite_runner/__init__.py b/projects/etos_suite_runner/src/etos_suite_runner/__init__.py index 3fae8f6..dfe8232 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/__init__.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/__init__.py @@ -14,9 +14,16 @@ # See the License for the specific language governing permissions and # limitations under the License. """ETOS suite runner module.""" +import logging import os from importlib.metadata import PackageNotFoundError, version +from opentelemetry import trace +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk.resources import SERVICE_NAME, SERVICE_NAMESPACE, SERVICE_VERSION, Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor + from etos_lib.logging.logger import setup_logging try: @@ -30,3 +37,25 @@ ENVIRONMENT = "development" if DEV else "production" os.environ["ENVIRONMENT_PROVIDER_DISABLE_LOGGING"] = "true" setup_logging("ETOS Suite Runner", VERSION, ENVIRONMENT) + + +LOGGER = logging.getLogger(__name__) + +# Setting OTEL_COLLECTOR_HOST will override the default OTEL collector endpoint. +# This is needed when using the centralized cluster-level OTEL collector instead of sidecar collector. +if os.getenv("OTEL_COLLECTOR_HOST"): + os.environ["OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"] = os.getenv("OTEL_COLLECTOR_HOST") + LOGGER.info("Using OTEL collector: %s", os.getenv("OTEL_COLLECTOR_HOST")) + +if os.getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"): + PROVIDER = TracerProvider( + resource=Resource.create( + {SERVICE_NAME: "etos-suite-runner", SERVICE_VERSION: VERSION, SERVICE_NAMESPACE: ENVIRONMENT} + ) + ) + EXPORTER = OTLPSpanExporter() + PROCESSOR = BatchSpanProcessor(EXPORTER) + PROVIDER.add_span_processor(PROCESSOR) + trace.set_tracer_provider(PROVIDER) +else: + LOGGER.info("Suite runner OTEL_EXPORTER_OTLP_TRACES_ENDPOINT not set!") \ No newline at end of file diff --git a/projects/etos_suite_runner/src/etos_suite_runner/__main__.py b/projects/etos_suite_runner/src/etos_suite_runner/__main__.py index 2891532..42478c5 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/__main__.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/__main__.py @@ -21,6 +21,7 @@ from .esr import ESR + LOGGER = logging.getLogger(__name__) @@ -38,11 +39,9 @@ def main(): esr.etos.publisher.stop() LOGGER.info("ESR Finished Executing.", extra={"user_log": True}) - def run(): """Entry point for console_scripts.""" main() - if __name__ == "__main__": run() diff --git a/projects/etos_suite_runner/src/etos_suite_runner/esr.py b/projects/etos_suite_runner/src/etos_suite_runner/esr.py index 212e2e9..611278f 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/esr.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/esr.py @@ -15,6 +15,7 @@ # limitations under the License. # -*- coding: utf-8 -*- """ETOS suite runner module.""" +import json import logging import os import signal @@ -28,14 +29,17 @@ from etos_lib import ETOS from etos_lib.logging.logger import FORMAT_CONFIG from jsontas.jsontas import JsonTas +import opentelemetry from .lib.esr_parameters import ESRParameters from .lib.exceptions import EnvironmentProviderException from .lib.runner import SuiteRunner +from .otel_tracing import get_current_context # Remove spam from pika. logging.getLogger("pika").setLevel(logging.WARNING) +SUBSUITE_CONTEXT = None class ESR: # pylint:disable=too-many-instance-attributes """Suite runner for ETOS main program. @@ -49,6 +53,7 @@ class ESR: # pylint:disable=too-many-instance-attributes def __init__(self) -> None: """Initialize ESR by creating a rabbitmq publisher.""" self.logger = logging.getLogger("ESR") + self.otel_tracer = opentelemetry.trace.get_tracer(__name__) self.etos = ETOS("ETOS Suite Runner", os.getenv("SOURCE_HOST"), "ETOS Suite Runner") signal.signal(signal.SIGTERM, self.graceful_exit) self.params = ESRParameters(self.etos) @@ -67,28 +72,32 @@ def _request_environment(self, ids: list[str]) -> None: :param ids: Generated suite runner IDs used to correlate environments and the suite runners. """ - try: - provider = EnvironmentProvider(self.params.tercc.meta.event_id, ids, copy=False) - result = provider.run() - except Exception: - self.params.set_status("FAILURE", "Failed to run environment provider") - self.logger.error( - "Environment provider has failed in creating an environment for test.", - extra={"user_log": True}, - ) - raise - if result.get("error") is not None: - self.params.set_status("FAILURE", result.get("error")) - self.logger.error( - "Environment provider has failed in creating an environment for test.", - extra={"user_log": True}, - ) - else: - self.params.set_status("SUCCESS", result.get("error")) - self.logger.info( - "Environment provider has finished creating an environment for test.", - extra={"user_log": True}, - ) + span_name = "request_environment" + suite_context = get_current_context() + with self.otel_tracer.start_as_current_span(span_name, context=suite_context) as span: + try: + provider = EnvironmentProvider(self.params.tercc.meta.event_id, ids, copy=False) + result = provider.run() + except Exception as exc: + self.params.set_status("FAILURE", "Failed to run environment provider") + self.logger.error( + "Environment provider has failed in creating an environment for test.", + extra={"user_log": True}, + ) + span.set_attribute("result", traceback.format_exc()) + raise + if result.get("error") is not None: + self.params.set_status("FAILURE", result.get("error")) + self.logger.error( + "Environment provider has failed in creating an environment for test.", + extra={"user_log": True}, + ) + else: + self.params.set_status("SUCCESS", result.get("error")) + self.logger.info( + "Environment provider has finished creating an environment for test.", + extra={"user_log": True}, + ) def _release_environment(self) -> None: """Release an environment from the environment provider.""" @@ -96,11 +105,14 @@ def _release_environment(self) -> None: # Passing variables as keyword argument to make it easier to transition to a function where # jsontas is not required. jsontas = JsonTas() - status, message = release_full_environment( - etos=self.etos, jsontas=jsontas, suite_id=self.params.tercc.meta.event_id - ) - if not status: - self.logger.error(message) + span_name = "release_full_environment" + suite_context = get_current_context() + with self.otel_tracer.start_as_current_span(span_name, context=suite_context) as span: + status, message = release_full_environment( + etos=self.etos, jsontas=jsontas, suite_id=self.params.tercc.meta.event_id + ) + if not status: + self.logger.error(message) def run_suites(self, triggered: EiffelActivityTriggeredEvent) -> list[str]: """Start up a suite runner handling multiple suites that execute within test runners. @@ -117,13 +129,11 @@ def run_suites(self, triggered: EiffelActivityTriggeredEvent) -> list[str]: "ESR Docker", {"CONTEXT": context}, image=os.getenv("SUITE_RUNNER") ) runner = SuiteRunner(self.params, self.etos) - ids = [] for suite in self.params.test_suite: suite["test_suite_started_id"] = str(uuid4()) ids.append(suite["test_suite_started_id"]) self.logger.info("Number of test suites to run: %d", len(ids), extra={"user_log": True}) - try: self.logger.info("Get test environment.") threading.Thread( @@ -176,7 +186,6 @@ def run(self) -> list[str]: executionType="AUTOMATED", triggers=[{"type": "EIFFEL_EVENT"}], ) - self.verify_input() context = triggered.meta.event_id except: # noqa diff --git a/projects/etos_suite_runner/src/etos_suite_runner/lib/executor.py b/projects/etos_suite_runner/src/etos_suite_runner/lib/executor.py index 04bb38e..3cfdc0e 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/lib/executor.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/lib/executor.py @@ -16,15 +16,17 @@ """Executor handler module.""" import logging import os -from json import JSONDecodeError +from json import JSONDecodeError, dumps from typing import Union from cryptography.fernet import Fernet from etos_lib import ETOS +from opentelemetry import trace from requests.auth import HTTPBasicAuth, HTTPDigestAuth from requests.exceptions import ConnectionError as RequestsConnectionError from requests.exceptions import HTTPError +from ..otel_tracing import get_current_context class TestStartException(Exception): """Exception when starting tests.""" @@ -47,6 +49,7 @@ def __init__(self, etos: ETOS) -> None: """ self.etos = etos self.etos.config.set("build_urls", []) + self.tracer = trace.get_tracer(__name__) def __decrypt(self, password: Union[str, dict]) -> str: """Decrypt a password using an encryption key. @@ -89,15 +92,20 @@ def run_tests(self, test_suite: dict) -> None: if request.get("auth"): request["auth"] = self.__auth(**request["auth"]) method = getattr(self.etos.http, request.pop("method").lower()) - try: - response = method(**request) - response.raise_for_status() - except HTTPError as http_error: + span_name = "start_execution_space" + with self.tracer.start_as_current_span(span_name) as span: + span.set_attribute("executor_id", executor['id']) + span.set_attribute("request", dumps(request, indent=4)) try: - raise TestStartException(http_error.response.json()) from http_error - except JSONDecodeError: - raise TestStartException({"error": http_error.response.text}) from http_error - except RequestsConnectionError as connection_error: - raise TestStartException({"error": str(connection_error)}) from connection_error - self.logger.info("%r", response) - self.logger.debug("%r", response.text) + response = method(**request) + response.raise_for_status() + except HTTPError as http_error: + span.set_attribute("http_error", str(http_error)) + try: + raise TestStartException(http_error.response.json()) from http_error + except JSONDecodeError: + raise TestStartException({"error": http_error.response.text}) from http_error + except RequestsConnectionError as connection_error: + raise TestStartException({"error": str(connection_error)}) from connection_error + self.logger.info("%r", response) + self.logger.debug("%r", response.text) diff --git a/projects/etos_suite_runner/src/etos_suite_runner/lib/runner.py b/projects/etos_suite_runner/src/etos_suite_runner/lib/runner.py index fd694c3..3093bd5 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/lib/runner.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/lib/runner.py @@ -20,8 +20,10 @@ from environment_provider.environment import release_full_environment from etos_lib.logging.logger import FORMAT_CONFIG from jsontas.jsontas import JsonTas +import opentelemetry from .exceptions import EnvironmentProviderException +from ..otel_tracing import get_current_context from .suite import TestSuite @@ -44,6 +46,8 @@ def __init__(self, params, etos): """ self.params = params self.etos = etos + self.otel_tracer = opentelemetry.trace.get_tracer(__name__) + self.otel_suite_context = get_current_context() def _release_environment(self): """Release an environment from the environment provider.""" @@ -51,17 +55,19 @@ def _release_environment(self): # Passing variables as keyword argument to make it easier to transition to a function where # jsontas is not required. jsontas = JsonTas() - status, message = release_full_environment( - etos=self.etos, jsontas=jsontas, suite_id=self.params.tercc.meta.event_id - ) - if not status: - self.logger.error(message) + span_name = "release_full_environment" + with self.otel_tracer.start_as_current_span(span_name, context=self.otel_suite_context): + status, message = release_full_environment( + etos=self.etos, jsontas=jsontas, suite_id=self.params.tercc.meta.event_id + ) + if not status: + self.logger.error(message) def start_suites_and_wait(self): """Get environments and start all test suites.""" try: test_suites = [ - TestSuite(self.etos, self.params, suite) for suite in self.params.test_suite + TestSuite(self.etos, self.params, suite, otel_context=self.otel_suite_context) for suite in self.params.test_suite ] with ThreadPool() as pool: pool.map(self.run, test_suites) diff --git a/projects/etos_suite_runner/src/etos_suite_runner/lib/suite.py b/projects/etos_suite_runner/src/etos_suite_runner/lib/suite.py index 782066c..8d9b468 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/lib/suite.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/lib/suite.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Test suite handler.""" +import json import logging import threading import time @@ -25,6 +26,7 @@ from etos_lib import ETOS from etos_lib.logging.logger import FORMAT_CONFIG from jsontas.jsontas import JsonTas +import opentelemetry from .esr_parameters import ESRParameters from .exceptions import EnvironmentProviderException @@ -37,6 +39,7 @@ request_test_suite_started, ) from .log_filter import DuplicateFilter +from ..otel_tracing import get_current_context class SubSuite: # pylint:disable=too-many-instance-attributes @@ -52,6 +55,7 @@ def __init__(self, etos: ETOS, environment: dict, main_suite_id: str) -> None: self.main_suite_id = main_suite_id self.logger = logging.getLogger(f"SubSuite - {self.environment.get('name')}") self.logger.addFilter(DuplicateFilter(self.logger)) + self.otel_tracer = opentelemetry.trace.get_tracer(__name__) self.test_suite_started = {} self.test_suite_finished = {} @@ -93,36 +97,41 @@ def outcome(self) -> dict: return self.test_suite_finished.get("data", {}).get("testSuiteOutcome", {}) return {} - def start(self, identifier: str) -> None: + def start(self, identifier: str, otel_context: opentelemetry.context.context.Context) -> None: """Start ETR for this sub suite. :param identifier: An identifier for logs in this sub suite. """ - FORMAT_CONFIG.identifier = identifier - self.logger.info("Starting up the ETOS test runner", extra={"user_log": True}) - executor = Executor(self.etos) - try: - executor.run_tests(self.environment) - except TestStartException as exception: - self.failed = True - self.logger.error( - "Failed to start sub suite: %s", exception.error, extra={"user_log": True} - ) - raise - self.logger.info("ETR triggered.") - timeout = time.time() + self.etos.debug.default_test_result_timeout - try: - while time.time() < timeout: - time.sleep(10) - if not self.started: - continue - self.logger.info("ETOS test runner has started", extra={"user_log": True}) - self.request_finished_event() - if self.finished: - self.logger.info("ETOS test runner has finished", extra={"user_log": True}) - break - finally: - self.release(identifier) + # OpenTelemetry context needs to be retrieved here, since the subsuite is running in a separate process + span_name = "execute_testrunner" + with self.otel_tracer.start_as_current_span(span_name, context=otel_context) as span: + span.set_attribute("subsuite_id", identifier) + FORMAT_CONFIG.identifier = identifier + self.logger.info("Starting up the ETOS test runner", extra={"user_log": True}) + executor = Executor(self.etos) + try: + executor.run_tests(self.environment) + except TestStartException as exception: + self.failed = True + self.logger.error( + "Failed to start sub suite: %s", exception.error, extra={"user_log": True} + ) + span.set_attribute("failed", str(exception)) + raise + self.logger.info("ETR triggered.") + timeout = time.time() + self.etos.debug.default_test_result_timeout + try: + while time.time() < timeout: + time.sleep(10) + if not self.started: + continue + self.logger.info("ETOS test runner has started", extra={"user_log": True}) + self.request_finished_event() + if self.finished: + self.logger.info("ETOS test runner has finished", extra={"user_log": True}) + break + finally: + self.release(identifier) def release(self, testrun_id) -> None: """Release this sub suite.""" @@ -136,16 +145,22 @@ def release(self, testrun_id) -> None: registry = ProviderRegistry(etos=self.etos, jsontas=jsontas, suite_id=testrun_id) self.logger.info(self.environment) - success = release_environment( - etos=self.etos, jsontas=jsontas, provider_registry=registry, sub_suite=self.environment - ) - if not success: - self.logger.exception( - "Failed to check in %r", self.environment["id"], extra={"user_log": True} + + span_name = "release_environment" + with self.otel_tracer.start_as_current_span(span_name) as span: + failure = release_environment( + etos=self.etos, jsontas=jsontas, provider_registry=registry, sub_suite=self.environment ) - return - self.logger.info("Checked in %r", self.environment["id"], extra={"user_log": True}) - self.released = True + span.set_attribute("testrun_id", testrun_id) + span.set_attribute("failure", str(failure)) + span.set_attribute("environment", json.dumps(self.environment, indent=4)) + if failure is not None: + self.logger.exception( + "Failed to check in %r", self.environment["id"], extra={"user_log": True} + ) + return + self.logger.info("Checked in %r", self.environment["id"], extra={"user_log": True}) + self.released = True class TestSuite: # pylint:disable=too-many-instance-attributes @@ -157,7 +172,7 @@ class TestSuite: # pylint:disable=too-many-instance-attributes __activity_triggered = None __activity_finished = None - def __init__(self, etos: ETOS, params: ESRParameters, suite: dict) -> None: + def __init__(self, etos: ETOS, params: ESRParameters, suite: dict, otel_context: opentelemetry.context.context.Context = None) -> None: """Initialize a TestSuite instance.""" self.etos = etos self.params = params @@ -165,6 +180,7 @@ def __init__(self, etos: ETOS, params: ESRParameters, suite: dict) -> None: self.logger = logging.getLogger(f"TestSuite - {self.suite.get('name')}") self.logger.addFilter(DuplicateFilter(self.logger)) self.sub_suites = [] + self.otel_context = otel_context @property def sub_suite_environments(self) -> Iterator[dict]: @@ -324,7 +340,7 @@ def start(self) -> None: ) self.sub_suites.append(sub_suite) thread = threading.Thread( - target=sub_suite.start, args=(self.params.tercc.meta.event_id,) + target=sub_suite.start, args=(self.params.tercc.meta.event_id, self.otel_context) ) threads.append(thread) thread.start() diff --git a/projects/etos_suite_runner/src/etos_suite_runner/otel_tracing.py b/projects/etos_suite_runner/src/etos_suite_runner/otel_tracing.py new file mode 100644 index 0000000..4c0cbd2 --- /dev/null +++ b/projects/etos_suite_runner/src/etos_suite_runner/otel_tracing.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python +# Copyright Axis Communications AB. +# +# For a full list of individual contributors, please see the commit history. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# -*- coding: utf-8 -*- +import logging +import os +import opentelemetry + +LOGGER = logging.getLogger(__name__) + +def get_current_context() -> opentelemetry.context.context.Context: + """Get current context (propagated via environment variable OTEL_CONTEXT).""" + carrier = {} + LOGGER.info("Current OpenTelemetry context env: %s", os.environ.get("OTEL_CONTEXT")) + for kv in os.environ.get("OTEL_CONTEXT", "").split(","): + if kv: + k, v = kv.split("=", 1) + carrier[k] = v + ctx = opentelemetry.propagate.extract(carrier) + LOGGER.info("Current OpenTelemetry context %s", ctx) + return ctx From 2390b2dc1eecdfd88d8ae951d71d6a672bd81585 Mon Sep 17 00:00:00 2001 From: Andrei Matveyeu Date: Mon, 22 Apr 2024 12:11:19 +0200 Subject: [PATCH 02/10] Minor fixes --- .../src/etos_suite_runner/__init__.py | 8 ++++++-- .../src/etos_suite_runner/__main__.py | 2 ++ .../src/etos_suite_runner/esr.py | 5 +++-- .../src/etos_suite_runner/lib/executor.py | 3 +-- .../src/etos_suite_runner/lib/runner.py | 3 ++- .../src/etos_suite_runner/lib/suite.py | 16 ++++++++++++---- .../src/etos_suite_runner/otel_tracing.py | 1 + 7 files changed, 27 insertions(+), 11 deletions(-) diff --git a/projects/etos_suite_runner/src/etos_suite_runner/__init__.py b/projects/etos_suite_runner/src/etos_suite_runner/__init__.py index dfe8232..75d468b 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/__init__.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/__init__.py @@ -50,7 +50,11 @@ if os.getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"): PROVIDER = TracerProvider( resource=Resource.create( - {SERVICE_NAME: "etos-suite-runner", SERVICE_VERSION: VERSION, SERVICE_NAMESPACE: ENVIRONMENT} + { + SERVICE_NAME: "etos-suite-runner", + SERVICE_VERSION: VERSION, + SERVICE_NAMESPACE: ENVIRONMENT, + } ) ) EXPORTER = OTLPSpanExporter() @@ -58,4 +62,4 @@ PROVIDER.add_span_processor(PROCESSOR) trace.set_tracer_provider(PROVIDER) else: - LOGGER.info("Suite runner OTEL_EXPORTER_OTLP_TRACES_ENDPOINT not set!") \ No newline at end of file + LOGGER.info("Suite runner OTEL_EXPORTER_OTLP_TRACES_ENDPOINT not set!") diff --git a/projects/etos_suite_runner/src/etos_suite_runner/__main__.py b/projects/etos_suite_runner/src/etos_suite_runner/__main__.py index 42478c5..5c4e7b7 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/__main__.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/__main__.py @@ -39,9 +39,11 @@ def main(): esr.etos.publisher.stop() LOGGER.info("ESR Finished Executing.", extra={"user_log": True}) + def run(): """Entry point for console_scripts.""" main() + if __name__ == "__main__": run() diff --git a/projects/etos_suite_runner/src/etos_suite_runner/esr.py b/projects/etos_suite_runner/src/etos_suite_runner/esr.py index 611278f..1e348d3 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/esr.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/esr.py @@ -41,6 +41,7 @@ SUBSUITE_CONTEXT = None + class ESR: # pylint:disable=too-many-instance-attributes """Suite runner for ETOS main program. @@ -78,7 +79,7 @@ def _request_environment(self, ids: list[str]) -> None: try: provider = EnvironmentProvider(self.params.tercc.meta.event_id, ids, copy=False) result = provider.run() - except Exception as exc: + except Exception: self.params.set_status("FAILURE", "Failed to run environment provider") self.logger.error( "Environment provider has failed in creating an environment for test.", @@ -107,7 +108,7 @@ def _release_environment(self) -> None: jsontas = JsonTas() span_name = "release_full_environment" suite_context = get_current_context() - with self.otel_tracer.start_as_current_span(span_name, context=suite_context) as span: + with self.otel_tracer.start_as_current_span(span_name, context=suite_context): status, message = release_full_environment( etos=self.etos, jsontas=jsontas, suite_id=self.params.tercc.meta.event_id ) diff --git a/projects/etos_suite_runner/src/etos_suite_runner/lib/executor.py b/projects/etos_suite_runner/src/etos_suite_runner/lib/executor.py index 3cfdc0e..e0dc87b 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/lib/executor.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/lib/executor.py @@ -26,7 +26,6 @@ from requests.exceptions import ConnectionError as RequestsConnectionError from requests.exceptions import HTTPError -from ..otel_tracing import get_current_context class TestStartException(Exception): """Exception when starting tests.""" @@ -94,7 +93,7 @@ def run_tests(self, test_suite: dict) -> None: method = getattr(self.etos.http, request.pop("method").lower()) span_name = "start_execution_space" with self.tracer.start_as_current_span(span_name) as span: - span.set_attribute("executor_id", executor['id']) + span.set_attribute("executor_id", executor["id"]) span.set_attribute("request", dumps(request, indent=4)) try: response = method(**request) diff --git a/projects/etos_suite_runner/src/etos_suite_runner/lib/runner.py b/projects/etos_suite_runner/src/etos_suite_runner/lib/runner.py index 3093bd5..ac4767f 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/lib/runner.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/lib/runner.py @@ -67,7 +67,8 @@ def start_suites_and_wait(self): """Get environments and start all test suites.""" try: test_suites = [ - TestSuite(self.etos, self.params, suite, otel_context=self.otel_suite_context) for suite in self.params.test_suite + TestSuite(self.etos, self.params, suite, otel_context=self.otel_suite_context) + for suite in self.params.test_suite ] with ThreadPool() as pool: pool.map(self.run, test_suites) diff --git a/projects/etos_suite_runner/src/etos_suite_runner/lib/suite.py b/projects/etos_suite_runner/src/etos_suite_runner/lib/suite.py index 8d9b468..342ece9 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/lib/suite.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/lib/suite.py @@ -39,7 +39,6 @@ request_test_suite_started, ) from .log_filter import DuplicateFilter -from ..otel_tracing import get_current_context class SubSuite: # pylint:disable=too-many-instance-attributes @@ -149,7 +148,10 @@ def release(self, testrun_id) -> None: span_name = "release_environment" with self.otel_tracer.start_as_current_span(span_name) as span: failure = release_environment( - etos=self.etos, jsontas=jsontas, provider_registry=registry, sub_suite=self.environment + etos=self.etos, + jsontas=jsontas, + provider_registry=registry, + sub_suite=self.environment, ) span.set_attribute("testrun_id", testrun_id) span.set_attribute("failure", str(failure)) @@ -172,7 +174,12 @@ class TestSuite: # pylint:disable=too-many-instance-attributes __activity_triggered = None __activity_finished = None - def __init__(self, etos: ETOS, params: ESRParameters, suite: dict, otel_context: opentelemetry.context.context.Context = None) -> None: + def __init__(self, + etos: ETOS, + params: ESRParameters, + suite: dict, + otel_context: opentelemetry.context.context.Context = None + ) -> None: """Initialize a TestSuite instance.""" self.etos = etos self.params = params @@ -340,7 +347,8 @@ def start(self) -> None: ) self.sub_suites.append(sub_suite) thread = threading.Thread( - target=sub_suite.start, args=(self.params.tercc.meta.event_id, self.otel_context) + target=sub_suite.start, + args=(self.params.tercc.meta.event_id, self.otel_context), ) threads.append(thread) thread.start() diff --git a/projects/etos_suite_runner/src/etos_suite_runner/otel_tracing.py b/projects/etos_suite_runner/src/etos_suite_runner/otel_tracing.py index 4c0cbd2..c42b3cf 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/otel_tracing.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/otel_tracing.py @@ -21,6 +21,7 @@ LOGGER = logging.getLogger(__name__) + def get_current_context() -> opentelemetry.context.context.Context: """Get current context (propagated via environment variable OTEL_CONTEXT).""" carrier = {} From 6bc6cf5b904efc5709ec621a3356dee52a946024 Mon Sep 17 00:00:00 2001 From: Andrei Matveyeu Date: Mon, 22 Apr 2024 12:16:50 +0200 Subject: [PATCH 03/10] Minor fixes --- .../src/etos_suite_runner/__init__.py | 2 +- .../etos_suite_runner/src/etos_suite_runner/esr.py | 1 - .../src/etos_suite_runner/lib/suite.py | 11 ++++++----- .../src/etos_suite_runner/otel_tracing.py | 1 + 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/projects/etos_suite_runner/src/etos_suite_runner/__init__.py b/projects/etos_suite_runner/src/etos_suite_runner/__init__.py index 75d468b..7355620 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/__init__.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/__init__.py @@ -42,7 +42,7 @@ LOGGER = logging.getLogger(__name__) # Setting OTEL_COLLECTOR_HOST will override the default OTEL collector endpoint. -# This is needed when using the centralized cluster-level OTEL collector instead of sidecar collector. +# This is needed when using the cluster-level OTEL collector instead of sidecar collector. if os.getenv("OTEL_COLLECTOR_HOST"): os.environ["OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"] = os.getenv("OTEL_COLLECTOR_HOST") LOGGER.info("Using OTEL collector: %s", os.getenv("OTEL_COLLECTOR_HOST")) diff --git a/projects/etos_suite_runner/src/etos_suite_runner/esr.py b/projects/etos_suite_runner/src/etos_suite_runner/esr.py index 1e348d3..7bfe978 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/esr.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/esr.py @@ -15,7 +15,6 @@ # limitations under the License. # -*- coding: utf-8 -*- """ETOS suite runner module.""" -import json import logging import os import signal diff --git a/projects/etos_suite_runner/src/etos_suite_runner/lib/suite.py b/projects/etos_suite_runner/src/etos_suite_runner/lib/suite.py index 342ece9..daa2968 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/lib/suite.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/lib/suite.py @@ -101,7 +101,8 @@ def start(self, identifier: str, otel_context: opentelemetry.context.context.Con :param identifier: An identifier for logs in this sub suite. """ - # OpenTelemetry context needs to be retrieved here, since the subsuite is running in a separate process + # OpenTelemetry context needs to be retrieved here: + # the subsuite is running in a separate process span_name = "execute_testrunner" with self.otel_tracer.start_as_current_span(span_name, context=otel_context) as span: span.set_attribute("subsuite_id", identifier) @@ -175,10 +176,10 @@ class TestSuite: # pylint:disable=too-many-instance-attributes __activity_finished = None def __init__(self, - etos: ETOS, - params: ESRParameters, - suite: dict, - otel_context: opentelemetry.context.context.Context = None + etos: ETOS, + params: ESRParameters, + suite: dict, + otel_context: opentelemetry.context.context.Context = None ) -> None: """Initialize a TestSuite instance.""" self.etos = etos diff --git a/projects/etos_suite_runner/src/etos_suite_runner/otel_tracing.py b/projects/etos_suite_runner/src/etos_suite_runner/otel_tracing.py index c42b3cf..c311340 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/otel_tracing.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/otel_tracing.py @@ -15,6 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -*- coding: utf-8 -*- +"""OpenTelemetry-related code.""" import logging import os import opentelemetry From 5d37536de762d74800d201b3fa13d52730c0969e Mon Sep 17 00:00:00 2001 From: Andrei Matveyeu Date: Mon, 22 Apr 2024 12:26:59 +0200 Subject: [PATCH 04/10] Minor fix --- .../etos_suite_runner/src/etos_suite_runner/lib/suite.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/projects/etos_suite_runner/src/etos_suite_runner/lib/suite.py b/projects/etos_suite_runner/src/etos_suite_runner/lib/suite.py index daa2968..edadaa1 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/lib/suite.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/lib/suite.py @@ -175,11 +175,12 @@ class TestSuite: # pylint:disable=too-many-instance-attributes __activity_triggered = None __activity_finished = None - def __init__(self, + def __init__( + self, etos: ETOS, params: ESRParameters, suite: dict, - otel_context: opentelemetry.context.context.Context = None + otel_context: opentelemetry.context.context.Context = None, ) -> None: """Initialize a TestSuite instance.""" self.etos = etos From 6282ae4e179b90f76e360bf81a5fd943a16e58ee Mon Sep 17 00:00:00 2001 From: Andrei Matveyeu Date: Mon, 22 Apr 2024 12:29:22 +0200 Subject: [PATCH 05/10] Minor fixes --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 90c9f9d..a2ed795 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,6 +7,7 @@ RUN python3 setup.py bdist_wheel FROM python:3.9-slim-buster COPY --from=build /src/projects/etos_suite_runner/dist/*.whl /tmp +# hadolint ignore=DL3008 # hadolint ignore=DL3013 RUN apt-get update && \ apt-get install -y gcc libc-dev --no-install-recommends && \ From 8f4bcfcc25156cc3e08cb89f6f5b406082b704b2 Mon Sep 17 00:00:00 2001 From: Andrei Matveyeu Date: Mon, 22 Apr 2024 12:33:21 +0200 Subject: [PATCH 06/10] Minor fixes --- Dockerfile | 15 +++++----- projects/etos_suite_runner/setup.cfg | 2 +- .../src/etos_suite_runner/esr.py | 10 ++++--- .../src/etos_suite_runner/lib/executor.py | 7 +++-- .../src/etos_suite_runner/lib/suite.py | 13 ++++---- .../src/etos_suite_runner/otel_tracing.py | 30 +++++++++++++++++++ 6 files changed, 57 insertions(+), 20 deletions(-) diff --git a/Dockerfile b/Dockerfile index a2ed795..8356e5a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,14 +7,15 @@ RUN python3 setup.py bdist_wheel FROM python:3.9-slim-buster COPY --from=build /src/projects/etos_suite_runner/dist/*.whl /tmp -# hadolint ignore=DL3008 # hadolint ignore=DL3013 -RUN apt-get update && \ - apt-get install -y gcc libc-dev --no-install-recommends && \ - pip install --no-cache-dir /tmp/*.whl && \ - apt-get purge -y --auto-remove gcc libc-dev && \ - rm -rf /var/lib/apt/lists/* && \ - groupadd -r etos && useradd -r -m -s /bin/false -g etos etos +# hadolint ignore=DL3008 +#RUN apt-get update && \ +# apt-get install -y gcc libc-dev --no-install-recommends && \ +# pip install --no-cache-dir /tmp/*.whl && \ +# apt-get purge -y --auto-remove gcc libc-dev && \ +# rm -rf /var/lib/apt/lists/* && \ +# groupadd -r etos && useradd -r -m -s /bin/false -g etos etos +RUN pip install --no-cache-dir /tmp/*.whl && groupadd -r etos && useradd -r -m -s /bin/false -g etos etos USER etos diff --git a/projects/etos_suite_runner/setup.cfg b/projects/etos_suite_runner/setup.cfg index ed6d447..9ae1d33 100644 --- a/projects/etos_suite_runner/setup.cfg +++ b/projects/etos_suite_runner/setup.cfg @@ -28,7 +28,7 @@ install_requires = PyScaffold==3.2.3 packageurl-python~=0.11 cryptography>=42.0.4,<43.0.0 - etos_lib==4.1.1 + #etos_lib==4.1.1 #etos_environment_provider~=4.0 opentelemetry-api~=1.21 opentelemetry-exporter-otlp~=1.21 diff --git a/projects/etos_suite_runner/src/etos_suite_runner/esr.py b/projects/etos_suite_runner/src/etos_suite_runner/esr.py index 7bfe978..5621f8f 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/esr.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/esr.py @@ -27,6 +27,7 @@ from environment_provider.environment import release_full_environment from etos_lib import ETOS from etos_lib.logging.logger import FORMAT_CONFIG +from etos_lib.opentelemetry.semconv import Attributes as SemConvAttributes from jsontas.jsontas import JsonTas import opentelemetry @@ -38,8 +39,6 @@ # Remove spam from pika. logging.getLogger("pika").setLevel(logging.WARNING) -SUBSUITE_CONTEXT = None - class ESR: # pylint:disable=too-many-instance-attributes """Suite runner for ETOS main program. @@ -78,13 +77,14 @@ def _request_environment(self, ids: list[str]) -> None: try: provider = EnvironmentProvider(self.params.tercc.meta.event_id, ids, copy=False) result = provider.run() - except Exception: + except Exception as exc: self.params.set_status("FAILURE", "Failed to run environment provider") self.logger.error( "Environment provider has failed in creating an environment for test.", extra={"user_log": True}, ) - span.set_attribute("result", traceback.format_exc()) + span.record_exception(exc) + span.set_status(opentelemetry.trace.Status(opentelemetry.trace.StatusCode.ERROR)) raise if result.get("error") is not None: self.params.set_status("FAILURE", result.get("error")) @@ -92,6 +92,8 @@ def _request_environment(self, ids: list[str]) -> None: "Environment provider has failed in creating an environment for test.", extra={"user_log": True}, ) + span.set_attribute("exception", str(result.get("error"))) + span.set_status(opentelemetry.trace.Status(opentelemetry.trace.StatusCode.ERROR)) else: self.params.set_status("SUCCESS", result.get("error")) self.logger.info( diff --git a/projects/etos_suite_runner/src/etos_suite_runner/lib/executor.py b/projects/etos_suite_runner/src/etos_suite_runner/lib/executor.py index e0dc87b..9c3d30e 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/lib/executor.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/lib/executor.py @@ -21,6 +21,7 @@ from cryptography.fernet import Fernet from etos_lib import ETOS +from etos_lib.opentelemetry.semconv import Attributes as SemConvAttributes from opentelemetry import trace from requests.auth import HTTPBasicAuth, HTTPDigestAuth from requests.exceptions import ConnectionError as RequestsConnectionError @@ -93,13 +94,13 @@ def run_tests(self, test_suite: dict) -> None: method = getattr(self.etos.http, request.pop("method").lower()) span_name = "start_execution_space" with self.tracer.start_as_current_span(span_name) as span: - span.set_attribute("executor_id", executor["id"]) - span.set_attribute("request", dumps(request, indent=4)) + span.set_attribute(SemConvAttributes.EXECUTOR_ID, executor["id"]) + span.set_attribute("http.request.body", dumps(request, indent=4)) try: response = method(**request) response.raise_for_status() except HTTPError as http_error: - span.set_attribute("http_error", str(http_error)) + span.set_attribute("error.type", str(http_error)) try: raise TestStartException(http_error.response.json()) from http_error except JSONDecodeError: diff --git a/projects/etos_suite_runner/src/etos_suite_runner/lib/suite.py b/projects/etos_suite_runner/src/etos_suite_runner/lib/suite.py index edadaa1..3a16b77 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/lib/suite.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/lib/suite.py @@ -25,6 +25,7 @@ from environment_provider.environment import release_environment from etos_lib import ETOS from etos_lib.logging.logger import FORMAT_CONFIG +from etos_lib.opentelemetry.semconv import Attributes as SemConvAttributes from jsontas.jsontas import JsonTas import opentelemetry @@ -105,7 +106,7 @@ def start(self, identifier: str, otel_context: opentelemetry.context.context.Con # the subsuite is running in a separate process span_name = "execute_testrunner" with self.otel_tracer.start_as_current_span(span_name, context=otel_context) as span: - span.set_attribute("subsuite_id", identifier) + span.set_attribute(SemConvAttributes.SUBSUITE_ID, identifier) FORMAT_CONFIG.identifier = identifier self.logger.info("Starting up the ETOS test runner", extra={"user_log": True}) executor = Executor(self.etos) @@ -116,7 +117,8 @@ def start(self, identifier: str, otel_context: opentelemetry.context.context.Con self.logger.error( "Failed to start sub suite: %s", exception.error, extra={"user_log": True} ) - span.set_attribute("failed", str(exception)) + span.record_exception(exception) + span.set_status(opentelemetry.trace.Status(opentelemetry.trace.StatusCode.ERROR)) raise self.logger.info("ETR triggered.") timeout = time.time() + self.etos.debug.default_test_result_timeout @@ -154,13 +156,14 @@ def release(self, testrun_id) -> None: provider_registry=registry, sub_suite=self.environment, ) - span.set_attribute("testrun_id", testrun_id) - span.set_attribute("failure", str(failure)) - span.set_attribute("environment", json.dumps(self.environment, indent=4)) + span.set_attribute(SemConvAttributes.TESTRUN_ID, testrun_id) + span.set_attribute(SemConvAttributes.ENVIRONMENT, json.dumps(self.environment, indent=4)) if failure is not None: self.logger.exception( "Failed to check in %r", self.environment["id"], extra={"user_log": True} ) + span.record_exception(failure) + span.set_status(opentelemetry.trace.Status(opentelemetry.trace.StatusCode.ERROR)) return self.logger.info("Checked in %r", self.environment["id"], extra={"user_log": True}) self.released = True diff --git a/projects/etos_suite_runner/src/etos_suite_runner/otel_tracing.py b/projects/etos_suite_runner/src/etos_suite_runner/otel_tracing.py index c311340..a80d94e 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/otel_tracing.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/otel_tracing.py @@ -18,12 +18,42 @@ """OpenTelemetry-related code.""" import logging import os + import opentelemetry +from opentelemetry.propagators.textmap import Getter, Setter +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator + LOGGER = logging.getLogger(__name__) +class EnvVarContextGetter(Getter): + def get(self, carrier, key): + value = os.getenv(carrier) + if value is not None: + pairs = value.split(',') + for pair in pairs: + k, v = pair.split('=', 1) + if k == key: + return [v] + return [] + + def keys(self, carrier): + value = os.getenv(carrier) + if value is not None: + return [pair.split('=')[0] for pair in value.split(',') if '=' in pair] + return [] + def get_current_context() -> opentelemetry.context.context.Context: + """Get current context propagated via environment variable OTEL_CONTEXT.""" + LOGGER.info("Current OpenTelemetry context env: %s", os.environ.get("OTEL_CONTEXT")) + propagator = TraceContextTextMapPropagator() + ctx = propagator.extract(carrier="OTEL_CONTEXT", getter=EnvVarContextGetter()) + LOGGER.info("Current OpenTelemetry context %s", ctx) + return ctx + + +def get_current_context_old() -> opentelemetry.context.context.Context: """Get current context (propagated via environment variable OTEL_CONTEXT).""" carrier = {} LOGGER.info("Current OpenTelemetry context env: %s", os.environ.get("OTEL_CONTEXT")) From 0fc44073a83643bc899a371ca20f7ee28c74c9e2 Mon Sep 17 00:00:00 2001 From: Andrei Matveyeu Date: Fri, 26 Apr 2024 09:23:58 +0200 Subject: [PATCH 07/10] Review updates --- Dockerfile | 7 ---- projects/etos_suite_runner/requirements.txt | 2 +- projects/etos_suite_runner/setup.cfg | 4 +-- .../src/etos_suite_runner/__init__.py | 13 +++++-- .../src/etos_suite_runner/esr.py | 25 +++++++------ .../src/etos_suite_runner/lib/executor.py | 21 +++++++---- .../{ => lib}/otel_tracing.py | 33 ++++++++--------- .../src/etos_suite_runner/lib/runner.py | 14 +++++--- .../src/etos_suite_runner/lib/suite.py | 36 ++++++++++++------- 9 files changed, 93 insertions(+), 62 deletions(-) rename projects/etos_suite_runner/src/etos_suite_runner/{ => lib}/otel_tracing.py (70%) diff --git a/Dockerfile b/Dockerfile index 8356e5a..ad30cd7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,13 +8,6 @@ FROM python:3.9-slim-buster COPY --from=build /src/projects/etos_suite_runner/dist/*.whl /tmp # hadolint ignore=DL3013 -# hadolint ignore=DL3008 -#RUN apt-get update && \ -# apt-get install -y gcc libc-dev --no-install-recommends && \ -# pip install --no-cache-dir /tmp/*.whl && \ -# apt-get purge -y --auto-remove gcc libc-dev && \ -# rm -rf /var/lib/apt/lists/* && \ -# groupadd -r etos && useradd -r -m -s /bin/false -g etos etos RUN pip install --no-cache-dir /tmp/*.whl && groupadd -r etos && useradd -r -m -s /bin/false -g etos etos USER etos diff --git a/projects/etos_suite_runner/requirements.txt b/projects/etos_suite_runner/requirements.txt index 9926fc8..d3712cb 100644 --- a/projects/etos_suite_runner/requirements.txt +++ b/projects/etos_suite_runner/requirements.txt @@ -23,7 +23,7 @@ etos_lib==4.0.0 etos_environment_provider~=4.1 ======= etos_lib==4.1.1 -#etos_environment_provider~=3.2 +etos_environment_provider~=4.2 opentelemetry-api~=1.21 opentelemetry-exporter-otlp~=1.21 opentelemetry-sdk~=1.21 diff --git a/projects/etos_suite_runner/setup.cfg b/projects/etos_suite_runner/setup.cfg index 9ae1d33..8ce7775 100644 --- a/projects/etos_suite_runner/setup.cfg +++ b/projects/etos_suite_runner/setup.cfg @@ -28,8 +28,8 @@ install_requires = PyScaffold==3.2.3 packageurl-python~=0.11 cryptography>=42.0.4,<43.0.0 - #etos_lib==4.1.1 - #etos_environment_provider~=4.0 + etos_lib==4.2.0 + etos_environment_provider~=4.2 opentelemetry-api~=1.21 opentelemetry-exporter-otlp~=1.21 opentelemetry-sdk~=1.21 diff --git a/projects/etos_suite_runner/src/etos_suite_runner/__init__.py b/projects/etos_suite_runner/src/etos_suite_runner/__init__.py index 7355620..cea339f 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/__init__.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/__init__.py @@ -42,12 +42,19 @@ LOGGER = logging.getLogger(__name__) # Setting OTEL_COLLECTOR_HOST will override the default OTEL collector endpoint. -# This is needed when using the cluster-level OTEL collector instead of sidecar collector. +# This is needed because Suite Runner uses the cluster-level OpenTelemetry collector +# instead of a sidecar collector. if os.getenv("OTEL_COLLECTOR_HOST"): os.environ["OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"] = os.getenv("OTEL_COLLECTOR_HOST") - LOGGER.info("Using OTEL collector: %s", os.getenv("OTEL_COLLECTOR_HOST")) +else: + LOGGER.debug("Environment variable OTEL_EXPORTER_OTLP_TRACES_ENDPOINT not used.") + LOGGER.debug("To specify an OpenTelemetry collector host use OTEL_COLLECTOR_HOST.") + del os.environ["OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"] if os.getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"): + LOGGER.info( + "Using OpenTelemetry collector: %s", os.getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT") + ) PROVIDER = TracerProvider( resource=Resource.create( { @@ -62,4 +69,4 @@ PROVIDER.add_span_processor(PROCESSOR) trace.set_tracer_provider(PROVIDER) else: - LOGGER.info("Suite runner OTEL_EXPORTER_OTLP_TRACES_ENDPOINT not set!") + LOGGER.info("OpenTelemetry not enabled. OTEL_COLLECTOR_HOST not set.") diff --git a/projects/etos_suite_runner/src/etos_suite_runner/esr.py b/projects/etos_suite_runner/src/etos_suite_runner/esr.py index 5621f8f..ea5f5e2 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/esr.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/esr.py @@ -34,13 +34,13 @@ from .lib.esr_parameters import ESRParameters from .lib.exceptions import EnvironmentProviderException from .lib.runner import SuiteRunner -from .otel_tracing import get_current_context +from .lib.otel_tracing import get_current_context, OpenTelemetryBase # Remove spam from pika. logging.getLogger("pika").setLevel(logging.WARNING) -class ESR: # pylint:disable=too-many-instance-attributes +class ESR(OpenTelemetryBase): # pylint:disable=too-many-instance-attributes """Suite runner for ETOS main program. Run this as a daemon on your system in order to trigger test suites within @@ -73,7 +73,9 @@ def _request_environment(self, ids: list[str]) -> None: """ span_name = "request_environment" suite_context = get_current_context() - with self.otel_tracer.start_as_current_span(span_name, context=suite_context) as span: + with self.otel_tracer.start_as_current_span( + span_name, context=suite_context, kind=opentelemetry.trace.SpanKind.CLIENT, + ) as span: try: provider = EnvironmentProvider(self.params.tercc.meta.event_id, ids, copy=False) result = provider.run() @@ -83,8 +85,7 @@ def _request_environment(self, ids: list[str]) -> None: "Environment provider has failed in creating an environment for test.", extra={"user_log": True}, ) - span.record_exception(exc) - span.set_status(opentelemetry.trace.Status(opentelemetry.trace.StatusCode.ERROR)) + self._record_exception(exc) raise if result.get("error") is not None: self.params.set_status("FAILURE", result.get("error")) @@ -92,8 +93,8 @@ def _request_environment(self, ids: list[str]) -> None: "Environment provider has failed in creating an environment for test.", extra={"user_log": True}, ) - span.set_attribute("exception", str(result.get("error"))) - span.set_status(opentelemetry.trace.Status(opentelemetry.trace.StatusCode.ERROR)) + exc = Exception(str(result.get("error"))) + self._record_exception(exc) else: self.params.set_status("SUCCESS", result.get("error")) self.logger.info( @@ -109,7 +110,9 @@ def _release_environment(self) -> None: jsontas = JsonTas() span_name = "release_full_environment" suite_context = get_current_context() - with self.otel_tracer.start_as_current_span(span_name, context=suite_context): + with self.otel_tracer.start_as_current_span( + span_name, context=suite_context, kind=opentelemetry.trace.SpanKind.CLIENT, + ): status, message = release_full_environment( etos=self.etos, jsontas=jsontas, suite_id=self.params.tercc.meta.event_id ) @@ -147,10 +150,11 @@ def run_suites(self, triggered: EiffelActivityTriggeredEvent) -> list[str]: self.logger.info("Starting ESR.") runner.start_suites_and_wait() return ids - except EnvironmentProviderException: + except EnvironmentProviderException as exc: self.logger.info("Release test environment.") self._release_environment() - raise + self._record_exception(exc) + raise exc @staticmethod def verify_input() -> None: @@ -222,6 +226,7 @@ def run(self) -> list[str]: "MAJOR", {"CONTEXT": context}, ) + self._record_exception(exception) raise def graceful_exit(self, *_) -> None: diff --git a/projects/etos_suite_runner/src/etos_suite_runner/lib/executor.py b/projects/etos_suite_runner/src/etos_suite_runner/lib/executor.py index 9c3d30e..aad45a0 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/lib/executor.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/lib/executor.py @@ -27,6 +27,7 @@ from requests.exceptions import ConnectionError as RequestsConnectionError from requests.exceptions import HTTPError +from .otel_tracing import OpenTelemetryBase class TestStartException(Exception): """Exception when starting tests.""" @@ -37,7 +38,7 @@ def __init__(self, message: dict): self.error = message.get("error", "Unknown error when starting tests") -class Executor: # pylint:disable=too-few-public-methods +class Executor(OpenTelemetryBase): # pylint:disable=too-few-public-methods """Executor for launching ETR.""" logger = logging.getLogger("ESR - Executor") @@ -93,19 +94,27 @@ def run_tests(self, test_suite: dict) -> None: request["auth"] = self.__auth(**request["auth"]) method = getattr(self.etos.http, request.pop("method").lower()) span_name = "start_execution_space" - with self.tracer.start_as_current_span(span_name) as span: + with self.tracer.start_as_current_span( + span_name, kind=trace.SpanKind.CLIENT + ) as span: span.set_attribute(SemConvAttributes.EXECUTOR_ID, executor["id"]) span.set_attribute("http.request.body", dumps(request, indent=4)) try: response = method(**request) response.raise_for_status() except HTTPError as http_error: - span.set_attribute("error.type", str(http_error)) try: - raise TestStartException(http_error.response.json()) from http_error + exc = TestStartException(http_error.response.json()) + self._record_exception(exc) + raise exc from http_error except JSONDecodeError: - raise TestStartException({"error": http_error.response.text}) from http_error + exc = TestStartException({"error": http_error.response.text}) + self._record_exception(exc) + raise exc from http_error except RequestsConnectionError as connection_error: - raise TestStartException({"error": str(connection_error)}) from connection_error + exc = TestStartException({"error": str(connection_error)}) + self._record_exception(exc) + raise exc from connection_error + self.logger.info("%r", response) self.logger.debug("%r", response.text) diff --git a/projects/etos_suite_runner/src/etos_suite_runner/otel_tracing.py b/projects/etos_suite_runner/src/etos_suite_runner/lib/otel_tracing.py similarity index 70% rename from projects/etos_suite_runner/src/etos_suite_runner/otel_tracing.py rename to projects/etos_suite_runner/src/etos_suite_runner/lib/otel_tracing.py index a80d94e..ee596fe 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/otel_tracing.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/lib/otel_tracing.py @@ -20,15 +20,30 @@ import os import opentelemetry -from opentelemetry.propagators.textmap import Getter, Setter +from opentelemetry.propagators.textmap import Getter from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator LOGGER = logging.getLogger(__name__) +class OpenTelemetryBase: + """Base functionality for OpenTelemetry data collection.""" + + @staticmethod + def _record_exception(exc) -> None: + """Record the given exception to the current OpenTelemetry span.""" + span = opentelemetry.trace.get_current_span() + span.set_attribute("error.type", exc.__class__.__name__) + span.record_exception(exc) + span.set_status(opentelemetry.trace.Status(opentelemetry.trace.StatusCode.ERROR)) + + class EnvVarContextGetter(Getter): + """OpenTelemetry context getter class for environment variables.""" + def get(self, carrier, key): + """Get value using the given carrier variable and key.""" value = os.getenv(carrier) if value is not None: pairs = value.split(',') @@ -39,6 +54,7 @@ def get(self, carrier, key): return [] def keys(self, carrier): + """Get keys of the given carrier variable.""" value = os.getenv(carrier) if value is not None: return [pair.split('=')[0] for pair in value.split(',') if '=' in pair] @@ -46,21 +62,6 @@ def keys(self, carrier): def get_current_context() -> opentelemetry.context.context.Context: """Get current context propagated via environment variable OTEL_CONTEXT.""" - LOGGER.info("Current OpenTelemetry context env: %s", os.environ.get("OTEL_CONTEXT")) propagator = TraceContextTextMapPropagator() ctx = propagator.extract(carrier="OTEL_CONTEXT", getter=EnvVarContextGetter()) - LOGGER.info("Current OpenTelemetry context %s", ctx) - return ctx - - -def get_current_context_old() -> opentelemetry.context.context.Context: - """Get current context (propagated via environment variable OTEL_CONTEXT).""" - carrier = {} - LOGGER.info("Current OpenTelemetry context env: %s", os.environ.get("OTEL_CONTEXT")) - for kv in os.environ.get("OTEL_CONTEXT", "").split(","): - if kv: - k, v = kv.split("=", 1) - carrier[k] = v - ctx = opentelemetry.propagate.extract(carrier) - LOGGER.info("Current OpenTelemetry context %s", ctx) return ctx diff --git a/projects/etos_suite_runner/src/etos_suite_runner/lib/runner.py b/projects/etos_suite_runner/src/etos_suite_runner/lib/runner.py index ac4767f..7ef2eb2 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/lib/runner.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/lib/runner.py @@ -17,17 +17,19 @@ import logging from multiprocessing.pool import ThreadPool +import opentelemetry.trace + from environment_provider.environment import release_full_environment from etos_lib.logging.logger import FORMAT_CONFIG from jsontas.jsontas import JsonTas import opentelemetry from .exceptions import EnvironmentProviderException -from ..otel_tracing import get_current_context +from .otel_tracing import get_current_context, OpenTelemetryBase from .suite import TestSuite -class SuiteRunner: # pylint:disable=too-few-public-methods +class SuiteRunner(OpenTelemetryBase): # pylint:disable=too-few-public-methods """Test suite runner. Splits test suites into sub suites based on number of products available. @@ -56,7 +58,9 @@ def _release_environment(self): # jsontas is not required. jsontas = JsonTas() span_name = "release_full_environment" - with self.otel_tracer.start_as_current_span(span_name, context=self.otel_suite_context): + with self.otel_tracer.start_as_current_span( + span_name, context=self.otel_suite_context, kind=opentelemetry.trace.SpanKind.CLIENT, + ): status, message = release_full_environment( etos=self.etos, jsontas=jsontas, suite_id=self.params.tercc.meta.event_id ) @@ -74,7 +78,9 @@ def start_suites_and_wait(self): pool.map(self.run, test_suites) status = self.params.get_status() if status.get("error") is not None: - raise EnvironmentProviderException(status["error"], self.etos.config.get("task_id")) + exc = EnvironmentProviderException(status["error"], self.etos.config.get("task_id")) + self._record_exception(exc) + raise exc finally: self.logger.info("Release the full test environment.") self._release_environment() diff --git a/projects/etos_suite_runner/src/etos_suite_runner/lib/suite.py b/projects/etos_suite_runner/src/etos_suite_runner/lib/suite.py index 3a16b77..d3b840f 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/lib/suite.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/lib/suite.py @@ -21,6 +21,7 @@ from typing import Iterator from eiffellib.events import EiffelTestSuiteStartedEvent +import opentelemetry.trace from environment_provider.lib.registry import ProviderRegistry from environment_provider.environment import release_environment from etos_lib import ETOS @@ -40,9 +41,10 @@ request_test_suite_started, ) from .log_filter import DuplicateFilter +from .otel_tracing import OpenTelemetryBase -class SubSuite: # pylint:disable=too-many-instance-attributes +class SubSuite(OpenTelemetryBase): # pylint:disable=too-many-instance-attributes """Handle test results and tracking of a single sub suite.""" released = False @@ -102,10 +104,12 @@ def start(self, identifier: str, otel_context: opentelemetry.context.context.Con :param identifier: An identifier for logs in this sub suite. """ - # OpenTelemetry context needs to be retrieved here: - # the subsuite is running in a separate process + # OpenTelemetry context needs to be explicitly given here when creating this new span. + # This is because the subsuite is running in a separate thread. span_name = "execute_testrunner" - with self.otel_tracer.start_as_current_span(span_name, context=otel_context) as span: + with self.otel_tracer.start_as_current_span( + span_name, context=otel_context, kind=opentelemetry.trace.SpanKind.CLIENT, + ) as span: span.set_attribute(SemConvAttributes.SUBSUITE_ID, identifier) FORMAT_CONFIG.identifier = identifier self.logger.info("Starting up the ETOS test runner", extra={"user_log": True}) @@ -117,8 +121,7 @@ def start(self, identifier: str, otel_context: opentelemetry.context.context.Con self.logger.error( "Failed to start sub suite: %s", exception.error, extra={"user_log": True} ) - span.record_exception(exception) - span.set_status(opentelemetry.trace.Status(opentelemetry.trace.StatusCode.ERROR)) + self._record_exception(exception) raise self.logger.info("ETR triggered.") timeout = time.time() + self.etos.debug.default_test_result_timeout @@ -149,7 +152,9 @@ def release(self, testrun_id) -> None: self.logger.info(self.environment) span_name = "release_environment" - with self.otel_tracer.start_as_current_span(span_name) as span: + with self.otel_tracer.start_as_current_span( + span_name, kind=opentelemetry.trace.SpanKind.CLIENT, + ) as span: failure = release_environment( etos=self.etos, jsontas=jsontas, @@ -162,14 +167,13 @@ def release(self, testrun_id) -> None: self.logger.exception( "Failed to check in %r", self.environment["id"], extra={"user_log": True} ) - span.record_exception(failure) - span.set_status(opentelemetry.trace.Status(opentelemetry.trace.StatusCode.ERROR)) + self._record_exception(failure) return self.logger.info("Checked in %r", self.environment["id"], extra={"user_log": True}) self.released = True -class TestSuite: # pylint:disable=too-many-instance-attributes +class TestSuite(OpenTelemetryBase): # pylint:disable=too-many-instance-attributes """Handle the starting and waiting for test suites in ETOS.""" test_suite_started = None @@ -214,9 +218,11 @@ def sub_suite_environments(self) -> Iterator[dict]: if activity_triggered is None: status = self.params.get_status() if status.get("status") == "FAILURE": - raise EnvironmentProviderException( + exc = EnvironmentProviderException( status.get("error"), self.etos.config.get("task_id") ) + self._record_exception(exc) + raise exc continue activity_finished = self.__environment_activity_finished( activity_triggered["meta"]["id"] @@ -229,16 +235,20 @@ def sub_suite_environments(self) -> Iterator[dict]: yield environment if activity_finished is not None: if activity_finished["data"]["activityOutcome"]["conclusion"] != "SUCCESSFUL": - raise EnvironmentProviderException( + exc = EnvironmentProviderException( activity_finished["data"]["activityOutcome"]["description"], self.etos.config.get("task_id"), ) + self._record_exception(exc) + raise exc if len(environments) > 0: # Must be at least 1 sub suite. return else: # pylint:disable=useless-else-on-loop - raise TimeoutError( + exc = TimeoutError( f"Timed out after {self.etos.config.get('WAIT_FOR_ENVIRONMENT_TIMEOUT')} seconds." ) + self._record_exception(exc) + raise exc @property def all_finished(self) -> bool: From aed50b793c244adfba55d8fb5dad52d60ef94ab7 Mon Sep 17 00:00:00 2001 From: Andrei Matveyeu Date: Fri, 3 May 2024 07:35:02 +0200 Subject: [PATCH 08/10] Code review changes --- projects/etos_suite_runner/requirements.txt | 4 ++++ .../src/etos_suite_runner/__init__.py | 7 ++++--- .../etos_suite_runner/src/etos_suite_runner/esr.py | 11 +++++++---- .../src/etos_suite_runner/lib/executor.py | 7 +++---- .../src/etos_suite_runner/lib/otel_tracing.py | 8 +++++--- .../src/etos_suite_runner/lib/runner.py | 6 +++--- .../src/etos_suite_runner/lib/suite.py | 10 ++++++---- 7 files changed, 32 insertions(+), 21 deletions(-) diff --git a/projects/etos_suite_runner/requirements.txt b/projects/etos_suite_runner/requirements.txt index d3712cb..3ef02bf 100644 --- a/projects/etos_suite_runner/requirements.txt +++ b/projects/etos_suite_runner/requirements.txt @@ -19,10 +19,14 @@ PyScaffold==3.2.3 packageurl-python~=0.11 cryptography>=42.0.4,<43.0.0 <<<<<<< HEAD +<<<<<<< HEAD etos_lib==4.0.0 etos_environment_provider~=4.1 ======= etos_lib==4.1.1 +======= +etos_lib==4.2.0 +>>>>>>> 11b187c (Code review changes) etos_environment_provider~=4.2 opentelemetry-api~=1.21 opentelemetry-exporter-otlp~=1.21 diff --git a/projects/etos_suite_runner/src/etos_suite_runner/__init__.py b/projects/etos_suite_runner/src/etos_suite_runner/__init__.py index cea339f..c921c33 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/__init__.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/__init__.py @@ -47,9 +47,10 @@ if os.getenv("OTEL_COLLECTOR_HOST"): os.environ["OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"] = os.getenv("OTEL_COLLECTOR_HOST") else: - LOGGER.debug("Environment variable OTEL_EXPORTER_OTLP_TRACES_ENDPOINT not used.") - LOGGER.debug("To specify an OpenTelemetry collector host use OTEL_COLLECTOR_HOST.") - del os.environ["OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"] + if "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT" in os.environ: + LOGGER.debug("Environment variable OTEL_EXPORTER_OTLP_TRACES_ENDPOINT not used.") + LOGGER.debug("To specify an OpenTelemetry collector host use OTEL_COLLECTOR_HOST.") + del os.environ["OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"] if os.getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"): LOGGER.info( diff --git a/projects/etos_suite_runner/src/etos_suite_runner/esr.py b/projects/etos_suite_runner/src/etos_suite_runner/esr.py index ea5f5e2..543dd40 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/esr.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/esr.py @@ -27,7 +27,6 @@ from environment_provider.environment import release_full_environment from etos_lib import ETOS from etos_lib.logging.logger import FORMAT_CONFIG -from etos_lib.opentelemetry.semconv import Attributes as SemConvAttributes from jsontas.jsontas import JsonTas import opentelemetry @@ -74,8 +73,10 @@ def _request_environment(self, ids: list[str]) -> None: span_name = "request_environment" suite_context = get_current_context() with self.otel_tracer.start_as_current_span( - span_name, context=suite_context, kind=opentelemetry.trace.SpanKind.CLIENT, - ) as span: + span_name, + context=suite_context, + kind=opentelemetry.trace.SpanKind.CLIENT, + ): try: provider = EnvironmentProvider(self.params.tercc.meta.event_id, ids, copy=False) result = provider.run() @@ -111,7 +112,9 @@ def _release_environment(self) -> None: span_name = "release_full_environment" suite_context = get_current_context() with self.otel_tracer.start_as_current_span( - span_name, context=suite_context, kind=opentelemetry.trace.SpanKind.CLIENT, + span_name, + context=suite_context, + kind=opentelemetry.trace.SpanKind.CLIENT, ): status, message = release_full_environment( etos=self.etos, jsontas=jsontas, suite_id=self.params.tercc.meta.event_id diff --git a/projects/etos_suite_runner/src/etos_suite_runner/lib/executor.py b/projects/etos_suite_runner/src/etos_suite_runner/lib/executor.py index aad45a0..41743ba 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/lib/executor.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/lib/executor.py @@ -29,6 +29,7 @@ from .otel_tracing import OpenTelemetryBase + class TestStartException(Exception): """Exception when starting tests.""" @@ -94,11 +95,9 @@ def run_tests(self, test_suite: dict) -> None: request["auth"] = self.__auth(**request["auth"]) method = getattr(self.etos.http, request.pop("method").lower()) span_name = "start_execution_space" - with self.tracer.start_as_current_span( - span_name, kind=trace.SpanKind.CLIENT - ) as span: + with self.tracer.start_as_current_span(span_name, kind=trace.SpanKind.CLIENT) as span: span.set_attribute(SemConvAttributes.EXECUTOR_ID, executor["id"]) - span.set_attribute("http.request.body", dumps(request, indent=4)) + span.set_attribute("http.request.body", dumps(request)) try: response = method(**request) response.raise_for_status() diff --git a/projects/etos_suite_runner/src/etos_suite_runner/lib/otel_tracing.py b/projects/etos_suite_runner/src/etos_suite_runner/lib/otel_tracing.py index ee596fe..76e9f63 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/lib/otel_tracing.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/lib/otel_tracing.py @@ -30,6 +30,7 @@ class OpenTelemetryBase: """Base functionality for OpenTelemetry data collection.""" + # pylint: disable=too-few-public-methods @staticmethod def _record_exception(exc) -> None: """Record the given exception to the current OpenTelemetry span.""" @@ -46,9 +47,9 @@ def get(self, carrier, key): """Get value using the given carrier variable and key.""" value = os.getenv(carrier) if value is not None: - pairs = value.split(',') + pairs = value.split(",") for pair in pairs: - k, v = pair.split('=', 1) + k, v = pair.split("=", 1) if k == key: return [v] return [] @@ -57,9 +58,10 @@ def keys(self, carrier): """Get keys of the given carrier variable.""" value = os.getenv(carrier) if value is not None: - return [pair.split('=')[0] for pair in value.split(',') if '=' in pair] + return [pair.split("=")[0] for pair in value.split(",") if "=" in pair] return [] + def get_current_context() -> opentelemetry.context.context.Context: """Get current context propagated via environment variable OTEL_CONTEXT.""" propagator = TraceContextTextMapPropagator() diff --git a/projects/etos_suite_runner/src/etos_suite_runner/lib/runner.py b/projects/etos_suite_runner/src/etos_suite_runner/lib/runner.py index 7ef2eb2..7624aaa 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/lib/runner.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/lib/runner.py @@ -17,8 +17,6 @@ import logging from multiprocessing.pool import ThreadPool -import opentelemetry.trace - from environment_provider.environment import release_full_environment from etos_lib.logging.logger import FORMAT_CONFIG from jsontas.jsontas import JsonTas @@ -59,7 +57,9 @@ def _release_environment(self): jsontas = JsonTas() span_name = "release_full_environment" with self.otel_tracer.start_as_current_span( - span_name, context=self.otel_suite_context, kind=opentelemetry.trace.SpanKind.CLIENT, + span_name, + context=self.otel_suite_context, + kind=opentelemetry.trace.SpanKind.CLIENT, ): status, message = release_full_environment( etos=self.etos, jsontas=jsontas, suite_id=self.params.tercc.meta.event_id diff --git a/projects/etos_suite_runner/src/etos_suite_runner/lib/suite.py b/projects/etos_suite_runner/src/etos_suite_runner/lib/suite.py index d3b840f..9a3284a 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/lib/suite.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/lib/suite.py @@ -21,7 +21,6 @@ from typing import Iterator from eiffellib.events import EiffelTestSuiteStartedEvent -import opentelemetry.trace from environment_provider.lib.registry import ProviderRegistry from environment_provider.environment import release_environment from etos_lib import ETOS @@ -108,7 +107,9 @@ def start(self, identifier: str, otel_context: opentelemetry.context.context.Con # This is because the subsuite is running in a separate thread. span_name = "execute_testrunner" with self.otel_tracer.start_as_current_span( - span_name, context=otel_context, kind=opentelemetry.trace.SpanKind.CLIENT, + span_name, + context=otel_context, + kind=opentelemetry.trace.SpanKind.CLIENT, ) as span: span.set_attribute(SemConvAttributes.SUBSUITE_ID, identifier) FORMAT_CONFIG.identifier = identifier @@ -153,7 +154,8 @@ def release(self, testrun_id) -> None: span_name = "release_environment" with self.otel_tracer.start_as_current_span( - span_name, kind=opentelemetry.trace.SpanKind.CLIENT, + span_name, + kind=opentelemetry.trace.SpanKind.CLIENT, ) as span: failure = release_environment( etos=self.etos, @@ -162,7 +164,7 @@ def release(self, testrun_id) -> None: sub_suite=self.environment, ) span.set_attribute(SemConvAttributes.TESTRUN_ID, testrun_id) - span.set_attribute(SemConvAttributes.ENVIRONMENT, json.dumps(self.environment, indent=4)) + span.set_attribute(SemConvAttributes.ENVIRONMENT, json.dumps(self.environment)) if failure is not None: self.logger.exception( "Failed to check in %r", self.environment["id"], extra={"user_log": True} From ebacc991b9724f4a7d053339a56272a7fca19576 Mon Sep 17 00:00:00 2001 From: Andrei Matveyeu Date: Fri, 3 May 2024 07:39:57 +0200 Subject: [PATCH 09/10] Code review changes --- projects/etos_suite_runner/requirements.txt | 9 --------- 1 file changed, 9 deletions(-) diff --git a/projects/etos_suite_runner/requirements.txt b/projects/etos_suite_runner/requirements.txt index 3ef02bf..1c02a7b 100644 --- a/projects/etos_suite_runner/requirements.txt +++ b/projects/etos_suite_runner/requirements.txt @@ -18,17 +18,8 @@ PyScaffold==3.2.3 packageurl-python~=0.11 cryptography>=42.0.4,<43.0.0 -<<<<<<< HEAD -<<<<<<< HEAD -etos_lib==4.0.0 -etos_environment_provider~=4.1 -======= -etos_lib==4.1.1 -======= etos_lib==4.2.0 ->>>>>>> 11b187c (Code review changes) etos_environment_provider~=4.2 opentelemetry-api~=1.21 opentelemetry-exporter-otlp~=1.21 opentelemetry-sdk~=1.21 ->>>>>>> e187c17 (Add OpenTelemetry instrumentation) From 68a44a256d08d6b8e88ea05335f9182ebd13cf6b Mon Sep 17 00:00:00 2001 From: Andrei Matveyeu Date: Fri, 3 May 2024 10:25:23 +0200 Subject: [PATCH 10/10] Code review changes --- .../etos_suite_runner/src/etos_suite_runner/lib/executor.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/projects/etos_suite_runner/src/etos_suite_runner/lib/executor.py b/projects/etos_suite_runner/src/etos_suite_runner/lib/executor.py index 41743ba..4210264 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/lib/executor.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/lib/executor.py @@ -96,7 +96,9 @@ def run_tests(self, test_suite: dict) -> None: method = getattr(self.etos.http, request.pop("method").lower()) span_name = "start_execution_space" with self.tracer.start_as_current_span(span_name, kind=trace.SpanKind.CLIENT) as span: - span.set_attribute(SemConvAttributes.EXECUTOR_ID, executor["id"]) + span.set_attribute( + SemConvAttributes.EXECUTOR_ID, executor["id"] if "id" in executor else "" + ) span.set_attribute("http.request.body", dumps(request)) try: response = method(**request)