Add OpenTelemetry instrumentation

Dockerfile

@@ -1,14 +1,20 @@
-FROM python:3.9.0-buster AS build
+FROM python:3.9-buster AS build
 
 COPY . /src
 WORKDIR /src/projects/etos_suite_runner
 RUN python3 setup.py bdist_wheel
 
-FROM python:3.9.0-slim-buster
+FROM python:3.9-slim-buster
 
 COPY --from=build /src/projects/etos_suite_runner/dist/*.whl /tmp
 # hadolint ignore=DL3013
-RUN pip install --no-cache-dir /tmp/*.whl && groupadd -r etos && useradd -r -m -s /bin/false -g etos etos
+RUN apt-get update && \
+    apt-get install -y gcc libc-dev --no-install-recommends && \
+    pip install --no-cache-dir /tmp/*.whl && \
+    apt-get purge -y --auto-remove gcc libc-dev && \
+    rm -rf /var/lib/apt/lists/* && \
+    groupadd -r etos && useradd -r -m -s /bin/false -g etos etos
+
 USER etos
 
 LABEL org.opencontainers.image.source=https://github.com/eiffel-community/etos-suite-runner

projects/etos_suite_runner/requirements.txt

@@ -18,5 +18,8 @@
 PyScaffold==3.2.3
 packageurl-python~=0.11
 cryptography>=42.0.4,<43.0.0
-etos_lib==4.0.0
-etos_environment_provider~=3.2
+etos_lib==4.1.1
+#etos_environment_provider~=3.2
+opentelemetry-api~=1.21
+opentelemetry-exporter-otlp~=1.21
+opentelemetry-sdk~=1.21

projects/etos_suite_runner/setup.cfg

@@ -28,8 +28,11 @@ install_requires =
     PyScaffold==3.2.3
     packageurl-python~=0.11
     cryptography>=42.0.4,<43.0.0
-    etos_lib==4.0.0
-    etos_environment_provider~=4.0
+    etos_lib==4.1.1
+    #etos_environment_provider~=4.0
+    opentelemetry-api~=1.21
+    opentelemetry-exporter-otlp~=1.21
+    opentelemetry-sdk~=1.21
 
 python_requires = >=3.4
 

projects/etos_suite_runner/src/etos_suite_runner/__init__.py

@@ -14,9 +14,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """ETOS suite runner module."""
+import logging
 import os
 from importlib.metadata import PackageNotFoundError, version
 
+from opentelemetry import trace
+from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
+from opentelemetry.sdk.resources import SERVICE_NAME, SERVICE_NAMESPACE, SERVICE_VERSION, Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+
 from etos_lib.logging.logger import setup_logging
 
 try:
@@ -30,3 +37,25 @@
 ENVIRONMENT = "development" if DEV else "production"
 os.environ["ENVIRONMENT_PROVIDER_DISABLE_LOGGING"] = "true"
 setup_logging("ETOS Suite Runner", VERSION, ENVIRONMENT)
+
+
+LOGGER = logging.getLogger(__name__)
+
+# Setting OTEL_COLLECTOR_HOST will override the default OTEL collector endpoint.
+# This is needed when using the centralized cluster-level OTEL collector instead of sidecar collector.
+if os.getenv("OTEL_COLLECTOR_HOST"):
+    os.environ["OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"] = os.getenv("OTEL_COLLECTOR_HOST")
+    LOGGER.info("Using OTEL collector: %s", os.getenv("OTEL_COLLECTOR_HOST"))
+
+if os.getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"):
+    PROVIDER = TracerProvider(
+        resource=Resource.create(
+            {SERVICE_NAME: "etos-suite-runner", SERVICE_VERSION: VERSION, SERVICE_NAMESPACE: ENVIRONMENT}
+        )
+    )
+    EXPORTER = OTLPSpanExporter()
+    PROCESSOR = BatchSpanProcessor(EXPORTER)
+    PROVIDER.add_span_processor(PROCESSOR)
+    trace.set_tracer_provider(PROVIDER)
+else:
+    LOGGER.info("Suite runner OTEL_EXPORTER_OTLP_TRACES_ENDPOINT not set!")

projects/etos_suite_runner/src/etos_suite_runner/__main__.py

@@ -21,6 +21,7 @@
 
 from .esr import ESR
 
+
 LOGGER = logging.getLogger(__name__)
 
 
@@ -38,11 +39,9 @@ def main():
         esr.etos.publisher.stop()
     LOGGER.info("ESR Finished Executing.", extra={"user_log": True})
 
-
 def run():
     """Entry point for console_scripts."""
     main()
 
-
 if __name__ == "__main__":
     run()

projects/etos_suite_runner/src/etos_suite_runner/esr.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 # -*- coding: utf-8 -*-
 """ETOS suite runner module."""
+import json
 import logging
 import os
 import signal
@@ -28,14 +29,17 @@
 from etos_lib import ETOS
 from etos_lib.logging.logger import FORMAT_CONFIG
 from jsontas.jsontas import JsonTas
+import opentelemetry
 
 from .lib.esr_parameters import ESRParameters
 from .lib.exceptions import EnvironmentProviderException
 from .lib.runner import SuiteRunner
+from .otel_tracing import get_current_context
 
 # Remove spam from pika.
 logging.getLogger("pika").setLevel(logging.WARNING)
 
+SUBSUITE_CONTEXT = None
 
 class ESR:  # pylint:disable=too-many-instance-attributes
     """Suite runner for ETOS main program.
@@ -49,6 +53,7 @@ class ESR:  # pylint:disable=too-many-instance-attributes
     def __init__(self) -> None:
         """Initialize ESR by creating a rabbitmq publisher."""
         self.logger = logging.getLogger("ESR")
+        self.otel_tracer = opentelemetry.trace.get_tracer(__name__)
         self.etos = ETOS("ETOS Suite Runner", os.getenv("SOURCE_HOST"), "ETOS Suite Runner")
         signal.signal(signal.SIGTERM, self.graceful_exit)
         self.params = ESRParameters(self.etos)
@@ -67,40 +72,47 @@ def _request_environment(self, ids: list[str]) -> None:
         :param ids: Generated suite runner IDs used to correlate environments and the suite
                     runners.
         """
-        try:
-            provider = EnvironmentProvider(self.params.tercc.meta.event_id, ids, copy=False)
-            result = provider.run()
-        except Exception:
-            self.params.set_status("FAILURE", "Failed to run environment provider")
-            self.logger.error(
-                "Environment provider has failed in creating an environment for test.",
-                extra={"user_log": True},
-            )
-            raise
-        if result.get("error") is not None:
-            self.params.set_status("FAILURE", result.get("error"))
-            self.logger.error(
-                "Environment provider has failed in creating an environment for test.",
-                extra={"user_log": True},
-            )
-        else:
-            self.params.set_status("SUCCESS", result.get("error"))
-            self.logger.info(
-                "Environment provider has finished creating an environment for test.",
-                extra={"user_log": True},
-            )
+        span_name = "request_environment"
+        suite_context = get_current_context()
+        with self.otel_tracer.start_as_current_span(span_name, context=suite_context) as span:
+            try:
+                provider = EnvironmentProvider(self.params.tercc.meta.event_id, ids, copy=False)
+                result = provider.run()
+            except Exception as exc:
+                self.params.set_status("FAILURE", "Failed to run environment provider")
+                self.logger.error(
+                    "Environment provider has failed in creating an environment for test.",
+                    extra={"user_log": True},
+                )
+                span.set_attribute("result", traceback.format_exc())
+                raise
+            if result.get("error") is not None:
+                self.params.set_status("FAILURE", result.get("error"))
+                self.logger.error(
+                    "Environment provider has failed in creating an environment for test.",
+                    extra={"user_log": True},
+                )
+            else:
+                self.params.set_status("SUCCESS", result.get("error"))
+                self.logger.info(
+                    "Environment provider has finished creating an environment for test.",
+                    extra={"user_log": True},
+                )
 
     def _release_environment(self) -> None:
         """Release an environment from the environment provider."""
         # TODO: We should remove jsontas as a requirement for this function.
         # Passing variables as keyword argument to make it easier to transition to a function where
         # jsontas is not required.
         jsontas = JsonTas()
-        status, message = release_full_environment(
-            etos=self.etos, jsontas=jsontas, suite_id=self.params.tercc.meta.event_id
-        )
-        if not status:
-            self.logger.error(message)
+        span_name = "release_full_environment"
+        suite_context = get_current_context()
+        with self.otel_tracer.start_as_current_span(span_name, context=suite_context) as span:
+            status, message = release_full_environment(
+                etos=self.etos, jsontas=jsontas, suite_id=self.params.tercc.meta.event_id
+            )
+            if not status:
+                self.logger.error(message)
 
     def run_suites(self, triggered: EiffelActivityTriggeredEvent) -> list[str]:
         """Start up a suite runner handling multiple suites that execute within test runners.
@@ -117,13 +129,11 @@ def run_suites(self, triggered: EiffelActivityTriggeredEvent) -> list[str]:
             "ESR Docker", {"CONTEXT": context}, image=os.getenv("SUITE_RUNNER")
         )
         runner = SuiteRunner(self.params, self.etos)
-
         ids = []
         for suite in self.params.test_suite:
             suite["test_suite_started_id"] = str(uuid4())
             ids.append(suite["test_suite_started_id"])
         self.logger.info("Number of test suites to run: %d", len(ids), extra={"user_log": True})
-
         try:
             self.logger.info("Get test environment.")
             threading.Thread(
@@ -176,7 +186,6 @@ def run(self) -> list[str]:
                 executionType="AUTOMATED",
                 triggers=[{"type": "EIFFEL_EVENT"}],
             )
-
             self.verify_input()
             context = triggered.meta.event_id
         except:  # noqa

projects/etos_suite_runner/src/etos_suite_runner/lib/executor.py

@@ -16,15 +16,17 @@
 """Executor handler module."""
 import logging
 import os
-from json import JSONDecodeError
+from json import JSONDecodeError, dumps
 from typing import Union
 
 from cryptography.fernet import Fernet
 from etos_lib import ETOS
+from opentelemetry import trace
 from requests.auth import HTTPBasicAuth, HTTPDigestAuth
 from requests.exceptions import ConnectionError as RequestsConnectionError
 from requests.exceptions import HTTPError
 
+from ..otel_tracing import get_current_context
 
 class TestStartException(Exception):
     """Exception when starting tests."""
@@ -47,6 +49,7 @@ def __init__(self, etos: ETOS) -> None:
         """
         self.etos = etos
         self.etos.config.set("build_urls", [])
+        self.tracer = trace.get_tracer(__name__)
 
     def __decrypt(self, password: Union[str, dict]) -> str:
         """Decrypt a password using an encryption key.
@@ -89,15 +92,20 @@ def run_tests(self, test_suite: dict) -> None:
         if request.get("auth"):
             request["auth"] = self.__auth(**request["auth"])
         method = getattr(self.etos.http, request.pop("method").lower())
-        try:
-            response = method(**request)
-            response.raise_for_status()
-        except HTTPError as http_error:
+        span_name = "start_execution_space"
+        with self.tracer.start_as_current_span(span_name) as span:
+            span.set_attribute("executor_id", executor['id'])
+            span.set_attribute("request", dumps(request, indent=4))
             try:
-                raise TestStartException(http_error.response.json()) from http_error
-            except JSONDecodeError:
-                raise TestStartException({"error": http_error.response.text}) from http_error
-        except RequestsConnectionError as connection_error:
-            raise TestStartException({"error": str(connection_error)}) from connection_error
-        self.logger.info("%r", response)
-        self.logger.debug("%r", response.text)
+                response = method(**request)
+                response.raise_for_status()
+            except HTTPError as http_error:
+                span.set_attribute("http_error", str(http_error))
+                try:
+                    raise TestStartException(http_error.response.json()) from http_error
+                except JSONDecodeError:
+                    raise TestStartException({"error": http_error.response.text}) from http_error
+            except RequestsConnectionError as connection_error:
+                raise TestStartException({"error": str(connection_error)}) from connection_error
+            self.logger.info("%r", response)
+            self.logger.debug("%r", response.text)

projects/etos_suite_runner/src/etos_suite_runner/lib/runner.py

@@ -20,8 +20,10 @@
 from environment_provider.environment import release_full_environment
 from etos_lib.logging.logger import FORMAT_CONFIG
 from jsontas.jsontas import JsonTas
+import opentelemetry
 
 from .exceptions import EnvironmentProviderException
+from ..otel_tracing import get_current_context
 from .suite import TestSuite
 
 
@@ -44,24 +46,28 @@ def __init__(self, params, etos):
         """
         self.params = params
         self.etos = etos
+        self.otel_tracer = opentelemetry.trace.get_tracer(__name__)
+        self.otel_suite_context = get_current_context()
 
     def _release_environment(self):
         """Release an environment from the environment provider."""
         # TODO: We should remove jsontas as a requirement for this function.
         # Passing variables as keyword argument to make it easier to transition to a function where
         # jsontas is not required.
         jsontas = JsonTas()
-        status, message = release_full_environment(
-            etos=self.etos, jsontas=jsontas, suite_id=self.params.tercc.meta.event_id
-        )
-        if not status:
-            self.logger.error(message)
+        span_name = "release_full_environment"
+        with self.otel_tracer.start_as_current_span(span_name, context=self.otel_suite_context):
+            status, message = release_full_environment(
+                etos=self.etos, jsontas=jsontas, suite_id=self.params.tercc.meta.event_id
+            )
+            if not status:
+                self.logger.error(message)
 
     def start_suites_and_wait(self):
         """Get environments and start all test suites."""
         try:
             test_suites = [
-                TestSuite(self.etos, self.params, suite) for suite in self.params.test_suite
+                TestSuite(self.etos, self.params, suite, otel_context=self.otel_suite_context) for suite in self.params.test_suite
             ]
             with ThreadPool() as pool:
                 pool.map(self.run, test_suites)