instruments: open_telemetry: port auto_instrumentation

- auto_instrumentation.initialize() uses a distro for loading instrumentation. This overrides some
needed defaults, that we use in our loader. E.g. it starts exporting telemetry somewhere
undesirable. So these changes port the loader to our infrastructure.

Author: danberdev

microbootstrap/instruments/opentelemetry_instrument.py

@@ -5,19 +5,25 @@
 import typing
 
 import pydantic
+import structlog
 from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
-from opentelemetry.instrumentation import auto_instrumentation
+from opentelemetry.instrumentation.dependencies import DependencyConflictError
+from opentelemetry.instrumentation.environment_variables import OTEL_PYTHON_DISABLED_INSTRUMENTATIONS
 from opentelemetry.instrumentation.instrumentor import BaseInstrumentor  # type: ignore[attr-defined] # noqa: TC002
 from opentelemetry.sdk import resources
 from opentelemetry.sdk.trace import ReadableSpan, Span, SpanProcessor
 from opentelemetry.sdk.trace import TracerProvider as SdkTracerProvider
 from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter, SimpleSpanProcessor
 from opentelemetry.semconv.resource import ResourceAttributes
 from opentelemetry.trace import format_span_id, set_tracer_provider
+from opentelemetry.util._importlib_metadata import entry_points
 
 from microbootstrap.instruments.base import BaseInstrumentConfig, Instrument
 
 
+LOGGER_OBJ = structlog.get_logger(__name__)
+
+
 try:
     import pyroscope  # type: ignore[import-untyped]
 except ImportError:  # pragma: no cover
@@ -76,6 +82,38 @@ def _format_span(readable_span: ReadableSpan) -> str:
     return typing.cast("str", readable_span.to_json(indent=None)) + os.linesep
 
 
+def _load_instrumentors(tracer_provider: TracerProvider) -> None:
+    package_to_exclude: typing.Final = [
+        one_package_to_exclude.strip()
+        for one_package_to_exclude in os.environ.get(OTEL_PYTHON_DISABLED_INSTRUMENTATIONS, "").split(",")
+    ]
+
+    for entry_point in entry_points(group="opentelemetry_instrumentor"):
+        if entry_point.name in package_to_exclude:
+            LOGGER_OBJ.info("Instrumentation skipped for library %s", entry_point.name)
+            continue
+
+        try:
+            entry_point.load()().instrument(tracer_provider=tracer_provider)
+            LOGGER_OBJ.info("Instrumented %s", entry_point.name)
+        except DependencyConflictError as exc:
+            LOGGER_OBJ.warning(
+                "Skipping instrumentation %s: %s",
+                entry_point.name,
+                exc.conflict,
+            )
+            continue
+        except ModuleNotFoundError as exc:
+            LOGGER_OBJ.warning("Skipping instrumentation %s: %s", entry_point.name, exc.msg)
+            continue
+        except ImportError:
+            LOGGER_OBJ.warning("Importing of %s failed, skipping it", entry_point.name)
+            continue
+        except Exception:
+            LOGGER_OBJ.warning("Instrumenting of %s failed", entry_point.name)
+            raise
+
+
 class BaseOpentelemetryInstrument(Instrument[OpentelemetryConfigT]):
     instrument_name = "Opentelemetry"
     ready_condition = "Provide all necessary config parameters"
@@ -88,8 +126,6 @@ def teardown(self) -> None:
             instrumentor_with_params.instrumentor.uninstrument(**instrumentor_with_params.additional_params)
 
     def bootstrap(self) -> None:
-        auto_instrumentation.initialize()
-
         attributes = {
             ResourceAttributes.SERVICE_NAME: self.instrument_config.opentelemetry_service_name
             or self.instrument_config.service_name,
@@ -123,6 +159,8 @@ def bootstrap(self) -> None:
                 **opentelemetry_instrumentor.additional_params,
             )
         set_tracer_provider(self.tracer_provider)
+        _load_instrumentors(tracer_provider=self.tracer_provider)
+        set_tracer_provider(self.tracer_provider)
 
 
 class OpentelemetryInstrument(BaseOpentelemetryInstrument[OpentelemetryConfig]):

tests/conftest.py

@@ -130,5 +130,5 @@ def reset_reloaded_settings_module() -> typing.Iterator[None]:
 
 
 @pytest.fixture(autouse=True)
-def disable_auto_instrumentation(monkeypatch: pytest.MonkeyPatch) -> None:
-    monkeypatch.setattr(opentelemetry_instrument, "auto_instrumentation", MagicMock())
+def patch_out_entry_points(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(opentelemetry_instrument, "entry_points", MagicMock(retrun_value=[]))

tests/instruments/test_opentelemetry.py

@@ -1,4 +1,5 @@
 import contextlib
+import os
 import typing
 from unittest.mock import AsyncMock, MagicMock, Mock, patch
 
@@ -8,51 +9,54 @@
 from fastapi.testclient import TestClient as FastAPITestClient
 from litestar.middleware.base import DefineMiddleware
 from litestar.testing import TestClient as LitestarTestClient
+from opentelemetry.instrumentation.dependencies import DependencyConflictError
+from opentelemetry.instrumentation.environment_variables import OTEL_PYTHON_DISABLED_INSTRUMENTATIONS
 
 from microbootstrap import OpentelemetryConfig
 from microbootstrap.bootstrappers.fastapi import FastApiOpentelemetryInstrument
 from microbootstrap.bootstrappers.litestar import LitestarOpentelemetryInstrument
+from microbootstrap.instruments import opentelemetry_instrument
 from microbootstrap.instruments.opentelemetry_instrument import OpentelemetryInstrument
 
 
 def test_opentelemetry_is_ready(
     minimal_opentelemetry_config: OpentelemetryConfig,
 ) -> None:
-    opentelemetry_instrument: typing.Final = OpentelemetryInstrument(minimal_opentelemetry_config)
-    assert opentelemetry_instrument.is_ready()
+    test_opentelemetry_instrument: typing.Final = OpentelemetryInstrument(minimal_opentelemetry_config)
+    assert test_opentelemetry_instrument.is_ready()
 
 
 def test_opentelemetry_bootstrap_is_not_ready(minimal_opentelemetry_config: OpentelemetryConfig) -> None:
     minimal_opentelemetry_config.service_debug = False
     minimal_opentelemetry_config.opentelemetry_endpoint = None
-    opentelemetry_instrument: typing.Final = OpentelemetryInstrument(minimal_opentelemetry_config)
-    assert not opentelemetry_instrument.is_ready()
+    test_opentelemetry_instrument: typing.Final = OpentelemetryInstrument(minimal_opentelemetry_config)
+    assert not test_opentelemetry_instrument.is_ready()
 
 
 def test_opentelemetry_bootstrap_after(
     default_litestar_app: litestar.Litestar,
     minimal_opentelemetry_config: OpentelemetryConfig,
 ) -> None:
-    opentelemetry_instrument: typing.Final = OpentelemetryInstrument(minimal_opentelemetry_config)
-    assert opentelemetry_instrument.bootstrap_after(default_litestar_app) == default_litestar_app
+    test_opentelemetry_instrument: typing.Final = OpentelemetryInstrument(minimal_opentelemetry_config)
+    assert test_opentelemetry_instrument.bootstrap_after(default_litestar_app) == default_litestar_app
 
 
 def test_opentelemetry_teardown(
     minimal_opentelemetry_config: OpentelemetryConfig,
 ) -> None:
-    opentelemetry_instrument: typing.Final = OpentelemetryInstrument(minimal_opentelemetry_config)
-    assert opentelemetry_instrument.teardown() is None  # type: ignore[func-returns-value]
+    test_opentelemetry_instrument: typing.Final = OpentelemetryInstrument(minimal_opentelemetry_config)
+    assert test_opentelemetry_instrument.teardown() is None  # type: ignore[func-returns-value]
 
 
 def test_litestar_opentelemetry_bootstrap(
     minimal_opentelemetry_config: OpentelemetryConfig,
     magic_mock: MagicMock,
 ) -> None:
     minimal_opentelemetry_config.opentelemetry_instrumentors = [magic_mock]
-    opentelemetry_instrument: typing.Final = LitestarOpentelemetryInstrument(minimal_opentelemetry_config)
+    test_opentelemetry_instrument: typing.Final = LitestarOpentelemetryInstrument(minimal_opentelemetry_config)
 
-    opentelemetry_instrument.bootstrap()
-    opentelemetry_bootstrap_result: typing.Final = opentelemetry_instrument.bootstrap_before()
+    test_opentelemetry_instrument.bootstrap()
+    opentelemetry_bootstrap_result: typing.Final = test_opentelemetry_instrument.bootstrap_before()
 
     assert opentelemetry_bootstrap_result
     assert "middleware" in opentelemetry_bootstrap_result
@@ -66,18 +70,18 @@ def test_litestar_opentelemetry_teardown(
     magic_mock: MagicMock,
 ) -> None:
     minimal_opentelemetry_config.opentelemetry_instrumentors = [magic_mock]
-    opentelemetry_instrument: typing.Final = LitestarOpentelemetryInstrument(minimal_opentelemetry_config)
+    test_opentelemetry_instrument: typing.Final = LitestarOpentelemetryInstrument(minimal_opentelemetry_config)
 
-    opentelemetry_instrument.teardown()
+    test_opentelemetry_instrument.teardown()
 
 
 def test_litestar_opentelemetry_bootstrap_working(
     minimal_opentelemetry_config: OpentelemetryConfig,
     async_mock: AsyncMock,
 ) -> None:
-    opentelemetry_instrument: typing.Final = LitestarOpentelemetryInstrument(minimal_opentelemetry_config)
-    opentelemetry_instrument.bootstrap()
-    opentelemetry_bootstrap_result: typing.Final = opentelemetry_instrument.bootstrap_before()
+    test_opentelemetry_instrument: typing.Final = LitestarOpentelemetryInstrument(minimal_opentelemetry_config)
+    test_opentelemetry_instrument.bootstrap()
+    opentelemetry_bootstrap_result: typing.Final = test_opentelemetry_instrument.bootstrap_before()
 
     opentelemetry_middleware = opentelemetry_bootstrap_result["middleware"][0]
     assert isinstance(opentelemetry_middleware, DefineMiddleware)
@@ -104,9 +108,9 @@ def test_fastapi_opentelemetry_bootstrap_working(
 ) -> None:
     monkeypatch.setattr("opentelemetry.sdk.trace.TracerProvider.shutdown", Mock())
 
-    opentelemetry_instrument: typing.Final = FastApiOpentelemetryInstrument(minimal_opentelemetry_config)
-    opentelemetry_instrument.bootstrap()
-    fastapi_application: typing.Final = opentelemetry_instrument.bootstrap_after(fastapi.FastAPI())
+    test_opentelemetry_instrument: typing.Final = FastApiOpentelemetryInstrument(minimal_opentelemetry_config)
+    test_opentelemetry_instrument.bootstrap()
+    fastapi_application: typing.Final = test_opentelemetry_instrument.bootstrap_after(fastapi.FastAPI())
 
     @fastapi_application.get("/test-handler")
     async def test_handler() -> None:
@@ -115,3 +119,47 @@ async def test_handler() -> None:
     with patch("opentelemetry.trace.use_span") as mock_capture_event:
         FastAPITestClient(app=fastapi_application).get("/test-handler")
         assert mock_capture_event.called
+
+
+@pytest.mark.parametrize(
+    ("instruments", "result"),
+    [
+        (
+            [
+                MagicMock(),
+                MagicMock(load=MagicMock(side_effect=ImportError)),
+                MagicMock(load=MagicMock(side_effect=DependencyConflictError("Hello"))),
+                MagicMock(load=MagicMock(side_effect=ModuleNotFoundError)),
+            ],
+            "ok",
+        ),
+        (
+            [
+                MagicMock(load=MagicMock(side_effect=ValueError)),
+            ],
+            "raise",
+        ),
+        (
+            [
+                MagicMock(load=MagicMock(side_effect=ValueError)),
+            ],
+            "exclude",
+        ),
+    ],
+)
+def test_instrumentator_loader(instruments: list[MagicMock], result: str, monkeypatch: pytest.MonkeyPatch) -> None:
+    if result == "exclude":
+        monkeypatch.setenv(OTEL_PYTHON_DISABLED_INSTRUMENTATIONS, "exclude_this,exclude_that")
+        instruments[0].name = "exclude_this"
+    monkeypatch.setattr(
+        opentelemetry_instrument,
+        "entry_points",
+        MagicMock(return_value=[*instruments]),
+    )
+
+    if result != "raise":
+        opentelemetry_instrument._load_instrumentors(MagicMock())  # noqa: SLF001
+        return
+
+    with pytest.raises(ValueError):  # noqa: PT011
+        opentelemetry_instrument._load_instrumentors(MagicMock())  # noqa: SLF001

tests/instruments/test_pyroscope.py

@@ -50,7 +50,6 @@ def test_opentelemetry_includes_pyroscope_2(
         async def test_handler() -> None: ...
 
         FastAPITestClient(app=fastapi_application).get("/test-handler")
-
         assert (
             add_thread_tag_mock.mock_calls
             == remove_thread_tag_mock.mock_calls