[FR] Add tracing support (#6211)

* [FR] Add tracing support
Fixes #6208

* move checks out of manage.py

* fixed reqs

* small cleanup

* move tracing init to settings
similar to how sentry is handled

* rephrase

* clean up imports

* added argument regarding console debugging

* fix typing

* added auth section

* remove empty headers

* made protocol configurable

* rename vars & cleanup template

* more docs for template

* add docs

InvenTree/InvenTree/settings.py

@@ -26,6 +26,7 @@ from dotenv import load_dotenv
 
 from InvenTree.config import get_boolean_setting, get_custom_file, get_setting
 from InvenTree.sentry import default_sentry_dsn, init_sentry
+from InvenTree.tracing import setup_instruments, setup_tracing
 from InvenTree.version import checkMinPythonVersion, inventreeApiVersion
 
 from . import config, locales
@@ -711,6 +712,14 @@ REMOTE_LOGIN_HEADER = get_setting(
     'INVENTREE_REMOTE_LOGIN_HEADER', 'remote_login_header', 'REMOTE_USER'
 )
 
+# region Tracing / error tracking
+inventree_tags = {
+    'testing': TESTING,
+    'docker': DOCKER,
+    'debug': DEBUG,
+    'remote': REMOTE_LOGIN,
+}
+
 # sentry.io integration for error reporting
 SENTRY_ENABLED = get_boolean_setting(
     'INVENTREE_SENTRY_ENABLED', 'sentry_enabled', False
@@ -723,15 +732,42 @@ SENTRY_SAMPLE_RATE = float(
 )
 
 if SENTRY_ENABLED and SENTRY_DSN:  # pragma: no cover
-    inventree_tags = {
-        'testing': TESTING,
-        'docker': DOCKER,
-        'debug': DEBUG,
-        'remote': REMOTE_LOGIN,
-    }
-
     init_sentry(SENTRY_DSN, SENTRY_SAMPLE_RATE, inventree_tags)
 
+# OpenTelemetry tracing
+TRACING_ENABLED = get_boolean_setting(
+    'INVENTREE_TRACING_ENABLED', 'tracing.enabled', False
+)
+if TRACING_ENABLED:  # pragma: no cover
+    _t_endpoint = get_setting('INVENTREE_TRACING_ENDPOINT', 'tracing.endpoint', None)
+    _t_headers = get_setting('INVENTREE_TRACING_HEADERS', 'tracing.headers', None, dict)
+    if _t_endpoint and _t_headers:
+        _t_resources = get_setting(
+            'INVENTREE_TRACING_RESOURCES', 'tracing.resources', {}, dict
+        )
+        cstm_tags = {'inventree.env.' + k: v for k, v in inventree_tags.items()}
+        tracing_resources = {**cstm_tags, **_t_resources}
+
+        setup_tracing(
+            _t_endpoint,
+            _t_headers,
+            tracing_resources,
+            get_boolean_setting('INVENTREE_TRACING_CONSOLE', 'tracing.console', False),
+            get_setting('INVENTREE_TRACING_AUTH', 'tracing.auth', {}),
+            get_setting('INVENTREE_TRACING_IS_HTTP', 'tracing.is_http', False),
+            get_boolean_setting(
+                'INVENTREE_TRACING_APPEND_HTTP', 'tracing.append_http', True
+            ),
+        )
+        # Run tracing/logging instrumentation
+        setup_instruments()
+    else:
+        logger.warning(
+            'OpenTelemetry tracing not enabled because endpoint or headers are not set'
+        )
+
+# endregion
+
 # Cache configuration
 cache_host = get_setting('INVENTREE_CACHE_HOST', 'cache.host', None)
 cache_port = get_setting('INVENTREE_CACHE_PORT', 'cache.port', '6379', typecast=int)

InvenTree/InvenTree/tracing.py

@@ -0,0 +1,142 @@
+"""OpenTelemetry setup functions."""
+
+import base64
+import logging
+from typing import Optional
+
+from opentelemetry import metrics, trace
+from opentelemetry.instrumentation.django import DjangoInstrumentor
+from opentelemetry.instrumentation.redis import RedisInstrumentor
+from opentelemetry.instrumentation.requests import RequestsInstrumentor
+from opentelemetry.sdk import _logs as logs
+from opentelemetry.sdk import resources
+from opentelemetry.sdk._logs import export as logs_export
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.metrics.export import (
+    ConsoleMetricExporter,
+    PeriodicExportingMetricReader,
+)
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter
+
+from InvenTree.version import inventreeVersion
+
+# Logger configuration
+logger = logging.getLogger('inventree')
+
+
+def setup_tracing(
+    endpoint: str,
+    headers: dict,
+    resources_input: Optional[dict] = None,
+    console: bool = False,
+    auth: Optional[dict] = None,
+    is_http: bool = False,
+    append_http: bool = True,
+):
+    """Set up tracing for the application in the current context.
+
+    Args:
+        endpoint: The endpoint to send the traces to.
+        headers: The headers to send with the traces.
+        resources_input: The resources to send with the traces.
+        console: Whether to output the traces to the console.
+    """
+    if resources_input is None:
+        resources_input = {}
+    if auth is None:
+        auth = {}
+
+    # Setup the auth headers
+    if 'basic' in auth:
+        basic_auth = auth['basic']
+        if 'username' in basic_auth and 'password' in basic_auth:
+            auth_raw = f'{basic_auth["username"]}:{basic_auth["password"]}'
+            auth_token = base64.b64encode(auth_raw.encode('utf-8')).decode('utf-8')
+            headers['Authorization'] = f'Basic {auth_token}'
+        else:
+            logger.warning('Basic auth is missing username or password')
+
+    # Clean up headers
+    headers = {k: v for k, v in headers.items() if v is not None}
+
+    # Initialize the OTLP Resource
+    resource = resources.Resource(
+        attributes={
+            resources.SERVICE_NAME: 'BACKEND',
+            resources.SERVICE_NAMESPACE: 'INVENTREE',
+            resources.SERVICE_VERSION: inventreeVersion(),
+            **resources_input,
+        }
+    )
+
+    # Import the OTLP exporters
+    if is_http:
+        from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
+        from opentelemetry.exporter.otlp.proto.http.metric_exporter import (
+            OTLPMetricExporter,
+        )
+        from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
+            OTLPSpanExporter,
+        )
+    else:
+        from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter
+        from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import (
+            OTLPMetricExporter,
+        )
+        from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import (
+            OTLPSpanExporter,
+        )
+
+    # Spans / Tracs
+    span_exporter = OTLPSpanExporter(
+        headers=headers,
+        endpoint=endpoint if not (is_http and append_http) else f'{endpoint}/v1/traces',
+    )
+    trace_processor = BatchSpanProcessor(span_exporter)
+    trace_provider = TracerProvider(resource=resource)
+    trace.set_tracer_provider(trace_provider)
+    trace_provider.add_span_processor(trace_processor)
+    # For debugging purposes, export the traces to the console
+    if console:
+        trace_provider.add_span_processor(BatchSpanProcessor(ConsoleSpanExporter()))
+
+    # Metrics
+    metric_perodic_reader = PeriodicExportingMetricReader(
+        OTLPMetricExporter(
+            headers=headers,
+            endpoint=endpoint
+            if not (is_http and append_http)
+            else f'{endpoint}/v1/metrics',
+        )
+    )
+    metric_readers = [metric_perodic_reader]
+
+    # For debugging purposes, export the metrics to the console
+    if console:
+        console_metric_exporter = ConsoleMetricExporter()
+        console_metric_reader = PeriodicExportingMetricReader(console_metric_exporter)
+        metric_readers.append(console_metric_reader)
+
+    meter_provider = MeterProvider(resource=resource, metric_readers=metric_readers)
+    metrics.set_meter_provider(meter_provider)
+
+    # Logs
+    log_exporter = OTLPLogExporter(
+        headers=headers,
+        endpoint=endpoint if not (is_http and append_http) else f'{endpoint}/v1/logs',
+    )
+    log_provider = logs.LoggerProvider(resource=resource)
+    log_provider.add_log_record_processor(
+        logs_export.BatchLogRecordProcessor(log_exporter)
+    )
+    handler = logs.LoggingHandler(level=logging.INFO, logger_provider=log_provider)
+    logger = logging.getLogger('inventree')
+    logger.addHandler(handler)
+
+
+def setup_instruments():
+    """Run auto-insturmentation for OpenTelemetry tracing."""
+    DjangoInstrumentor().instrument()
+    RedisInstrumentor().instrument()
+    RequestsInstrumentor().instrument()

InvenTree/config_template.yaml

@@ -136,6 +136,25 @@ sentry_enabled: False
 #sentry_sample_rate: 0.1
 #sentry_dsn: https://custom@custom.ingest.sentry.io/custom
 
+# OpenTelemetry tracing/metrics - disabled by default
+# This can be used to send tracing data, logs and metrics to OpenTelemtry compatible backends
+# See https://opentelemetry.io/ecosystem/vendors/ for a list of supported backends
+# Alternatively, use environment variables eg. INVENTREE_TRACING_ENABLED, INVENTREE_TRACING_HEADERS, INVENTREE_TRACING_AUTH
+#tracing:
+#  enabled: true
+#  endpoint: https://otlp-gateway-prod-eu-west-0.grafana.net/otlp
+#  headers:
+#    api-key: 'sample'
+#  auth:
+#    basic:
+#      username: '******'
+#      password: 'glc_****'
+#  is_http: true
+#  append_http: true
+#  console: false
+#  resources:
+#    CUSTOM_KEY: 'CUSTOM_VALUE'
+
 # Set this variable to True to enable InvenTree Plugins
 # Alternatively, use the environment variable INVENTREE_PLUGINS_ENABLED
 plugins_enabled: False

InvenTree/manage.py

@@ -8,6 +8,7 @@ import sys
 def main():
     """Run administrative tasks."""
     os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'InvenTree.settings')
+
     try:
         from django.core.management import execute_from_command_line
     except ImportError as exc:  # pragma: no cover