[FR] Add tracing support (#6211)

* [FR] Add tracing support
Fixes #6208

* move checks out of manage.py

* fixed reqs

* small cleanup

* move tracing init to settings
similar to how sentry is handled

* rephrase

* clean up imports

* added argument regarding console debugging

* fix typing

* added auth section

* remove empty headers

* made protocol configurable

* rename vars & cleanup template

* more docs for template

* add docs

InvenTree/InvenTree/settings.py

@@ -26,6 +26,7 @@ from dotenv import load_dotenv
 
 from InvenTree.config import get_boolean_setting, get_custom_file, get_setting
 from InvenTree.sentry import default_sentry_dsn, init_sentry
+from InvenTree.tracing import setup_instruments, setup_tracing
 from InvenTree.version import checkMinPythonVersion, inventreeApiVersion
 
 from . import config, locales
@@ -711,6 +712,14 @@ REMOTE_LOGIN_HEADER = get_setting(
     'INVENTREE_REMOTE_LOGIN_HEADER', 'remote_login_header', 'REMOTE_USER'
 )
 
+# region Tracing / error tracking
+inventree_tags = {
+    'testing': TESTING,
+    'docker': DOCKER,
+    'debug': DEBUG,
+    'remote': REMOTE_LOGIN,
+}
+
 # sentry.io integration for error reporting
 SENTRY_ENABLED = get_boolean_setting(
     'INVENTREE_SENTRY_ENABLED', 'sentry_enabled', False
@@ -723,15 +732,42 @@ SENTRY_SAMPLE_RATE = float(
 )
 
 if SENTRY_ENABLED and SENTRY_DSN:  # pragma: no cover
-    inventree_tags = {
-        'testing': TESTING,
-        'docker': DOCKER,
-        'debug': DEBUG,
-        'remote': REMOTE_LOGIN,
-    }
-
     init_sentry(SENTRY_DSN, SENTRY_SAMPLE_RATE, inventree_tags)
 
+# OpenTelemetry tracing
+TRACING_ENABLED = get_boolean_setting(
+    'INVENTREE_TRACING_ENABLED', 'tracing.enabled', False
+)
+if TRACING_ENABLED:  # pragma: no cover
+    _t_endpoint = get_setting('INVENTREE_TRACING_ENDPOINT', 'tracing.endpoint', None)
+    _t_headers = get_setting('INVENTREE_TRACING_HEADERS', 'tracing.headers', None, dict)
+    if _t_endpoint and _t_headers:
+        _t_resources = get_setting(
+            'INVENTREE_TRACING_RESOURCES', 'tracing.resources', {}, dict
+        )
+        cstm_tags = {'inventree.env.' + k: v for k, v in inventree_tags.items()}
+        tracing_resources = {**cstm_tags, **_t_resources}
+
+        setup_tracing(
+            _t_endpoint,
+            _t_headers,
+            tracing_resources,
+            get_boolean_setting('INVENTREE_TRACING_CONSOLE', 'tracing.console', False),
+            get_setting('INVENTREE_TRACING_AUTH', 'tracing.auth', {}),
+            get_setting('INVENTREE_TRACING_IS_HTTP', 'tracing.is_http', False),
+            get_boolean_setting(
+                'INVENTREE_TRACING_APPEND_HTTP', 'tracing.append_http', True
+            ),
+        )
+        # Run tracing/logging instrumentation
+        setup_instruments()
+    else:
+        logger.warning(
+            'OpenTelemetry tracing not enabled because endpoint or headers are not set'
+        )
+
+# endregion
+
 # Cache configuration
 cache_host = get_setting('INVENTREE_CACHE_HOST', 'cache.host', None)
 cache_port = get_setting('INVENTREE_CACHE_PORT', 'cache.port', '6379', typecast=int)

InvenTree/InvenTree/tracing.py

@@ -0,0 +1,142 @@
+"""OpenTelemetry setup functions."""
+
+import base64
+import logging
+from typing import Optional
+
+from opentelemetry import metrics, trace
+from opentelemetry.instrumentation.django import DjangoInstrumentor
+from opentelemetry.instrumentation.redis import RedisInstrumentor
+from opentelemetry.instrumentation.requests import RequestsInstrumentor
+from opentelemetry.sdk import _logs as logs
+from opentelemetry.sdk import resources
+from opentelemetry.sdk._logs import export as logs_export
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.metrics.export import (
+    ConsoleMetricExporter,
+    PeriodicExportingMetricReader,
+)
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter
+
+from InvenTree.version import inventreeVersion
+
+# Logger configuration
+logger = logging.getLogger('inventree')
+
+
+def setup_tracing(
+    endpoint: str,
+    headers: dict,
+    resources_input: Optional[dict] = None,
+    console: bool = False,
+    auth: Optional[dict] = None,
+    is_http: bool = False,
+    append_http: bool = True,
+):
+    """Set up tracing for the application in the current context.
+
+    Args:
+        endpoint: The endpoint to send the traces to.
+        headers: The headers to send with the traces.
+        resources_input: The resources to send with the traces.
+        console: Whether to output the traces to the console.
+    """
+    if resources_input is None:
+        resources_input = {}
+    if auth is None:
+        auth = {}
+
+    # Setup the auth headers
+    if 'basic' in auth:
+        basic_auth = auth['basic']
+        if 'username' in basic_auth and 'password' in basic_auth:
+            auth_raw = f'{basic_auth["username"]}:{basic_auth["password"]}'
+            auth_token = base64.b64encode(auth_raw.encode('utf-8')).decode('utf-8')
+            headers['Authorization'] = f'Basic {auth_token}'
+        else:
+            logger.warning('Basic auth is missing username or password')
+
+    # Clean up headers
+    headers = {k: v for k, v in headers.items() if v is not None}
+
+    # Initialize the OTLP Resource
+    resource = resources.Resource(
+        attributes={
+            resources.SERVICE_NAME: 'BACKEND',
+            resources.SERVICE_NAMESPACE: 'INVENTREE',
+            resources.SERVICE_VERSION: inventreeVersion(),
+            **resources_input,
+        }
+    )
+
+    # Import the OTLP exporters
+    if is_http:
+        from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
+        from opentelemetry.exporter.otlp.proto.http.metric_exporter import (
+            OTLPMetricExporter,
+        )
+        from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
+            OTLPSpanExporter,
+        )
+    else:
+        from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter
+        from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import (
+            OTLPMetricExporter,
+        )
+        from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import (
+            OTLPSpanExporter,
+        )
+
+    # Spans / Tracs
+    span_exporter = OTLPSpanExporter(
+        headers=headers,
+        endpoint=endpoint if not (is_http and append_http) else f'{endpoint}/v1/traces',
+    )
+    trace_processor = BatchSpanProcessor(span_exporter)
+    trace_provider = TracerProvider(resource=resource)
+    trace.set_tracer_provider(trace_provider)
+    trace_provider.add_span_processor(trace_processor)
+    # For debugging purposes, export the traces to the console
+    if console:
+        trace_provider.add_span_processor(BatchSpanProcessor(ConsoleSpanExporter()))
+
+    # Metrics
+    metric_perodic_reader = PeriodicExportingMetricReader(
+        OTLPMetricExporter(
+            headers=headers,
+            endpoint=endpoint
+            if not (is_http and append_http)
+            else f'{endpoint}/v1/metrics',
+        )
+    )
+    metric_readers = [metric_perodic_reader]
+
+    # For debugging purposes, export the metrics to the console
+    if console:
+        console_metric_exporter = ConsoleMetricExporter()
+        console_metric_reader = PeriodicExportingMetricReader(console_metric_exporter)
+        metric_readers.append(console_metric_reader)
+
+    meter_provider = MeterProvider(resource=resource, metric_readers=metric_readers)
+    metrics.set_meter_provider(meter_provider)
+
+    # Logs
+    log_exporter = OTLPLogExporter(
+        headers=headers,
+        endpoint=endpoint if not (is_http and append_http) else f'{endpoint}/v1/logs',
+    )
+    log_provider = logs.LoggerProvider(resource=resource)
+    log_provider.add_log_record_processor(
+        logs_export.BatchLogRecordProcessor(log_exporter)
+    )
+    handler = logs.LoggingHandler(level=logging.INFO, logger_provider=log_provider)
+    logger = logging.getLogger('inventree')
+    logger.addHandler(handler)
+
+
+def setup_instruments():
+    """Run auto-insturmentation for OpenTelemetry tracing."""
+    DjangoInstrumentor().instrument()
+    RedisInstrumentor().instrument()
+    RequestsInstrumentor().instrument()

InvenTree/config_template.yaml

@@ -136,6 +136,25 @@ sentry_enabled: False
 #sentry_sample_rate: 0.1
 #sentry_dsn: https://custom@custom.ingest.sentry.io/custom
 
+# OpenTelemetry tracing/metrics - disabled by default
+# This can be used to send tracing data, logs and metrics to OpenTelemtry compatible backends
+# See https://opentelemetry.io/ecosystem/vendors/ for a list of supported backends
+# Alternatively, use environment variables eg. INVENTREE_TRACING_ENABLED, INVENTREE_TRACING_HEADERS, INVENTREE_TRACING_AUTH
+#tracing:
+#  enabled: true
+#  endpoint: https://otlp-gateway-prod-eu-west-0.grafana.net/otlp
+#  headers:
+#    api-key: 'sample'
+#  auth:
+#    basic:
+#      username: '******'
+#      password: 'glc_****'
+#  is_http: true
+#  append_http: true
+#  console: false
+#  resources:
+#    CUSTOM_KEY: 'CUSTOM_VALUE'
+
 # Set this variable to True to enable InvenTree Plugins
 # Alternatively, use the environment variable INVENTREE_PLUGINS_ENABLED
 plugins_enabled: False

InvenTree/manage.py

@@ -8,6 +8,7 @@ import sys
 def main():
     """Run administrative tasks."""
     os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'InvenTree.settings')
+
     try:
         from django.core.management import execute_from_command_line
     except ImportError as exc:  # pragma: no cover

docs/docs/start/advanced.md

@@ -71,3 +71,20 @@ Next you can start configuring the connection. Either use the config file or set
 | `ldap.require_group` | `INVENTREE_LDAP_REQUIRE_GROUP` | If set, users _must_ be in this group to log in to InvenTree |
 | `ldap.deny_group` | `INVENTREE_LDAP_DENY_GROUP` | If set, users _must not_ be in this group to log in to InvenTree |
 | `ldap.user_flags_by_group` | `INVENTREE_LDAP_USER_FLAGS_BY_GROUP` | LDAP group to InvenTree user flag map, can be json if used as env, in yml directly specify the object. See config template for example, default: `{}` |
+
+
+## Tracing support
+
+Starting with 0.14.0 InvenTree supports sending traces, logs and metrics to OpenTelemetry compatible endpoints (both HTTP and gRPC). A [list of vendors](https://opentelemetry.io/ecosystem/vendors) is available on the project site.
+This can be used to track usage and performance of the InvenTree backend and connected services like databases, caches and more.
+
+| config key | ENV Variable | Description |
+| --- | --- | --- |
+| `tracing.enabled` | `INVENTREE_TRACING_ENABLED` | Set this to `True` to enable OpenTelemetry. |
+| `tracing.endpoint` | `INVENTREE_TRACING_ENDPOINT` | General endpoint for information (not specific trace/log url) |
+| `tracing.headers` | `INVENTREE_TRACING_HEADERS` | HTTP headers that should be send with every request (often used for authentication). Format as a dict. |
+| `tracing.auth.basic` | `INVENTREE_TRACING_AUTH_BASIC` | Auth headers that should be send with every requests (will be encoded to b64 and overwrite auth headers) |
+| `tracing.is_http` | `INVENTREE_TRACING_IS_HTTP` | Are the endpoints HTTP (True, default) or gRPC (False) |
+| `tracing.append_http` | `INVENTREE_TRACING_APPEND_HTTP` | Append default url routes (v1) to `tracing.endpoint` |
+| `tracing.console` | `INVENTREE_TRACING_CONSOLE` | Print out all exports (additionally) to the console for debugging. Do not use in production |
+| `tracing.resources` | `INVENTREE_TRACING_RESOURCES` | Add additional resources to all exports. This can be used to add custom tags to the traces. Format as a dict. |

requirements.in

@@ -49,3 +49,11 @@ sentry-sdk                              # Error reporting (optional)
 setuptools                              # Standard dependency
 tablib[xls,xlsx,yaml]                   # Support for XLS and XLSX formats
 weasyprint                              # PDF generation
+
+# OpenTelemetry dependencies
+opentelemetry-api
+opentelemetry-sdk
+opentelemetry-exporter-otlp
+opentelemetry-instrumentation-django
+opentelemetry-instrumentation-requests
+opentelemetry-instrumentation-redis

requirements.txt

@@ -14,6 +14,11 @@ attrs==23.1.0
     #   referencing
 babel==2.13.1
     # via py-moneyed
+backoff==2.2.1
+    # via
+    #   opentelemetry-exporter-otlp-proto-common
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
 bleach[css]==6.1.0
     # via
     #   bleach
@@ -45,6 +50,11 @@ defusedxml==0.7.1
     # via
     #   odfpy
     #   python3-openid
+deprecated==1.2.14
+    # via
+    #   opentelemetry-api
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
 diff-match-patch==20230430
     # via django-import-export
 dj-rest-auth==5.0.2
@@ -167,6 +177,12 @@ fonttools[woff]==4.44.0
     # via
     #   fonttools
     #   weasyprint
+googleapis-common-protos==1.62.0
+    # via
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+grpcio==1.60.0
+    # via opentelemetry-exporter-otlp-proto-grpc
 gunicorn==21.2.0
     # via -r requirements.in
 html5lib==1.1
@@ -179,6 +195,7 @@ importlib-metadata==6.8.0
     # via
     #   django-q2
     #   markdown
+    #   opentelemetry-api
 inflection==0.5.1
     # via drf-spectacular
 itypes==1.2.0
@@ -201,6 +218,63 @@ odfpy==1.4.1
     # via tablib
 openpyxl==3.1.2
     # via tablib
+opentelemetry-api==1.22.0
+    # via
+    #   -r requirements.in
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+    #   opentelemetry-instrumentation
+    #   opentelemetry-instrumentation-django
+    #   opentelemetry-instrumentation-redis
+    #   opentelemetry-instrumentation-requests
+    #   opentelemetry-instrumentation-wsgi
+    #   opentelemetry-sdk
+opentelemetry-exporter-otlp==1.22.0
+    # via -r requirements.in
+opentelemetry-exporter-otlp-proto-common==1.22.0
+    # via
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+opentelemetry-exporter-otlp-proto-grpc==1.22.0
+    # via opentelemetry-exporter-otlp
+opentelemetry-exporter-otlp-proto-http==1.22.0
+    # via opentelemetry-exporter-otlp
+opentelemetry-instrumentation==0.43b0
+    # via
+    #   opentelemetry-instrumentation-django
+    #   opentelemetry-instrumentation-redis
+    #   opentelemetry-instrumentation-requests
+    #   opentelemetry-instrumentation-wsgi
+opentelemetry-instrumentation-django==0.43b0
+    # via -r requirements.in
+opentelemetry-instrumentation-redis==0.43b0
+    # via -r requirements.in
+opentelemetry-instrumentation-requests==0.43b0
+    # via -r requirements.in
+opentelemetry-instrumentation-wsgi==0.43b0
+    # via opentelemetry-instrumentation-django
+opentelemetry-proto==1.22.0
+    # via
+    #   opentelemetry-exporter-otlp-proto-common
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+opentelemetry-sdk==1.22.0
+    # via
+    #   -r requirements.in
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+opentelemetry-semantic-conventions==0.43b0
+    # via
+    #   opentelemetry-instrumentation-django
+    #   opentelemetry-instrumentation-redis
+    #   opentelemetry-instrumentation-requests
+    #   opentelemetry-instrumentation-wsgi
+    #   opentelemetry-sdk
+opentelemetry-util-http==0.43b0
+    # via
+    #   opentelemetry-instrumentation-django
+    #   opentelemetry-instrumentation-requests
+    #   opentelemetry-instrumentation-wsgi
 packaging==23.2
     # via gunicorn
 pdf2image==1.16.3
@@ -215,6 +289,10 @@ pillow==10.1.0
     #   weasyprint
 pint==0.21
     # via -r requirements.in
+protobuf==4.25.2
+    # via
+    #   googleapis-common-protos
+    #   opentelemetry-proto
 py-moneyed==3.0
     # via django-money
 pycparser==2.21
@@ -271,6 +349,7 @@ requests==2.31.0
     # via
     #   coreapi
     #   django-allauth
+    #   opentelemetry-exporter-otlp-proto-http
     #   requests-oauthlib
 requests-oauthlib==1.3.1
     # via django-allauth
@@ -305,6 +384,7 @@ tinycss2==1.2.1
 typing-extensions==4.8.0
     # via
     #   asgiref
+    #   opentelemetry-sdk
     #   py-moneyed
     #   qrcode
 uritemplate==4.1.1
@@ -326,6 +406,11 @@ webencodings==0.5.1
     #   cssselect2
     #   html5lib
     #   tinycss2
+wrapt==1.16.0
+    # via
+    #   deprecated
+    #   opentelemetry-instrumentation
+    #   opentelemetry-instrumentation-redis
 xlrd==2.0.1
     # via tablib
 xlwt==1.3.0