From 7b181bb5ae9bb42df147a045ad20acaeb15b456b Mon Sep 17 00:00:00 2001 From: Oliver Date: Wed, 17 Dec 2025 20:20:59 +1100 Subject: [PATCH] [API] Query improvements (#11034) * Improve prefetch fields for API * Cache ContentType queryset for getModelsWithMixin - Called a LOT of times for an options request - Store the list in the session cache - Much faster than redis - and expires after the session is complete * Skip optional prefetch for options requests * Custom implementation of DjangoModelPermission - Cache the queryset against the view - Prevents multiple hits for OPTIONS request - Saves > 100ms on /stock/ options request --- src/backend/InvenTree/InvenTree/cache.py | 21 ++++++++++++++ .../InvenTree/InvenTree/helpers_model.py | 29 +++++++++++++------ src/backend/InvenTree/InvenTree/metadata.py | 6 +++- .../InvenTree/InvenTree/permissions.py | 25 +++++++++++++++- .../InvenTree/InvenTree/serializers.py | 15 ++++++++-- src/backend/InvenTree/InvenTree/settings.py | 2 +- src/backend/InvenTree/company/serializers.py | 6 +++- src/backend/InvenTree/order/api.py | 6 ++-- src/backend/InvenTree/order/serializers.py | 13 +++++++-- 9 files changed, 101 insertions(+), 22 deletions(-) diff --git a/src/backend/InvenTree/InvenTree/cache.py b/src/backend/InvenTree/InvenTree/cache.py index 3d56a11815..00214610cc 100644 --- a/src/backend/InvenTree/InvenTree/cache.py +++ b/src/backend/InvenTree/InvenTree/cache.py @@ -4,6 +4,8 @@ import socket import threading from typing import Any +from django.db.utils import OperationalError, ProgrammingError + import structlog import InvenTree.config @@ -169,3 +171,22 @@ def set_session_cache(key: str, value: Any) -> None: if request_cache is not None: request_cache[key] = value + + +def get_cached_content_types(cache_key: str = 'all_content_types') -> list: + """Return a list of all ContentType objects, using session cache if possible.""" + from django.contrib.contenttypes.models import ContentType + + # Attempt to retrieve a list of ContentType objects from session cache + if content_types := get_session_cache(cache_key): + return content_types + + try: + content_types = list(ContentType.objects.all()) + if len(content_types) > 0: + set_session_cache(cache_key, content_types) + except (OperationalError, ProgrammingError): + # Database is likely not yet ready + content_types = [] + + return content_types diff --git a/src/backend/InvenTree/InvenTree/helpers_model.py b/src/backend/InvenTree/InvenTree/helpers_model.py index 2756b7b33a..5fa5ae3cea 100644 --- a/src/backend/InvenTree/InvenTree/helpers_model.py +++ b/src/backend/InvenTree/InvenTree/helpers_model.py @@ -24,6 +24,11 @@ from common.notifications import ( trigger_notification, ) from common.settings import get_global_setting +from InvenTree.cache import ( + get_cached_content_types, + get_session_cache, + set_session_cache, +) from InvenTree.format import format_money from InvenTree.ready import ignore_ready_warning @@ -270,17 +275,23 @@ def getModelsWithMixin(mixin_class) -> list: Returns: List of models that inherit from the given mixin class """ - from django.contrib.contenttypes.models import ContentType + # First, look in the session cache - to prevent repeated expensive comparisons + cache_key = f'models_with_mixin_{mixin_class.__name__}' - try: - db_models = [ - x.model_class() for x in ContentType.objects.all() if x is not None - ] - except (OperationalError, ProgrammingError): - # Database is likely not yet ready - db_models = [] + if cached_models := get_session_cache(cache_key): + return cached_models - return [x for x in db_models if x is not None and issubclass(x, mixin_class)] + content_types = get_cached_content_types() + + db_models = [x.model_class() for x in content_types if x is not None] + + models_with_mixin = [ + x for x in db_models if x is not None and issubclass(x, mixin_class) + ] + + # Store the result in the session cache + set_session_cache(cache_key, models_with_mixin) + return models_with_mixin def notify_responsible( diff --git a/src/backend/InvenTree/InvenTree/metadata.py b/src/backend/InvenTree/InvenTree/metadata.py index e7fb922d48..c45c20e832 100644 --- a/src/backend/InvenTree/InvenTree/metadata.py +++ b/src/backend/InvenTree/InvenTree/metadata.py @@ -23,7 +23,7 @@ logger = structlog.get_logger('inventree') class InvenTreeMetadata(SimpleMetadata): """Custom metadata class for the DRF API. - This custom metadata class imits the available "actions", + This custom metadata class limits the available "actions", based on the user's role permissions. Thus when a client send an OPTIONS request to an API endpoint, @@ -50,6 +50,10 @@ class InvenTreeMetadata(SimpleMetadata): for method in {'PUT', 'POST', 'GET'} & set(view.allowed_methods): view.request = clone_request(request, method) + + # Mark this request, to prevent expensive prefetching + view.request._metadata_requested = True + try: # Test global permissions if hasattr(view, 'check_permissions'): diff --git a/src/backend/InvenTree/InvenTree/permissions.py b/src/backend/InvenTree/InvenTree/permissions.py index bc48d862ea..bf4335e94c 100644 --- a/src/backend/InvenTree/InvenTree/permissions.py +++ b/src/backend/InvenTree/InvenTree/permissions.py @@ -149,7 +149,7 @@ class InvenTreeRoleScopeMixin(OASTokenMixin): class InvenTreeTokenMatchesOASRequirements(InvenTreeRoleScopeMixin): """Combines InvenTree role-based scope handling with OpenAPI schema token requirements. - Usesd as default permission class. + Used as default permission class. """ def has_permission(self, request, view): @@ -166,6 +166,29 @@ class InvenTreeTokenMatchesOASRequirements(InvenTreeRoleScopeMixin): return True +class ModelPermission(permissions.DjangoModelPermissions): + """Custom ModelPermission implementation which provides cached lookup of queryset. + + This is entirely for optimization purposes. + """ + + def _queryset(self, view): + """Return the queryset associated with this view, with caching. + + This is because in a metadata OPTIONS request, the view is copied multiple times. + We can cache the queryset to avoid repeated calculation. + """ + if getattr(view, '_cached_queryset', None) is not None: + return view._cached_queryset + + queryset = super()._queryset(view) + + if queryset is not None: + view._cached_queryset = queryset + + return queryset + + class RolePermission(InvenTreeRoleScopeMixin, permissions.BasePermission): """Role mixin for API endpoints, allowing us to specify the user "role" which is required for certain operations. diff --git a/src/backend/InvenTree/InvenTree/serializers.py b/src/backend/InvenTree/InvenTree/serializers.py index 6c21d9b2d6..3fc8e04db5 100644 --- a/src/backend/InvenTree/InvenTree/serializers.py +++ b/src/backend/InvenTree/InvenTree/serializers.py @@ -133,6 +133,15 @@ class FilterableSerializerMixin: Returns: The modified queryset with prefetching applied. """ + # If we are inside an OPTIONS request, DO NOT PREFETCH + if request := getattr(self, 'request', None): + if method := getattr(request, 'method', None): + if str(method).lower() == 'options': + return queryset + + if getattr(request, '_metadata_requested', False): + return queryset + # Gather up the set of simple 'prefetch' fields and functions prefetch_fields = set() @@ -797,6 +806,8 @@ class ContentTypeField(serializers.ChoiceField): Args: mixin_class: Optional mixin class to restrict valid content types. """ + from InvenTree.cache import get_cached_content_types + self.mixin_class = mixin_class # Override the 'choices' field, to limit to the appropriate models @@ -811,7 +822,7 @@ class ContentTypeField(serializers.ChoiceField): for model in models ] else: - content_types = ContentType.objects.all() + content_types = get_cached_content_types() kwargs['choices'] = [ (f'{ct.app_label}.{ct.model}', str(ct)) for ct in content_types @@ -828,8 +839,6 @@ class ContentTypeField(serializers.ChoiceField): def to_internal_value(self, data): """Convert string representation back to ContentType instance.""" - from django.contrib.contenttypes.models import ContentType - content_type = None if data in ['', None]: diff --git a/src/backend/InvenTree/InvenTree/settings.py b/src/backend/InvenTree/InvenTree/settings.py index 7becfe63c5..bdb8ababa9 100644 --- a/src/backend/InvenTree/InvenTree/settings.py +++ b/src/backend/InvenTree/InvenTree/settings.py @@ -592,7 +592,7 @@ REST_FRAMEWORK = { 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination', 'DEFAULT_PERMISSION_CLASSES': [ 'rest_framework.permissions.IsAuthenticated', - 'rest_framework.permissions.DjangoModelPermissions', + 'InvenTree.permissions.ModelPermission', 'InvenTree.permissions.RolePermission', 'InvenTree.permissions.InvenTreeTokenMatchesOASRequirements', ], diff --git a/src/backend/InvenTree/company/serializers.py b/src/backend/InvenTree/company/serializers.py index d68e3973fd..b3c936ae49 100644 --- a/src/backend/InvenTree/company/serializers.py +++ b/src/backend/InvenTree/company/serializers.py @@ -268,7 +268,11 @@ class ManufacturerPartSerializer( source='part', many=False, read_only=True, allow_null=True ), True, - prefetch_fields=['part'], + prefetch_fields=[ + Prefetch( + 'part', queryset=part.models.Part.objects.select_related('pricing_data') + ) + ], ) pretty_name = enable_filter( diff --git a/src/backend/InvenTree/order/api.py b/src/backend/InvenTree/order/api.py index a81e338084..899f2b4411 100644 --- a/src/backend/InvenTree/order/api.py +++ b/src/backend/InvenTree/order/api.py @@ -1501,10 +1501,10 @@ class ReturnOrderMixin(SerializerContextMixin): def get_queryset(self, *args, **kwargs): """Return annotated queryset for this endpoint.""" queryset = super().get_queryset(*args, **kwargs) - - queryset = queryset.prefetch_related('customer', 'created_by') - queryset = serializers.ReturnOrderSerializer.annotate_queryset(queryset) + queryset = queryset.prefetch_related( + 'contact', 'created_by', 'customer', 'responsible' + ) return queryset diff --git a/src/backend/InvenTree/order/serializers.py b/src/backend/InvenTree/order/serializers.py index e8ca521915..33e5b6cf6f 100644 --- a/src/backend/InvenTree/order/serializers.py +++ b/src/backend/InvenTree/order/serializers.py @@ -1298,9 +1298,6 @@ class SalesOrderShipmentSerializer( @staticmethod def annotate_queryset(queryset): """Annotate the queryset with extra information.""" - # Prefetch related objects - queryset = queryset.prefetch_related('order', 'order__customer', 'allocations') - queryset = queryset.annotate(allocated_items=SubqueryCount('allocations')) return queryset @@ -1314,6 +1311,7 @@ class SalesOrderShipmentSerializer( source='checked_by', many=False, read_only=True, allow_null=True ), True, + prefetch_fields=['checked_by'], ) order_detail = enable_filter( @@ -1321,6 +1319,13 @@ class SalesOrderShipmentSerializer( source='order', read_only=True, allow_null=True, many=False ), True, + prefetch_fields=[ + 'order', + 'order__customer', + 'order__created_by', + 'order__responsible', + 'order__project_code', + ], ) customer_detail = enable_filter( @@ -1328,6 +1333,7 @@ class SalesOrderShipmentSerializer( source='order.customer', many=False, read_only=True, allow_null=True ), False, + prefetch_fields=['order__customer'], ) shipment_address_detail = enable_filter( @@ -1335,6 +1341,7 @@ class SalesOrderShipmentSerializer( source='shipment_address', many=False, read_only=True, allow_null=True ), True, + prefetch_fields=['shipment_address'], )