[API] Query improvements (#11034)

* Improve prefetch fields for API

* Cache ContentType queryset for getModelsWithMixin

- Called a LOT of times for an options request
- Store the list in the session cache
- Much faster than redis - and expires after the session is complete

* Skip optional prefetch for options requests

* Custom implementation of DjangoModelPermission

- Cache the queryset against the view
- Prevents multiple hits for OPTIONS request
- Saves > 100ms on /stock/ options request

src/backend/InvenTree/InvenTree/cache.py

@@ -4,6 +4,8 @@ import socket
 import threading
 from typing import Any
 
+from django.db.utils import OperationalError, ProgrammingError
+
 import structlog
 
 import InvenTree.config
@@ -169,3 +171,22 @@ def set_session_cache(key: str, value: Any) -> None:
 
     if request_cache is not None:
         request_cache[key] = value
+
+
+def get_cached_content_types(cache_key: str = 'all_content_types') -> list:
+    """Return a list of all ContentType objects, using session cache if possible."""
+    from django.contrib.contenttypes.models import ContentType
+
+    # Attempt to retrieve a list of ContentType objects from session cache
+    if content_types := get_session_cache(cache_key):
+        return content_types
+
+    try:
+        content_types = list(ContentType.objects.all())
+        if len(content_types) > 0:
+            set_session_cache(cache_key, content_types)
+    except (OperationalError, ProgrammingError):
+        # Database is likely not yet ready
+        content_types = []
+
+    return content_types

src/backend/InvenTree/InvenTree/helpers_model.py

@@ -24,6 +24,11 @@ from common.notifications import (
     trigger_notification,
 )
 from common.settings import get_global_setting
+from InvenTree.cache import (
+    get_cached_content_types,
+    get_session_cache,
+    set_session_cache,
+)
 from InvenTree.format import format_money
 from InvenTree.ready import ignore_ready_warning
 
@@ -270,17 +275,23 @@ def getModelsWithMixin(mixin_class) -> list:
     Returns:
         List of models that inherit from the given mixin class
     """
-    from django.contrib.contenttypes.models import ContentType
+    # First, look in the session cache - to prevent repeated expensive comparisons
+    cache_key = f'models_with_mixin_{mixin_class.__name__}'
 
-    try:
+    if cached_models := get_session_cache(cache_key):
-        db_models = [
+        return cached_models
-            x.model_class() for x in ContentType.objects.all() if x is not None
+
+    content_types = get_cached_content_types()
+
+    db_models = [x.model_class() for x in content_types if x is not None]
+
+    models_with_mixin = [
+        x for x in db_models if x is not None and issubclass(x, mixin_class)
     ]
-    except (OperationalError, ProgrammingError):
-        # Database is likely not yet ready
-        db_models = []
 
-    return [x for x in db_models if x is not None and issubclass(x, mixin_class)]
+    # Store the result in the session cache
+    set_session_cache(cache_key, models_with_mixin)
+    return models_with_mixin
 
 
 def notify_responsible(

src/backend/InvenTree/InvenTree/metadata.py

@@ -23,7 +23,7 @@ logger = structlog.get_logger('inventree')
 class InvenTreeMetadata(SimpleMetadata):
     """Custom metadata class for the DRF API.
 
-    This custom metadata class imits the available "actions",
+    This custom metadata class limits the available "actions",
     based on the user's role permissions.
 
     Thus when a client send an OPTIONS request to an API endpoint,
@@ -50,6 +50,10 @@ class InvenTreeMetadata(SimpleMetadata):
 
         for method in {'PUT', 'POST', 'GET'} & set(view.allowed_methods):
             view.request = clone_request(request, method)
+
+            # Mark this request, to prevent expensive prefetching
+            view.request._metadata_requested = True
+
             try:
                 # Test global permissions
                 if hasattr(view, 'check_permissions'):

src/backend/InvenTree/InvenTree/permissions.py

@@ -149,7 +149,7 @@ class InvenTreeRoleScopeMixin(OASTokenMixin):
 class InvenTreeTokenMatchesOASRequirements(InvenTreeRoleScopeMixin):
     """Combines InvenTree role-based scope handling with OpenAPI schema token requirements.
 
-    Usesd as default permission class.
+    Used as default permission class.
     """
 
     def has_permission(self, request, view):
@@ -166,6 +166,29 @@ class InvenTreeTokenMatchesOASRequirements(InvenTreeRoleScopeMixin):
         return True
 
 
+class ModelPermission(permissions.DjangoModelPermissions):
+    """Custom ModelPermission implementation which provides cached lookup of queryset.
+
+    This is entirely for optimization purposes.
+    """
+
+    def _queryset(self, view):
+        """Return the queryset associated with this view, with caching.
+
+        This is because in a metadata OPTIONS request, the view is copied multiple times.
+        We can cache the queryset to avoid repeated calculation.
+        """
+        if getattr(view, '_cached_queryset', None) is not None:
+            return view._cached_queryset
+
+        queryset = super()._queryset(view)
+
+        if queryset is not None:
+            view._cached_queryset = queryset
+
+        return queryset
+
+
 class RolePermission(InvenTreeRoleScopeMixin, permissions.BasePermission):
     """Role mixin for API endpoints, allowing us to specify the user "role" which is required for certain operations.
 

src/backend/InvenTree/InvenTree/serializers.py

@@ -133,6 +133,15 @@ class FilterableSerializerMixin:
         Returns:
             The modified queryset with prefetching applied.
         """
+        # If we are inside an OPTIONS request, DO NOT PREFETCH
+        if request := getattr(self, 'request', None):
+            if method := getattr(request, 'method', None):
+                if str(method).lower() == 'options':
+                    return queryset
+
+            if getattr(request, '_metadata_requested', False):
+                return queryset
+
         # Gather up the set of simple 'prefetch' fields and functions
         prefetch_fields = set()
 
@@ -797,6 +806,8 @@ class ContentTypeField(serializers.ChoiceField):
         Args:
             mixin_class: Optional mixin class to restrict valid content types.
         """
+        from InvenTree.cache import get_cached_content_types
+
         self.mixin_class = mixin_class
 
         # Override the 'choices' field, to limit to the appropriate models
@@ -811,7 +822,7 @@ class ContentTypeField(serializers.ChoiceField):
                 for model in models
             ]
         else:
-            content_types = ContentType.objects.all()
+            content_types = get_cached_content_types()
 
             kwargs['choices'] = [
                 (f'{ct.app_label}.{ct.model}', str(ct)) for ct in content_types
@@ -828,8 +839,6 @@ class ContentTypeField(serializers.ChoiceField):
 
     def to_internal_value(self, data):
         """Convert string representation back to ContentType instance."""
-        from django.contrib.contenttypes.models import ContentType
-
         content_type = None
 
         if data in ['', None]:

src/backend/InvenTree/InvenTree/settings.py

@@ -592,7 +592,7 @@ REST_FRAMEWORK = {
     'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination',
     'DEFAULT_PERMISSION_CLASSES': [
         'rest_framework.permissions.IsAuthenticated',
-        'rest_framework.permissions.DjangoModelPermissions',
+        'InvenTree.permissions.ModelPermission',
         'InvenTree.permissions.RolePermission',
         'InvenTree.permissions.InvenTreeTokenMatchesOASRequirements',
     ],

src/backend/InvenTree/company/serializers.py

@@ -268,7 +268,11 @@ class ManufacturerPartSerializer(
             source='part', many=False, read_only=True, allow_null=True
         ),
         True,
-        prefetch_fields=['part'],
+        prefetch_fields=[
+            Prefetch(
+                'part', queryset=part.models.Part.objects.select_related('pricing_data')
+            )
+        ],
     )
 
     pretty_name = enable_filter(

src/backend/InvenTree/order/api.py

@@ -1501,10 +1501,10 @@ class ReturnOrderMixin(SerializerContextMixin):
     def get_queryset(self, *args, **kwargs):
         """Return annotated queryset for this endpoint."""
         queryset = super().get_queryset(*args, **kwargs)
-
-        queryset = queryset.prefetch_related('customer', 'created_by')
-
         queryset = serializers.ReturnOrderSerializer.annotate_queryset(queryset)
+        queryset = queryset.prefetch_related(
+            'contact', 'created_by', 'customer', 'responsible'
+        )
 
         return queryset
 

src/backend/InvenTree/order/serializers.py

@@ -1298,9 +1298,6 @@ class SalesOrderShipmentSerializer(
     @staticmethod
     def annotate_queryset(queryset):
         """Annotate the queryset with extra information."""
-        # Prefetch related objects
-        queryset = queryset.prefetch_related('order', 'order__customer', 'allocations')
-
         queryset = queryset.annotate(allocated_items=SubqueryCount('allocations'))
 
         return queryset
@@ -1314,6 +1311,7 @@ class SalesOrderShipmentSerializer(
             source='checked_by', many=False, read_only=True, allow_null=True
         ),
         True,
+        prefetch_fields=['checked_by'],
     )
 
     order_detail = enable_filter(
@@ -1321,6 +1319,13 @@ class SalesOrderShipmentSerializer(
             source='order', read_only=True, allow_null=True, many=False
         ),
         True,
+        prefetch_fields=[
+            'order',
+            'order__customer',
+            'order__created_by',
+            'order__responsible',
+            'order__project_code',
+        ],
     )
 
     customer_detail = enable_filter(
@@ -1328,6 +1333,7 @@ class SalesOrderShipmentSerializer(
             source='order.customer', many=False, read_only=True, allow_null=True
         ),
         False,
+        prefetch_fields=['order__customer'],
     )
 
     shipment_address_detail = enable_filter(
@@ -1335,6 +1341,7 @@ class SalesOrderShipmentSerializer(
             source='shipment_address', many=False, read_only=True, allow_null=True
         ),
         True,
+        prefetch_fields=['shipment_address'],
     )