From 891553a9c783244941bed90e1cff41c3e7da2e77 Mon Sep 17 00:00:00 2001 From: Oliver Date: Fri, 13 Nov 2026 19:50:37 +1100 Subject: [PATCH] API / query refactoring (#13258) * Improve caching for full_name template * More efficient thumbnail field * better prefetching * cache base media URL * InvenTreeModelSerializer now has 'fast_representation' optimization * Revert "InvenTreeModelSerializer now has 'fast_representation' optimization" This reverts commit 98571998a66c36a5bebf25bb71627a1d80edc7fa. * Fix OptionalField on AuditLog API endpoints * better prefetching * error if OutputOptionsMixin not correctly applied * better select_related for AttachmentMixin * additional pre-fetch improvements --- src/backend/InvenTree/InvenTree/apps.py | 6 + src/backend/InvenTree/InvenTree/checks.py | 107 ++++++++++++++++++ src/backend/InvenTree/InvenTree/helpers.py | 65 ++++++++++- .../InvenTree/InvenTree/serializers.py | 27 ++++- .../InvenTree/InvenTree/test_checks.py | 64 +++++++++++ .../InvenTree/InvenTree/test_media_url.py | 68 +++++++++++ src/backend/InvenTree/audit/api.py | 3 +- src/backend/InvenTree/build/serializers.py | 27 ++--- src/backend/InvenTree/common/api.py | 15 ++- .../InvenTree/common/setting/system.py | 8 ++ src/backend/InvenTree/company/api.py | 9 +- src/backend/InvenTree/company/serializers.py | 3 +- src/backend/InvenTree/order/api.py | 36 +++++- src/backend/InvenTree/order/serializers.py | 4 +- src/backend/InvenTree/part/helpers.py | 72 +++++++++--- src/backend/InvenTree/part/serializers.py | 4 +- src/backend/InvenTree/stock/serializers.py | 17 ++- src/backend/InvenTree/users/api.py | 2 +- src/backend/InvenTree/users/models.py | 16 ++- 19 files changed, 489 insertions(+), 64 deletions(-) create mode 100644 src/backend/InvenTree/InvenTree/checks.py create mode 100644 src/backend/InvenTree/InvenTree/test_checks.py create mode 100644 src/backend/InvenTree/InvenTree/test_media_url.py diff --git a/src/backend/InvenTree/InvenTree/apps.py b/src/backend/InvenTree/InvenTree/apps.py index 10560122ce..24a047c433 100644 --- a/src/backend/InvenTree/InvenTree/apps.py +++ b/src/backend/InvenTree/InvenTree/apps.py @@ -44,6 +44,12 @@ class InvenTreeConfig(AppConfig): - Collecting state transition methods - Adding users set in the current environment """ + from django.core import checks + + from InvenTree.checks import check_optional_field_prefetch + + checks.register(check_optional_field_prefetch, checks.Tags.urls) + # skip loading if plugin registry is not loaded or we run in a background thread if not InvenTree.ready.isPluginRegistryLoaded(): diff --git a/src/backend/InvenTree/InvenTree/checks.py b/src/backend/InvenTree/InvenTree/checks.py new file mode 100644 index 0000000000..fdd7ae9774 --- /dev/null +++ b/src/backend/InvenTree/InvenTree/checks.py @@ -0,0 +1,107 @@ +"""Django system checks for the InvenTree API.""" + +import inspect +from pathlib import Path + +from django.conf import settings +from django.core import checks +from django.urls import URLPattern, URLResolver, get_resolver + +from rest_framework.mixins import ListModelMixin + +# System check IDs for a list view which silently ignores 'OptionalField.prefetch_fields' +# - Error: core InvenTree view (blocks runserver / migrate / test / check) +# - Warning: view provided by an external plugin (must not prevent the server from starting) +MISSING_OUTPUT_OPTIONS_MIXIN = 'InvenTree.E001' +MISSING_OUTPUT_OPTIONS_MIXIN_PLUGIN = 'InvenTree.W001' + + +def is_core_view(view_cls) -> bool: + """Return True if the view class is defined within the InvenTree source tree.""" + try: + source = Path(inspect.getfile(view_cls)).resolve() + except (TypeError, OSError): + return False + + return source.is_relative_to(Path(settings.BASE_DIR).resolve()) + + +def iter_view_classes(patterns=None): + """Yield every (DRF) view class registered in the URL configuration.""" + if patterns is None: + patterns = get_resolver().url_patterns + + for pattern in patterns: + if isinstance(pattern, URLResolver): + yield from iter_view_classes(pattern.url_patterns) + elif isinstance(pattern, URLPattern): + if view_cls := getattr(pattern.callback, 'cls', None): + yield view_cls + + +def serializer_prefetch_fields(serializer_class) -> list[str]: + """Return the names of OptionalFields (on the serializer) which define prefetch_fields.""" + from InvenTree.serializers import FilterableSerializerMixin, OptionalField + + if not isinstance(serializer_class, type) or not issubclass( + serializer_class, FilterableSerializerMixin + ): + return [] + + names = set() + + for base in serializer_class.__mro__: + for name, field in vars(base).items(): + if isinstance(field, OptionalField) and field.prefetch_fields: + names.add(name) + + return sorted(names) + + +def check_optional_field_prefetch(app_configs=None, **kwargs): + """Warn about list views which would silently ignore 'OptionalField.prefetch_fields'. + + The prefetch_fields declared against an OptionalField are only applied to the + queryset by the OutputOptionsMixin. A list view without this mixin + will instead perform (at least) one extra query per row for each included field. + + This is reported as an error for core views, so that it is caught before release. + External plugin views are reported as a warning only. + """ + from InvenTree.mixins import OutputOptionsMixin + + errors = [] + seen = set() + + for view_cls in iter_view_classes(): + if view_cls in seen: + continue + + seen.add(view_cls) + + # Only list views are affected (a detail view returns a single object) + if not issubclass(view_cls, ListModelMixin): + continue + + if issubclass(view_cls, OutputOptionsMixin): + continue + + serializer_class = getattr(view_cls, 'serializer_class', None) + + if fields := serializer_prefetch_fields(serializer_class): + if is_core_view(view_cls): + level, check_id = checks.Error, MISSING_OUTPUT_OPTIONS_MIXIN + else: + level, check_id = checks.Warning, MISSING_OUTPUT_OPTIONS_MIXIN_PLUGIN + + errors.append( + level( + f'{view_cls.__module__}.{view_cls.__qualname__} ignores prefetch_fields ' + f'declared on {serializer_class.__name__} ({", ".join(fields)})', + hint='Add OutputOptionsMixin to the view, so that the prefetch_fields are applied', + obj=view_cls, + id=check_id, + ) + ) + + return errors diff --git a/src/backend/InvenTree/InvenTree/helpers.py b/src/backend/InvenTree/InvenTree/helpers.py index 9daeacccbb..7000fd4d5d 100644 --- a/src/backend/InvenTree/InvenTree/helpers.py +++ b/src/backend/InvenTree/InvenTree/helpers.py @@ -182,7 +182,70 @@ def getMediaUrl( if name is not None: file = regenerate_imagefile(file, name) - return default_storage.url(file.name) + return get_media_url(file.name) + + +# Request cache key for the default storage base URL (see get_media_url) +MEDIA_BASE_URL_CACHE_KEY = 'media_storage_base_url' + +# Matches '.' or '..' path segments, which urljoin() would resolve +_DOT_SEGMENT_REGEX = re.compile(r'(^|/)\.\.?(/|$)') + + +def _get_filesystem_base_url() -> str: + """Return the base URL of the default storage, if it is a plain FileSystemStorage. + + Returns an empty string for any other storage backend + (e.g. S3, or a subclass which may override url()), + or if the base URL is not in the expected format. + + The result is cached for the duration of the current request. + """ + from django.core.files.storage import FileSystemStorage + + from InvenTree.cache import get_session_cache, set_session_cache + + base_url = get_session_cache(MEDIA_BASE_URL_CACHE_KEY) + + if base_url is None: + base_url = '' + + # Note: default_storage is a LazyObject - __class__ is proxied to the wrapped storage + if default_storage.__class__ is FileSystemStorage: + storage_base_url = default_storage.base_url + + if isinstance(storage_base_url, str) and storage_base_url.endswith('/'): + base_url = storage_base_url + + set_session_cache(MEDIA_BASE_URL_CACHE_KEY, base_url) + + return base_url + + +def get_media_url(name: str) -> str: + """Return the URL for the named file in the default (media) storage. + + Equivalent to default_storage.url(name), but for the default FileSystemStorage + backend the URL is constructed directly, which avoids the overhead of urljoin(). + This is significant when rendering URLs for large API list queries. + + Arguments: + name: The name (path) of the file within the storage + """ + if base_url := _get_filesystem_base_url(): + from django.utils.encoding import filepath_to_uri + + url = filepath_to_uri(name) + + if url is not None: + url = url.lstrip('/') + + # urljoin() resolves '.' and '..' segments, and collapses empty segments ('//') + # Defer to the storage backend for these (rare) cases + if '//' not in url and not _DOT_SEGMENT_REGEX.search(url): + return base_url + url + + return default_storage.url(name) def regenerate_imagefile(_file, _name: str): diff --git a/src/backend/InvenTree/InvenTree/serializers.py b/src/backend/InvenTree/InvenTree/serializers.py index 20a94b36c2..1483f95b55 100644 --- a/src/backend/InvenTree/InvenTree/serializers.py +++ b/src/backend/InvenTree/InvenTree/serializers.py @@ -9,7 +9,6 @@ from typing import Optional from django.contrib.auth import get_user_model from django.contrib.contenttypes.models import ContentType from django.core.exceptions import ValidationError as DjangoValidationError -from django.core.files.storage import default_storage from django.db import models from django.db.models import QuerySet from django.utils.translation import gettext_lazy as _ @@ -29,7 +28,7 @@ from taggit.serializers import TaggitSerializer import InvenTree.ready from common.currency import currency_code_default, currency_code_mappings from InvenTree.fields import InvenTreeRestURLField, InvenTreeURLField -from InvenTree.helpers import str2bool +from InvenTree.helpers import get_media_url, str2bool from InvenTree.helpers_model import getModelsWithMixin @@ -941,7 +940,27 @@ class InvenTreeAttachmentSerializerField(serializers.FileField): if not value: return None - return default_storage.url(str(value)) + return get_media_url(str(value)) + + +class InvenTreeThumbnailField(serializers.CharField): + """Read-only field which renders the thumbnail URL for a model instance. + + The instance must implement get_thumbnail_url() (e.g. InvenTreeImageMixin). + + Note: This is more efficient than CharField(source='get_thumbnail_url'), + as DRF inspects the signature of a callable source for every serialized row. + """ + + def __init__(self, **kwargs): + """Initialize the field as read-only, sourced from the instance itself.""" + kwargs['source'] = '*' + kwargs['read_only'] = True + super().__init__(**kwargs) + + def to_representation(self, value): + """Return the thumbnail URL for the provided instance.""" + return value.get_thumbnail_url() class InvenTreeImageSerializerField(serializers.ImageField): @@ -955,7 +974,7 @@ class InvenTreeImageSerializerField(serializers.ImageField): if not value: return None - return default_storage.url(str(value)) + return get_media_url(str(value)) class InvenTreeDecimalField(serializers.FloatField): diff --git a/src/backend/InvenTree/InvenTree/test_checks.py b/src/backend/InvenTree/InvenTree/test_checks.py new file mode 100644 index 0000000000..1021ef168e --- /dev/null +++ b/src/backend/InvenTree/InvenTree/test_checks.py @@ -0,0 +1,64 @@ +"""Tests for the InvenTree system checks.""" + +from unittest import mock + +from django.core import checks +from django.test import SimpleTestCase + +from rest_framework.generics import ListAPIView + +from InvenTree.checks import ( + MISSING_OUTPUT_OPTIONS_MIXIN, + MISSING_OUTPUT_OPTIONS_MIXIN_PLUGIN, + check_optional_field_prefetch, +) +from InvenTree.mixins import OutputOptionsMixin +from order.serializers import SalesOrderShipmentSerializer + + +class OptionalFieldPrefetchCheckTest(SimpleTestCase): + """Tests for the check_optional_field_prefetch system check.""" + + def test_no_warnings(self): + """No registered list view may silently ignore OptionalField.prefetch_fields.""" + warnings = check_optional_field_prefetch() + + self.assertEqual(warnings, [], '\n'.join(str(w) for w in warnings)) + + def test_detects_missing_mixin(self): + """A core list view without OutputOptionsMixin is reported as an error.""" + + class BadList(ListAPIView): + serializer_class = SalesOrderShipmentSerializer + + class GoodList(OutputOptionsMixin, ListAPIView): + serializer_class = SalesOrderShipmentSerializer + + with mock.patch( + 'InvenTree.checks.iter_view_classes', return_value=[BadList, GoodList] + ): + errors = check_optional_field_prefetch() + + self.assertEqual(len(errors), 1) + self.assertIsInstance(errors[0], checks.Error) + self.assertTrue(errors[0].is_serious()) + self.assertEqual(errors[0].id, MISSING_OUTPUT_OPTIONS_MIXIN) + self.assertIs(errors[0].obj, BadList) + self.assertIn('order_detail', errors[0].msg) + + def test_external_plugin_view(self): + """A view defined outside the InvenTree source tree is only a warning.""" + + class PluginList(ListAPIView): + serializer_class = SalesOrderShipmentSerializer + + with ( + mock.patch('InvenTree.checks.iter_view_classes', return_value=[PluginList]), + mock.patch('InvenTree.checks.is_core_view', return_value=False), + ): + errors = check_optional_field_prefetch() + + self.assertEqual(len(errors), 1) + self.assertNotIsInstance(errors[0], checks.Error) + self.assertFalse(errors[0].is_serious()) + self.assertEqual(errors[0].id, MISSING_OUTPUT_OPTIONS_MIXIN_PLUGIN) diff --git a/src/backend/InvenTree/InvenTree/test_media_url.py b/src/backend/InvenTree/InvenTree/test_media_url.py new file mode 100644 index 0000000000..1a56ec05b9 --- /dev/null +++ b/src/backend/InvenTree/InvenTree/test_media_url.py @@ -0,0 +1,68 @@ +"""Tests for fast media URL generation (InvenTree.helpers.get_media_url).""" + +import random + +from django.core.files.storage import default_storage +from django.test import TestCase, override_settings + +from InvenTree.helpers import get_media_url + + +class MediaUrlTest(TestCase): + """Tests for InvenTree.helpers.get_media_url.""" + + NAMES = [ + '', + 'a.png', + '/a.png', + '//a.png', + 'part_images/x.thumbnail.png', + 'a/./b.png', + 'a/../b.png', + '../x', + './x', + '.', + '..', + 'a/..b/c', + 'a//b', + 'a?b#c;d:e', + 'http://evil.com/x', + 'C:\\x\\y.png', + 'ünï cödé.png', + '%2e%2e/x', + 'x/', + ] + + def check_names(self, names): + """Assert get_media_url matches default_storage.url for all names.""" + for name in names: + self.assertEqual(get_media_url(name), default_storage.url(name), name) + + def test_equivalence(self): + """Output matches default_storage.url() for the filesystem backend.""" + rnd = random.Random(0) + alphabet = "ab./\\?#;:%~!*()'é " + fuzz = [ + ''.join(rnd.choice(alphabet) for _ in range(rnd.randint(0, 12))) + for _ in range(2000) + ] + + for media_url in ('/media/', 'https://cdn.example.com/media/'): + with override_settings(MEDIA_URL=media_url): + self.check_names(self.NAMES + fuzz) + + @override_settings( + STORAGES={ + 'default': { + 'BACKEND': 'django.core.files.storage.InMemoryStorage', + 'OPTIONS': {'base_url': '/memory/'}, + }, + 'staticfiles': { + 'BACKEND': 'django.contrib.staticfiles.storage.StaticFilesStorage' + }, + } + ) + def test_other_backend(self): + """Other storage backends are delegated to default_storage.url().""" + self.assertEqual(get_media_url('a/b.png'), '/memory/a/b.png') + self.check_names(self.NAMES) diff --git a/src/backend/InvenTree/audit/api.py b/src/backend/InvenTree/audit/api.py index ca1a5a7c47..0692ef4a12 100644 --- a/src/backend/InvenTree/audit/api.py +++ b/src/backend/InvenTree/audit/api.py @@ -22,6 +22,7 @@ from audit.permissions import ( from common.filters import determine_content_type from InvenTree.filters import SEARCH_ORDER_FILTER from InvenTree.helpers_api import InvenTreeApiRouter +from InvenTree.mixins import OutputOptionsMixin from InvenTree.permissions import IsAuthenticatedOrReadScope @@ -166,7 +167,7 @@ class AuditLogFilter(rest_filters.FilterSet): return queryset.filter(own) -class AuditLogViewSet(viewsets.ReadOnlyModelViewSet): +class AuditLogViewSet(OutputOptionsMixin, viewsets.ReadOnlyModelViewSet): """API endpoint for audit log entries (read-only). - GET (list): Return a list of audit log entries diff --git a/src/backend/InvenTree/build/serializers.py b/src/backend/InvenTree/build/serializers.py index 6d5804aae1..5ff4057011 100644 --- a/src/backend/InvenTree/build/serializers.py +++ b/src/backend/InvenTree/build/serializers.py @@ -164,7 +164,7 @@ class BuildSerializer( }, default_include=True, filter_name='user_detail', - prefetch_fields=['responsible'], + prefetch_fields=['responsible', 'responsible__owner'], ) barcode_hash = serializers.CharField(read_only=True) @@ -1558,25 +1558,18 @@ class BuildLineSerializer( Note: If the 'build' is provided, we can use it to filter available stock, depending on the specified location for the build """ - queryset = queryset.select_related( - 'build', - 'build__part', - 'build__part__pricing_data', - 'bom_item', - 'bom_item__part', - 'bom_item__part__pricing_data', - 'bom_item__sub_part', - 'bom_item__sub_part__pricing_data', + # Note: Only join the tables which are required for every row. + # The build (and its part) are shared across many rows, so a prefetch is cheaper + # than constructing duplicate model instances for each row via select_related. + # Other related data is prefetched by the optional fields (where required). + queryset = queryset.select_related('bom_item', 'bom_item__sub_part') + + queryset = queryset.prefetch_related( + 'build', 'build__part', 'build__part__pricing_data' ) # Defer expensive fields which we do not need for this serializer - - queryset = queryset.defer( - 'build__metadata', - 'bom_item__metadata', - 'bom_item__part__metadata', - 'bom_item__sub_part__metadata', - ) + queryset = queryset.defer('bom_item__metadata', 'bom_item__sub_part__metadata') # Annotate the "allocated" quantity queryset = queryset.annotate( diff --git a/src/backend/InvenTree/common/api.py b/src/backend/InvenTree/common/api.py index 09ac537dc6..eb3a23a644 100644 --- a/src/backend/InvenTree/common/api.py +++ b/src/backend/InvenTree/common/api.py @@ -910,12 +910,14 @@ def get_viewable_attachment_model_types(user) -> set: class AttachmentMixin: """Mixin class for Attachment views.""" - queryset = common.models.Attachment.objects.all() + queryset = common.models.Attachment.objects.all().select_related('upload_user') serializer_class = common.serializers.AttachmentSerializer permission_classes = [IsAuthenticatedOrReadScope] -class AttachmentList(AttachmentMixin, BulkDeleteMixin, ListCreateAPI): +class AttachmentList( + AttachmentMixin, BulkDeleteMixin, OutputOptionsMixin, ListCreateAPI +): """List API endpoint for Attachment objects.""" filter_backends = SEARCH_ORDER_FILTER @@ -1034,7 +1036,9 @@ class NoteMixin: # Note: This is handled explicitly in the 'save' method of the Note model SAFE_FIELDS = ['content'] - queryset = common.models.Note.objects.all() + queryset = common.models.Note.objects.all().select_related( + 'updated_by', 'model_type' + ) serializer_class = common.serializers.NoteSerializer permission_classes = [IsAuthenticatedOrReadScope] @@ -1068,7 +1072,7 @@ class NoteMixin: return qs.filter(Q(template=True) | Q(model_type__in=allowed_ct_ids)) -class NoteList(NoteMixin, ListCreateAPI): +class NoteList(NoteMixin, OutputOptionsMixin, ListCreateAPI): """List API endpoint for Note objects.""" filter_backends = SEARCH_ORDER_FILTER @@ -1548,8 +1552,7 @@ class InstanceInfoView(APIView): # Route through NoteList's own get_queryset() (rather than # re-deriving the view-permission check here) so this count can # never drift from what the Notes list endpoint actually shows. - note_list_view = NoteList() - note_list_view.request = request + note_list_view = NoteList(request=request, format_kwarg=None) counts['note_count'] = ( note_list_view .get_queryset() diff --git a/src/backend/InvenTree/common/setting/system.py b/src/backend/InvenTree/common/setting/system.py index c9c3e8b28c..ec9191f1c7 100644 --- a/src/backend/InvenTree/common/setting/system.py +++ b/src/backend/InvenTree/common/setting/system.py @@ -102,6 +102,13 @@ def update_instance_name(setting): site_obj.save() +def reset_part_name_format(setting): + """Clear the cached Part 'full_name' template when PART_NAME_FORMAT changes.""" + from part.helpers import reset_full_name_template + + reset_full_name_template() + + def update_instance_url(setting): """Update the first site objects domain to url.""" if site_obj := _get_site_object(): @@ -579,6 +586,7 @@ SYSTEM_SETTINGS: dict[str, InvenTreeSettingsKeyType] = { 'default': "{{ part.IPN if part.IPN }}{{ ' | ' if part.IPN }}{{ part.name }}{{ ' | ' if part.revision }}" '{{ part.revision if part.revision }}', 'validator': validate_part_name_format, + 'after_save': reset_part_name_format, }, 'PART_CATEGORY_DEFAULT_ICON': { 'name': _('Part Category Default Icon'), diff --git a/src/backend/InvenTree/company/api.py b/src/backend/InvenTree/company/api.py index e7cb91dd5e..886c83bcc3 100644 --- a/src/backend/InvenTree/company/api.py +++ b/src/backend/InvenTree/company/api.py @@ -107,7 +107,7 @@ class CompanyDetail(CompanyMixin, RetrieveUpdateDestroyAPI): class ContactList(DataExportViewMixin, ListCreateDestroyAPIView): """API endpoint for list view of Company model.""" - queryset = Contact.objects.all() + queryset = Contact.objects.all().select_related('company') serializer_class = ContactSerializer filter_backends = SEARCH_ORDER_FILTER @@ -124,7 +124,7 @@ class ContactList(DataExportViewMixin, ListCreateDestroyAPIView): class ContactDetail(RetrieveUpdateDestroyAPI): """Detail endpoint for Company model.""" - queryset = Contact.objects.all() + queryset = Contact.objects.all().select_related('company') serializer_class = ContactSerializer @@ -494,6 +494,11 @@ class SupplierPriceBreakMixin: queryset = SupplierPriceBreak.objects.all() serializer_class = SupplierPriceBreakSerializer + def get_queryset(self): + """Return annotated queryset for the SupplierPriceBreak endpoints.""" + queryset = super().get_queryset() + return SupplierPriceBreakSerializer.annotate_queryset(queryset) + class SupplierPriceBreakOutputOptions(OutputConfiguration): """Available output options for the SupplierPriceBreak endpoints.""" diff --git a/src/backend/InvenTree/company/serializers.py b/src/backend/InvenTree/company/serializers.py index 35d7b99acd..89e3468e6c 100644 --- a/src/backend/InvenTree/company/serializers.py +++ b/src/backend/InvenTree/company/serializers.py @@ -23,6 +23,7 @@ from InvenTree.serializers import ( InvenTreeModelSerializer, InvenTreeMoneySerializer, InvenTreeTaggitSerializer, + InvenTreeThumbnailField, OptionalField, apply_duplicate_copy_options, ) @@ -58,7 +59,7 @@ class CompanyBriefSerializer(InvenTreeModelSerializer): image = InvenTreeImageSerializerField(read_only=True) - thumbnail = serializers.CharField(source='get_thumbnail_url', read_only=True) + thumbnail = InvenTreeThumbnailField() @register_importer() diff --git a/src/backend/InvenTree/order/api.py b/src/backend/InvenTree/order/api.py index b2d75d18dc..cac8b3e195 100644 --- a/src/backend/InvenTree/order/api.py +++ b/src/backend/InvenTree/order/api.py @@ -460,8 +460,14 @@ class PurchaseOrderViewSet( def get_order(self): """Return the PurchaseOrder object associated with this API endpoint.""" + pk = self.kwargs.get('pk', None) + + # No order is associated with a list request: avoid a pointless DB query + if pk is None: + raise NotFound(_('Purchase order not found')) + try: - return models.PurchaseOrder.objects.get(pk=self.kwargs.get('pk', None)) + return models.PurchaseOrder.objects.get(pk=pk) except (ValueError, models.PurchaseOrder.DoesNotExist): raise NotFound(_('Purchase order not found')) @@ -1176,8 +1182,14 @@ class SalesOrderContextMixin: def get_order(self): """Return the SalesOrder object associated with this API endpoint.""" + pk = self.kwargs.get('pk', None) + + # No order is associated with a list request: avoid a pointless DB query + if pk is None: + raise NotFound(_('Sales order not found')) + try: - return models.SalesOrder.objects.get(pk=self.kwargs.get('pk', None)) + return models.SalesOrder.objects.get(pk=pk) except (ValueError, models.SalesOrder.DoesNotExist): raise NotFound(_('Sales order not found')) @@ -1551,7 +1563,9 @@ class SalesOrderShipmentMixin: @search_provider(ref_names=['salesordershipment']) -class SalesOrderShipmentList(SalesOrderShipmentMixin, ListCreateAPI): +class SalesOrderShipmentList( + SalesOrderShipmentMixin, OutputOptionsMixin, ListCreateAPI +): """API list endpoint for SalesOrderShipment model.""" filterset_class = SalesOrderShipmentFilter @@ -1759,8 +1773,14 @@ class ReturnOrderContextMixin: def get_order(self): """Return the ReturnOrder object associated with this API endpoint.""" + pk = self.kwargs.get('pk', None) + + # No order is associated with a list request: avoid a pointless DB query + if pk is None: + raise NotFound(_('Return order not found')) + try: - return models.ReturnOrder.objects.get(pk=self.kwargs.get('pk', None)) + return models.ReturnOrder.objects.get(pk=pk) except (ValueError, models.ReturnOrder.DoesNotExist): raise NotFound(_('Return order not found')) @@ -2062,8 +2082,14 @@ class TransferOrderContextMixin: def get_order(self): """Return the TransferOrder object associated with this API endpoint.""" + pk = self.kwargs.get('pk', None) + + # No order is associated with a list request: avoid a pointless DB query + if pk is None: + raise NotFound(_('Transfer order not found')) + try: - return models.TransferOrder.objects.get(pk=self.kwargs.get('pk', None)) + return models.TransferOrder.objects.get(pk=pk) except (ValueError, models.TransferOrder.DoesNotExist): raise NotFound(_('Transfer order not found')) diff --git a/src/backend/InvenTree/order/serializers.py b/src/backend/InvenTree/order/serializers.py index 028daead69..cf57d39da0 100644 --- a/src/backend/InvenTree/order/serializers.py +++ b/src/backend/InvenTree/order/serializers.py @@ -131,7 +131,7 @@ class AbstractOrderSerializer( 'many': False, }, default_include=True, - prefetch_fields=['responsible'], + prefetch_fields=['responsible', 'responsible__owner'], ) project_code_label = common.filters.enable_project_label_filter() @@ -2559,6 +2559,7 @@ class TransferOrderSerializer( 'allow_null': True, }, default_include=True, + prefetch_fields=['take_from', 'take_from__location_type'], ) destination_detail = OptionalField( @@ -2570,6 +2571,7 @@ class TransferOrderSerializer( 'allow_null': True, }, default_include=True, + prefetch_fields=['destination', 'destination__location_type'], ) diff --git a/src/backend/InvenTree/part/helpers.py b/src/backend/InvenTree/part/helpers.py index 7c55c9810e..69654292bc 100644 --- a/src/backend/InvenTree/part/helpers.py +++ b/src/backend/InvenTree/part/helpers.py @@ -8,6 +8,7 @@ import structlog from jinja2.sandbox import SandboxedEnvironment from common.settings import get_global_setting +from InvenTree.cache import get_session_cache, set_session_cache logger = structlog.get_logger('inventree') @@ -16,34 +17,77 @@ logger = structlog.get_logger('inventree') _part_full_name_template = None _part_full_name_template_string = '' +# Request cache key for the compiled 'full_name' template +PART_NAME_TEMPLATE_CACHE_KEY = 'part_full_name_template' + + +class DefaultPartNameTemplate: + """Fast (non-jinja) renderer for the default 'PART_NAME_FORMAT' template. + + Produces identical output to the default template: + "{{ part.IPN if part.IPN }}{{ ' | ' if part.IPN }}{{ part.name }}{{ ' | ' if part.revision }}{{ part.revision if part.revision }}" + """ + + def render(self, part) -> str: + """Render the 'full_name' for the provided part.""" + name = f'{part.IPN} | {part.name}' if part.IPN else str(part.name) + + if part.revision: + name += f' | {part.revision}' + + return name + + +def reset_full_name_template(*args, **kwargs): + """Clear the request-cached 'full_name' template. + + Called when the 'PART_NAME_FORMAT' setting is saved, + so that the new format is applied within the current request. + """ + set_session_cache(PART_NAME_TEMPLATE_CACHE_KEY, None) + def compile_full_name_template(*args, **kwargs): - """Recompile the template for rendering the 'full_name' attribute of a Part. + """Return the compiled template for rendering the 'full_name' attribute of a Part. - This function is called whenever the 'PART_NAME_FORMAT' setting is changed. + The 'PART_NAME_FORMAT' setting is checked (at most) once per request, + as this function is called for every Part rendered (e.g. large API list queries). + The template is only recompiled when the setting value changes. """ global _part_full_name_template global _part_full_name_template_string + if (template := get_session_cache(PART_NAME_TEMPLATE_CACHE_KEY)) is not None: + return template + template_string = get_global_setting('PART_NAME_FORMAT', cache=True) - # Skip if the template string has not changed + # Only recompile if the template string has changed if ( - template_string == _part_full_name_template_string - and _part_full_name_template is not None + template_string != _part_full_name_template_string + or _part_full_name_template is None ): - return _part_full_name_template + from common.setting.system import SYSTEM_SETTINGS - # Cache the template string - _part_full_name_template_string = template_string + # Cache the template string + _part_full_name_template_string = template_string - env = SandboxedEnvironment(variable_start_string='{{', variable_end_string='}}') + if template_string == SYSTEM_SETTINGS['PART_NAME_FORMAT']['default']: + # Default format - avoid the overhead of jinja rendering + _part_full_name_template = DefaultPartNameTemplate() + else: + env = SandboxedEnvironment( + variable_start_string='{{', variable_end_string='}}' + ) - # Compile the template - try: - _part_full_name_template = env.from_string(template_string) - except Exception: - _part_full_name_template = None + # Compile the template + try: + _part_full_name_template = env.from_string(template_string) + except Exception: + _part_full_name_template = None + + # Note: An invalid (None) template is not cached, and will be re-checked next call + set_session_cache(PART_NAME_TEMPLATE_CACHE_KEY, _part_full_name_template) return _part_full_name_template diff --git a/src/backend/InvenTree/part/serializers.py b/src/backend/InvenTree/part/serializers.py index d13ea6a691..32551a2a47 100644 --- a/src/backend/InvenTree/part/serializers.py +++ b/src/backend/InvenTree/part/serializers.py @@ -323,7 +323,7 @@ class PartBriefSerializer( image = InvenTree.serializers.InvenTreeImageSerializerField( read_only=True, allow_null=True ) - thumbnail = serializers.CharField(source='get_thumbnail_url', read_only=True) + thumbnail = InvenTree.serializers.InvenTreeThumbnailField() IPN = serializers.CharField( required=False, @@ -840,7 +840,7 @@ class PartSerializer( image = InvenTree.serializers.InvenTreeImageSerializerField( required=False, allow_null=True ) - thumbnail = serializers.CharField(source='get_thumbnail_url', read_only=True) + thumbnail = InvenTree.serializers.InvenTreeThumbnailField() starred = serializers.SerializerMethodField() # PrimaryKeyRelated fields (Note: enforcing field type here results in much faster queries, somehow...) diff --git a/src/backend/InvenTree/stock/serializers.py b/src/backend/InvenTree/stock/serializers.py index 9da237e2bb..68cfdea321 100644 --- a/src/backend/InvenTree/stock/serializers.py +++ b/src/backend/InvenTree/stock/serializers.py @@ -518,17 +518,25 @@ class StockItemSerializer( @staticmethod def annotate_queryset(queryset): """Add some extra annotations to the queryset, performing database queries as efficiently as possible.""" + # Note: 'part' must be loaded via Prefetch (not select_related), + # otherwise the select_related cache causes the Prefetch to be skipped, + # and the 'category_default_location' annotation is silently lost. + # Prefetching also loads each distinct Part (and PartPricing) only once, + # rather than once per StockItem row. queryset = queryset.prefetch_related( 'location', 'sales_order', 'purchase_order', Prefetch( 'part', - queryset=part_models.Part.objects.annotate( + queryset=part_models.Part.objects + .annotate( category_default_location=part_filters.annotate_default_location( 'category__' ) - ).prefetch_related(None), + ) + .prefetch_related(None) + .select_related('pricing_data'), ), 'parent', 'part__category', @@ -536,9 +544,8 @@ class StockItemSerializer( 'supplier_part__manufacturer_part', 'customer', 'belongs_to', - 'sales_order', 'consumed_by', - ).select_related('part', 'part__pricing_data') + ) # Annotate the queryset with the total allocated to sales orders queryset = queryset.annotate( @@ -1472,7 +1479,7 @@ class LocationSerializer( """Annotate extra information to the queryset.""" # Annotate the number of stock items which exist in this category (including subcategories) - queryset = queryset.prefetch_related('tags') + queryset = queryset.prefetch_related('tags', 'location_type') queryset = queryset.annotate( items=stock.filters.annotate_location_items(), diff --git a/src/backend/InvenTree/users/api.py b/src/backend/InvenTree/users/api.py index 0c01ac0539..7ac87bfec8 100644 --- a/src/backend/InvenTree/users/api.py +++ b/src/backend/InvenTree/users/api.py @@ -292,7 +292,7 @@ class UserList(ListCreateAPI): - Otherwise authenticated users have read-only access """ - queryset = User.objects.all().prefetch_related('groups') + queryset = User.objects.all().select_related('profile').prefetch_related('groups') serializer_class = UserCreateSerializer # User must have the right role, AND be a staff user, else read-only diff --git a/src/backend/InvenTree/users/models.py b/src/backend/InvenTree/users/models.py index ab6f98f098..c3f30084f7 100644 --- a/src/backend/InvenTree/users/models.py +++ b/src/backend/InvenTree/users/models.py @@ -557,19 +557,27 @@ class Owner(models.Model): owner = GenericForeignKey('owner_type', 'owner_id') + @property + def owner_type_name(self) -> str: + """Return the name of the owner type (e.g. 'user' or 'group'). + + Uses the ContentType cache, rather than a database query for each Owner instance. + """ + return ContentType.objects.get_for_id(self.owner_type_id).name + def __str__(self): """Defines the owner string representation.""" - if self.owner_type.name == 'user' and get_global_setting( + if self.owner_type_name == 'user' and get_global_setting( 'DISPLAY_FULL_NAMES', cache=True ): display_name = self.owner.get_full_name() else: display_name = str(self.owner) - return f'{display_name} ({self.owner_type.name})' + return f'{display_name} ({self.owner_type_name})' def name(self): """Return the 'name' of this owner.""" - if self.owner_type.name == 'user' and get_global_setting( + if self.owner_type_name == 'user' and get_global_setting( 'DISPLAY_FULL_NAMES', cache=True ): if self.owner and hasattr(self.owner, 'get_full_name'): @@ -581,7 +589,7 @@ class Owner(models.Model): def label(self): """Return the 'type' label of this owner i.e. 'user' or 'group'.""" - return str(self.owner_type.name) + return str(self.owner_type_name) @classmethod def create(cls, obj):