diff --git a/src/backend/InvenTree/InvenTree/fast_representation.py b/src/backend/InvenTree/InvenTree/fast_representation.py new file mode 100644 index 0000000000..4f1c90ec37 --- /dev/null +++ b/src/backend/InvenTree/InvenTree/fast_representation.py @@ -0,0 +1,239 @@ +"""Fast-path serializer output for large API list queries. + +DRF's Serializer.to_representation() handles every field generically: + + attribute = field.get_attribute(instance) # nested getattr, callable checks, error handling + value = field.to_representation(attribute) # type coercion + +For a 1,000 row list with ~70 fields per row, that machinery is a large fraction of the request +time, even though for most fields it reduces to a plain getattr() of a value which is already the +correct output type. + +This module builds (once per serializer instance) a "representation plan", which classifies each +readable field as either: + +- VALUE: simple field with single attribute source, whose to_representation() is an identity + for values of known exact types (e.g. CharField and str). Read directly via getattr. +- PK: PrimaryKeyRelatedField for a forward relation - read the FK column (_id) directly, + exactly as DRF's 'pk only' optimization does via Model.serializable_value(). +- DATETIME: DateTimeField using the default output format and timezone, with the + current timezone resolved once per row rather than once per value. +- GENERIC: anything else - handled exactly as DRF does. + +Any value which does not match the expected type (or raises an exception) falls back to the +standard DRF code path for that field, so output is identical to DRF's. +""" + +import datetime +from typing import Any + +from django.conf import settings +from django.utils import timezone + +from rest_framework import serializers +from rest_framework.fields import SkipField +from rest_framework.relations import PKOnlyObject +from rest_framework.settings import api_settings + +# Field plan modes +GENERIC = 0 +VALUE = 1 +PK = 2 +DATETIME = 3 + +# Serializer field classes (exact type) for which to_representation(value) == value, +# when type(value) is one of the listed (exact) types +IDENTITY_TYPES: dict[type, tuple[type, ...]] = { + serializers.CharField: (str,), + serializers.IntegerField: (int,), + serializers.FloatField: (float,), + serializers.BooleanField: (bool,), + serializers.ReadOnlyField: (str, int, float, bool), +} + +# Cache of whether a serializer class may use the fast path (see check_serializer_class) +_class_check_cache: dict[type, bool] = {} + + +def check_serializer_class(cls: type, base: type) -> bool: + """Determine whether the provided serializer class can use the fast path. + + The fast path replaces Serializer.to_representation() - so it is only valid if the + next to_representation() implementation in the MRO after 'base' is DRF's own. + """ + if cls in _class_check_cache: + return _class_check_cache[cls] + + mro = cls.__mro__ + result = True + + for klass in mro[mro.index(base) + 1 :]: + if 'to_representation' in klass.__dict__: + result = klass is serializers.Serializer + break + + _class_check_cache[cls] = result + return result + + +def classify_field(field, model_class: type) -> tuple: + """Return the plan entry (mode, attribute, extra) for a single serializer field.""" + source_attrs = getattr(field, 'source_attrs', None) + + # Only fields which read a single attribute directly from the instance + if not source_attrs or len(source_attrs) != 1: + return (GENERIC, None, None) + + attr = source_attrs[0] + field_type = type(field) + + if field_type in IDENTITY_TYPES: + return (VALUE, attr, IDENTITY_TYPES[field_type]) + + if field_type is serializers.PrimaryKeyRelatedField and field.pk_field is None: + try: + model_field = model_class._meta.get_field(attr) + except Exception: + return (GENERIC, None, None) + + if ( + model_field.concrete + and model_field.is_relation + and (model_field.many_to_one or model_field.one_to_one) + ): + return (PK, model_field.attname, None) + + if field_type is serializers.DateTimeField: + # Only the default format + timezone behavior is supported + if not hasattr(field, 'format') and not hasattr(field, 'timezone'): + return (DATETIME, attr, api_settings.DATETIME_FORMAT) + + return (GENERIC, None, None) + + +def build_plan(serializer, model_class: type) -> list[tuple]: + """Construct the representation plan for the provided serializer.""" + return [ + (field.field_name, field, *classify_field(field, model_class)) + for field in serializer._readable_fields + ] + + +def represent_generic(field, instance, ret: dict) -> None: + """Represent a single field exactly as DRF Serializer.to_representation() does.""" + try: + attribute = field.get_attribute(instance) + except SkipField: + return + + check_for_none = attribute.pk if isinstance(attribute, PKOnlyObject) else attribute + + if check_for_none is None: + ret[field.field_name] = None + else: + ret[field.field_name] = field.to_representation(attribute) + + +class _FallbackError(Exception): + """Internal signal to use the generic code path.""" + + +def represent_datetime(value: datetime.datetime, tz, output_format): + """Equivalent to DRF DateTimeField.to_representation(), with a pre-resolved timezone. + + Raises _FallbackError for edge cases (naive datetime with USE_TZ, overflow, etc). + """ + # Note: DRF returns the value *without* timezone conversion if no format is specified + if output_format is None: + return value + + aware = value.utcoffset() is not None + + if tz is not None: + if not aware: + raise _FallbackError + try: + value = value.astimezone(tz) + except OverflowError: + raise _FallbackError + elif aware: + raise _FallbackError + + if output_format.lower() == 'iso-8601': + value = value.isoformat() + if value.endswith('+00:00'): + value = value[:-6] + 'Z' + return value + + return value.strftime(output_format) + + +def fast_to_representation(serializer, instance) -> dict[str, Any]: + """Return the serialized representation of 'instance', using the fast path where possible. + + The output is identical to DRF Serializer.to_representation() + """ + model_class = type(instance) + + plan_cache = getattr(serializer, '_fast_repr_plan', None) + + if plan_cache is None or plan_cache[0] is not model_class: + plan_cache = (model_class, build_plan(serializer, model_class)) + serializer._fast_repr_plan = plan_cache + + plan = plan_cache[1] + + ret = {} + tz = None + tz_resolved = False + + for name, field, mode, attr, extra in plan: + if mode == VALUE: + try: + value = getattr(instance, attr) + except Exception: + # Let DRF handle defaults / missing attributes / related object lookups + represent_generic(field, instance, ret) + continue + + if value is None: + ret[name] = None + elif type(value) in extra: + ret[name] = value + elif callable(value): + represent_generic(field, instance, ret) + else: + ret[name] = field.to_representation(value) + + elif mode == PK: + ret[name] = getattr(instance, attr) + + elif mode == DATETIME: + try: + value = getattr(instance, attr) + except Exception: + represent_generic(field, instance, ret) + continue + + if value is None: + ret[name] = None + continue + + if type(value) is not datetime.datetime: + represent_generic(field, instance, ret) + continue + + if not tz_resolved: + # Matches DRF DateTimeField.default_timezone() + tz = timezone.get_current_timezone() if settings.USE_TZ else None + tz_resolved = True + + try: + ret[name] = represent_datetime(value, tz, extra) + except _FallbackError: + ret[name] = field.to_representation(value) + + else: + represent_generic(field, instance, ret) + + return ret diff --git a/src/backend/InvenTree/InvenTree/serializers.py b/src/backend/InvenTree/InvenTree/serializers.py index cb3297c5fc..dcadd9e6c8 100644 --- a/src/backend/InvenTree/InvenTree/serializers.py +++ b/src/backend/InvenTree/InvenTree/serializers.py @@ -9,7 +9,6 @@ from typing import Optional from django.contrib.auth import get_user_model from django.contrib.contenttypes.models import ContentType from django.core.exceptions import ValidationError as DjangoValidationError -from django.core.files.storage import default_storage from django.db import models from django.db.models import QuerySet from django.utils.translation import gettext_lazy as _ @@ -26,10 +25,11 @@ from rest_framework.serializers import DecimalField, Serializer from rest_framework.utils import model_meta from taggit.serializers import TaggitSerializer +import InvenTree.fast_representation as fast_representation import InvenTree.ready from common.currency import currency_code_default, currency_code_mappings from InvenTree.fields import InvenTreeRestURLField, InvenTreeURLField -from InvenTree.helpers import str2bool +from InvenTree.helpers import get_media_url, str2bool from InvenTree.helpers_model import getModelsWithMixin @@ -784,6 +784,27 @@ class InvenTreeModelSerializer(serializers.ModelSerializer): super().__init__(instance, data, **kwargs) + # Use the fast representation path (see InvenTree.fast_representation) + # Set to False on a subclass to always use the standard DRF implementation + FAST_REPRESENTATION = True + + def to_representation(self, instance): + """Serialize the provided instance. + + Uses a fast path which produces identical output to DRF, but avoids + the generic per-field overhead for simple fields. + """ + if ( + self.FAST_REPRESENTATION + and isinstance(instance, models.Model) + and fast_representation.check_serializer_class( + type(self), InvenTreeModelSerializer + ) + ): + return fast_representation.fast_to_representation(self, instance) + + return super().to_representation(instance) + def get_initial(self): """Construct initial data for the serializer. @@ -941,7 +962,7 @@ class InvenTreeAttachmentSerializerField(serializers.FileField): if not value: return None - return default_storage.url(str(value)) + return get_media_url(str(value)) class InvenTreeThumbnailField(serializers.CharField): @@ -975,7 +996,7 @@ class InvenTreeImageSerializerField(serializers.ImageField): if not value: return None - return default_storage.url(str(value)) + return get_media_url(str(value)) class InvenTreeDecimalField(serializers.FloatField): diff --git a/src/backend/InvenTree/InvenTree/test_fast_representation.py b/src/backend/InvenTree/InvenTree/test_fast_representation.py new file mode 100644 index 0000000000..26782a66c5 --- /dev/null +++ b/src/backend/InvenTree/InvenTree/test_fast_representation.py @@ -0,0 +1,302 @@ +"""Tests for the fast serializer representation path, and fast media URL generation.""" + +import datetime +import random +from zoneinfo import ZoneInfo + +from django.core.files.storage import default_storage +from django.test import TestCase, override_settings +from django.urls import reverse +from django.utils import timezone + +from rest_framework import serializers + +import InvenTree.fast_representation as fr +from InvenTree.helpers import get_media_url +from InvenTree.serializers import InvenTreeModelSerializer +from InvenTree.unit_test import InvenTreeAPITestCase +from stock.models import StockItem + + +class SampleStockSerializer(InvenTreeModelSerializer): + """Serializer exercising each fast-path mode, plus fallback cases.""" + + class Meta: + """Meta options.""" + + model = StockItem + fields = [ + 'pk', + 'batch', + 'serial', + 'delete_on_deplete', + 'is_building', + 'part', + 'location', + 'updated', + 'creation_date', + 'char_from_int', + 'int_from_bool', + 'float_from_decimal', + 'readonly_value', + 'callable_source', + 'missing_null', + 'missing_skip', + ] + + # VALUE fields with mismatched types -> field.to_representation() coercion + char_from_int = serializers.CharField(source='part_id', read_only=True) + int_from_bool = serializers.IntegerField(source='is_building', read_only=True) + float_from_decimal = serializers.FloatField(source='quantity', read_only=True) + readonly_value = serializers.ReadOnlyField(source='quantity') + + # Callable source -> generic path (DRF calls the method) + callable_source = serializers.CharField(source='get_absolute_url', read_only=True) + + # Missing attributes -> DRF default handling (None, or key omitted) + missing_null = serializers.CharField( + source='no_such_attribute', read_only=True, allow_null=True + ) + missing_skip = serializers.CharField(source='no_such_attribute', read_only=True) + + +class FastRepresentationTest(TestCase): + """Unit tests for InvenTree.fast_representation.""" + + fixtures = ['category', 'part', 'location', 'stock'] + + def compare(self, serializer_class, instances, **kwargs): + """Assert that fast and standard (DRF) output are identical.""" + self.assertGreater(len(instances), 0) + + try: + serializer_class.FAST_REPRESENTATION = False + expected = serializer_class(instances, many=True, **kwargs).data + finally: + serializer_class.FAST_REPRESENTATION = True + + result = serializer_class(instances, many=True, **kwargs).data + + self.assertEqual(len(result), len(expected)) + + for a, b in zip(result, expected, strict=True): + self.assertEqual(a, b) + # Key order must also be preserved + self.assertEqual(list(a.keys()), list(b.keys())) + + return result + + def test_plan_modes(self): + """Fields are classified into the expected fast-path modes.""" + serializer = SampleStockSerializer() + plan = { + name: mode for name, _, mode, *_ in fr.build_plan(serializer, StockItem) + } + + self.assertEqual(plan['batch'], fr.VALUE) + self.assertEqual(plan['delete_on_deplete'], fr.VALUE) + self.assertEqual(plan['readonly_value'], fr.VALUE) + self.assertEqual(plan['part'], fr.PK) + self.assertEqual(plan['location'], fr.PK) + self.assertEqual(plan['updated'], fr.DATETIME) + self.assertEqual(plan['creation_date'], fr.DATETIME) + + # Custom datetime format -> not supported by fast path + field = serializers.DateTimeField(format='%Y') + field.bind('x', serializer) + self.assertEqual(fr.classify_field(field, StockItem)[0], fr.GENERIC) + + # Dotted source -> generic + field = serializers.CharField(source='part.name') + field.bind('y', serializer) + self.assertEqual(fr.classify_field(field, StockItem)[0], fr.GENERIC) + + def test_sample_serializer(self): + """Fast output matches DRF for all field modes and fallbacks.""" + items = list(StockItem.objects.all()) + + # Include some edge-case values + items[0].batch = None + items[1].location = None + items[2].updated = None + items[3].serial = 12345 # Non-str value for a CharField + + result = self.compare(SampleStockSerializer, items) + + self.assertNotIn('missing_skip', result[0]) + self.assertIsNone(result[0]['missing_null']) + self.assertEqual(result[3]['serial'], '12345') + + def test_datetime_timezones(self): + """Datetime output matches DRF across timezone / format settings.""" + items = list(StockItem.objects.all()[:10]) + + aware = datetime.datetime(2024, 6, 30, 23, 45, 12, tzinfo=datetime.UTC) + naive = datetime.datetime(2024, 1, 15, 3, 4, 5) + + for idx, item in enumerate(items): + item.updated = aware if idx % 2 else naive + item.creation_date = naive if idx % 2 else aware + + for use_tz in (True, False): + for tz in ('UTC', 'Australia/Sydney', 'America/New_York'): + for fmt in ('%Y-%m-%d %H:%M', 'iso-8601', None): + with ( + self.subTest(use_tz=use_tz, tz=tz, fmt=fmt), + override_settings( + USE_TZ=use_tz, + TIME_ZONE=tz, + REST_FRAMEWORK={'DATETIME_FORMAT': fmt}, + ), + timezone.override(ZoneInfo(tz)), + ): + self.compare(SampleStockSerializer, items) + + def test_mro_guard(self): + """A to_representation() override after the base class disables the fast path.""" + + class LateMixin(serializers.Serializer): + def to_representation(self, instance): + data = super().to_representation(instance) + data['late'] = True + return data + + class GuardedSerializer(InvenTreeModelSerializer, LateMixin): + class Meta: + model = StockItem + fields = ['pk', 'batch'] + + self.assertFalse( + fr.check_serializer_class(GuardedSerializer, InvenTreeModelSerializer) + ) + self.assertTrue( + fr.check_serializer_class(SampleStockSerializer, InvenTreeModelSerializer) + ) + + data = GuardedSerializer(StockItem.objects.first()).data + self.assertTrue(data['late']) + + def test_non_model_instance(self): + """Non-model instances (e.g. dicts) use the standard path.""" + + class DictSerializer(InvenTreeModelSerializer): + class Meta: + model = StockItem + fields = ['batch', 'serial'] + + data = DictSerializer({'batch': 'B1', 'serial': 7}).data + self.assertEqual(data, {'batch': 'B1', 'serial': '7'}) + + +class FastRepresentationApiTest(InvenTreeAPITestCase): + """Compare API list output with the fast path enabled vs disabled.""" + + fixtures = [ + 'category', + 'part', + 'location', + 'stock', + 'bom', + 'company', + 'supplier_part', + 'manufacturer_part', + 'params', + 'test_templates', + ] + + roles = 'all' + superuser = True + + URLS = [ + ( + 'api-stock-list', + 'part_detail=true&location_detail=true&supplier_part_detail=true', + ), + ('api-part-list', 'category_detail=true&location_detail=true¶meters=true'), + ('api-part-category-list', 'path_detail=true'), + ('api-location-list', 'path_detail=true'), + ('api-company-list', ''), + ( + 'api-supplier-part-list', + 'part_detail=true&supplier_detail=true&manufacturer_detail=true', + ), + ('api-manufacturer-part-list', 'part_detail=true&manufacturer_detail=true'), + ('api-bom-list', 'part_detail=true&sub_part_detail=true'), + ] + + def test_api_equivalence(self): + """API responses are identical with and without the fast path.""" + for name, query in self.URLS: + url = f'{reverse(name)}?{query}' + + with self.subTest(url=url): + try: + InvenTreeModelSerializer.FAST_REPRESENTATION = False + expected = self.get(url, expected_code=200).data + finally: + InvenTreeModelSerializer.FAST_REPRESENTATION = True + + result = self.get(url, expected_code=200).data + + self.assertGreater(len(result), 0) + self.assertEqual(result, expected) + + +class MediaUrlTest(TestCase): + """Tests for InvenTree.helpers.get_media_url.""" + + NAMES = [ + '', + 'a.png', + '/a.png', + '//a.png', + 'part_images/x.thumbnail.png', + 'a/./b.png', + 'a/../b.png', + '../x', + './x', + '.', + '..', + 'a/..b/c', + 'a//b', + 'a?b#c;d:e', + 'http://evil.com/x', + 'C:\\x\\y.png', + 'ünï cödé.png', + '%2e%2e/x', + 'x/', + ] + + def check_names(self, names): + """Assert get_media_url matches default_storage.url for all names.""" + for name in names: + self.assertEqual(get_media_url(name), default_storage.url(name), name) + + def test_equivalence(self): + """Output matches default_storage.url() for the filesystem backend.""" + rnd = random.Random(0) + alphabet = "ab./\\?#;:%~!*()'é " + fuzz = [ + ''.join(rnd.choice(alphabet) for _ in range(rnd.randint(0, 12))) + for _ in range(2000) + ] + + for media_url in ('/media/', 'https://cdn.example.com/media/'): + with override_settings(MEDIA_URL=media_url): + self.check_names(self.NAMES + fuzz) + + @override_settings( + STORAGES={ + 'default': { + 'BACKEND': 'django.core.files.storage.InMemoryStorage', + 'OPTIONS': {'base_url': '/memory/'}, + }, + 'staticfiles': { + 'BACKEND': 'django.contrib.staticfiles.storage.StaticFilesStorage' + }, + } + ) + def test_other_backend(self): + """Other storage backends are delegated to default_storage.url().""" + self.assertEqual(get_media_url('a/b.png'), '/memory/a/b.png') + self.check_names(self.NAMES)