InvenTreeModelSerializer now has 'fast_representation' optimization

src/backend/InvenTree/InvenTree/fast_representation.py

@@ -0,0 +1,239 @@
+"""Fast-path serializer output for large API list queries.
+
+DRF's Serializer.to_representation() handles every field generically:
+
+    attribute = field.get_attribute(instance)   # nested getattr, callable checks, error handling
+    value = field.to_representation(attribute)  # type coercion
+
+For a 1,000 row list with ~70 fields per row, that machinery is a large fraction of the request
+time, even though for most fields it reduces to a plain getattr() of a value which is already the
+correct output type.
+
+This module builds (once per serializer instance) a "representation plan", which classifies each
+readable field as either:
+
+- VALUE:    simple field with single attribute source, whose to_representation() is an identity
+            for values of known exact types (e.g. CharField and str). Read directly via getattr.
+- PK:       PrimaryKeyRelatedField for a forward relation - read the FK column (<name>_id) directly,
+            exactly as DRF's 'pk only' optimization does via Model.serializable_value().
+- DATETIME: DateTimeField using the default output format and timezone, with the
+            current timezone resolved once per row rather than once per value.
+- GENERIC:  anything else - handled exactly as DRF does.
+
+Any value which does not match the expected type (or raises an exception) falls back to the
+standard DRF code path for that field, so output is identical to DRF's.
+"""
+
+import datetime
+from typing import Any
+
+from django.conf import settings
+from django.utils import timezone
+
+from rest_framework import serializers
+from rest_framework.fields import SkipField
+from rest_framework.relations import PKOnlyObject
+from rest_framework.settings import api_settings
+
+# Field plan modes
+GENERIC = 0
+VALUE = 1
+PK = 2
+DATETIME = 3
+
+# Serializer field classes (exact type) for which to_representation(value) == value,
+# when type(value) is one of the listed (exact) types
+IDENTITY_TYPES: dict[type, tuple[type, ...]] = {
+    serializers.CharField: (str,),
+    serializers.IntegerField: (int,),
+    serializers.FloatField: (float,),
+    serializers.BooleanField: (bool,),
+    serializers.ReadOnlyField: (str, int, float, bool),
+}
+
+# Cache of whether a serializer class may use the fast path (see check_serializer_class)
+_class_check_cache: dict[type, bool] = {}
+
+
+def check_serializer_class(cls: type, base: type) -> bool:
+    """Determine whether the provided serializer class can use the fast path.
+
+    The fast path replaces Serializer.to_representation() - so it is only valid if the
+    next to_representation() implementation in the MRO after 'base' is DRF's own.
+    """
+    if cls in _class_check_cache:
+        return _class_check_cache[cls]
+
+    mro = cls.__mro__
+    result = True
+
+    for klass in mro[mro.index(base) + 1 :]:
+        if 'to_representation' in klass.__dict__:
+            result = klass is serializers.Serializer
+            break
+
+    _class_check_cache[cls] = result
+    return result
+
+
+def classify_field(field, model_class: type) -> tuple:
+    """Return the plan entry (mode, attribute, extra) for a single serializer field."""
+    source_attrs = getattr(field, 'source_attrs', None)
+
+    # Only fields which read a single attribute directly from the instance
+    if not source_attrs or len(source_attrs) != 1:
+        return (GENERIC, None, None)
+
+    attr = source_attrs[0]
+    field_type = type(field)
+
+    if field_type in IDENTITY_TYPES:
+        return (VALUE, attr, IDENTITY_TYPES[field_type])
+
+    if field_type is serializers.PrimaryKeyRelatedField and field.pk_field is None:
+        try:
+            model_field = model_class._meta.get_field(attr)
+        except Exception:
+            return (GENERIC, None, None)
+
+        if (
+            model_field.concrete
+            and model_field.is_relation
+            and (model_field.many_to_one or model_field.one_to_one)
+        ):
+            return (PK, model_field.attname, None)
+
+    if field_type is serializers.DateTimeField:
+        # Only the default format + timezone behavior is supported
+        if not hasattr(field, 'format') and not hasattr(field, 'timezone'):
+            return (DATETIME, attr, api_settings.DATETIME_FORMAT)
+
+    return (GENERIC, None, None)
+
+
+def build_plan(serializer, model_class: type) -> list[tuple]:
+    """Construct the representation plan for the provided serializer."""
+    return [
+        (field.field_name, field, *classify_field(field, model_class))
+        for field in serializer._readable_fields
+    ]
+
+
+def represent_generic(field, instance, ret: dict) -> None:
+    """Represent a single field exactly as DRF Serializer.to_representation() does."""
+    try:
+        attribute = field.get_attribute(instance)
+    except SkipField:
+        return
+
+    check_for_none = attribute.pk if isinstance(attribute, PKOnlyObject) else attribute
+
+    if check_for_none is None:
+        ret[field.field_name] = None
+    else:
+        ret[field.field_name] = field.to_representation(attribute)
+
+
+class _FallbackError(Exception):
+    """Internal signal to use the generic code path."""
+
+
+def represent_datetime(value: datetime.datetime, tz, output_format):
+    """Equivalent to DRF DateTimeField.to_representation(), with a pre-resolved timezone.
+
+    Raises _FallbackError for edge cases (naive datetime with USE_TZ, overflow, etc).
+    """
+    # Note: DRF returns the value *without* timezone conversion if no format is specified
+    if output_format is None:
+        return value
+
+    aware = value.utcoffset() is not None
+
+    if tz is not None:
+        if not aware:
+            raise _FallbackError
+        try:
+            value = value.astimezone(tz)
+        except OverflowError:
+            raise _FallbackError
+    elif aware:
+        raise _FallbackError
+
+    if output_format.lower() == 'iso-8601':
+        value = value.isoformat()
+        if value.endswith('+00:00'):
+            value = value[:-6] + 'Z'
+        return value
+
+    return value.strftime(output_format)
+
+
+def fast_to_representation(serializer, instance) -> dict[str, Any]:
+    """Return the serialized representation of 'instance', using the fast path where possible.
+
+    The output is identical to DRF Serializer.to_representation()
+    """
+    model_class = type(instance)
+
+    plan_cache = getattr(serializer, '_fast_repr_plan', None)
+
+    if plan_cache is None or plan_cache[0] is not model_class:
+        plan_cache = (model_class, build_plan(serializer, model_class))
+        serializer._fast_repr_plan = plan_cache
+
+    plan = plan_cache[1]
+
+    ret = {}
+    tz = None
+    tz_resolved = False
+
+    for name, field, mode, attr, extra in plan:
+        if mode == VALUE:
+            try:
+                value = getattr(instance, attr)
+            except Exception:
+                # Let DRF handle defaults / missing attributes / related object lookups
+                represent_generic(field, instance, ret)
+                continue
+
+            if value is None:
+                ret[name] = None
+            elif type(value) in extra:
+                ret[name] = value
+            elif callable(value):
+                represent_generic(field, instance, ret)
+            else:
+                ret[name] = field.to_representation(value)
+
+        elif mode == PK:
+            ret[name] = getattr(instance, attr)
+
+        elif mode == DATETIME:
+            try:
+                value = getattr(instance, attr)
+            except Exception:
+                represent_generic(field, instance, ret)
+                continue
+
+            if value is None:
+                ret[name] = None
+                continue
+
+            if type(value) is not datetime.datetime:
+                represent_generic(field, instance, ret)
+                continue
+
+            if not tz_resolved:
+                # Matches DRF DateTimeField.default_timezone()
+                tz = timezone.get_current_timezone() if settings.USE_TZ else None
+                tz_resolved = True
+
+            try:
+                ret[name] = represent_datetime(value, tz, extra)
+            except _FallbackError:
+                ret[name] = field.to_representation(value)
+
+        else:
+            represent_generic(field, instance, ret)
+
+    return ret

src/backend/InvenTree/InvenTree/serializers.py

@@ -9,7 +9,6 @@ from typing import Optional
 from django.contrib.auth import get_user_model
 from django.contrib.contenttypes.models import ContentType
 from django.core.exceptions import ValidationError as DjangoValidationError
-from django.core.files.storage import default_storage
 from django.db import models
 from django.db.models import QuerySet
 from django.utils.translation import gettext_lazy as _
@@ -26,10 +25,11 @@ from rest_framework.serializers import DecimalField, Serializer
 from rest_framework.utils import model_meta
 from taggit.serializers import TaggitSerializer
 
+import InvenTree.fast_representation as fast_representation
 import InvenTree.ready
 from common.currency import currency_code_default, currency_code_mappings
 from InvenTree.fields import InvenTreeRestURLField, InvenTreeURLField
-from InvenTree.helpers import str2bool
+from InvenTree.helpers import get_media_url, str2bool
 from InvenTree.helpers_model import getModelsWithMixin
 
 
@@ -784,6 +784,27 @@ class InvenTreeModelSerializer(serializers.ModelSerializer):
 
         super().__init__(instance, data, **kwargs)
 
+    # Use the fast representation path (see InvenTree.fast_representation)
+    # Set to False on a subclass to always use the standard DRF implementation
+    FAST_REPRESENTATION = True
+
+    def to_representation(self, instance):
+        """Serialize the provided instance.
+
+        Uses a fast path which produces identical output to DRF, but avoids
+        the generic per-field overhead for simple fields.
+        """
+        if (
+            self.FAST_REPRESENTATION
+            and isinstance(instance, models.Model)
+            and fast_representation.check_serializer_class(
+                type(self), InvenTreeModelSerializer
+            )
+        ):
+            return fast_representation.fast_to_representation(self, instance)
+
+        return super().to_representation(instance)
+
     def get_initial(self):
         """Construct initial data for the serializer.
 
@@ -941,7 +962,7 @@ class InvenTreeAttachmentSerializerField(serializers.FileField):
         if not value:
             return None
 
-        return default_storage.url(str(value))
+        return get_media_url(str(value))
 
 
 class InvenTreeThumbnailField(serializers.CharField):
@@ -975,7 +996,7 @@ class InvenTreeImageSerializerField(serializers.ImageField):
         if not value:
             return None
 
-        return default_storage.url(str(value))
+        return get_media_url(str(value))
 
 
 class InvenTreeDecimalField(serializers.FloatField):

src/backend/InvenTree/InvenTree/test_fast_representation.py

@@ -0,0 +1,302 @@
+"""Tests for the fast serializer representation path, and fast media URL generation."""
+
+import datetime
+import random
+from zoneinfo import ZoneInfo
+
+from django.core.files.storage import default_storage
+from django.test import TestCase, override_settings
+from django.urls import reverse
+from django.utils import timezone
+
+from rest_framework import serializers
+
+import InvenTree.fast_representation as fr
+from InvenTree.helpers import get_media_url
+from InvenTree.serializers import InvenTreeModelSerializer
+from InvenTree.unit_test import InvenTreeAPITestCase
+from stock.models import StockItem
+
+
+class SampleStockSerializer(InvenTreeModelSerializer):
+    """Serializer exercising each fast-path mode, plus fallback cases."""
+
+    class Meta:
+        """Meta options."""
+
+        model = StockItem
+        fields = [
+            'pk',
+            'batch',
+            'serial',
+            'delete_on_deplete',
+            'is_building',
+            'part',
+            'location',
+            'updated',
+            'creation_date',
+            'char_from_int',
+            'int_from_bool',
+            'float_from_decimal',
+            'readonly_value',
+            'callable_source',
+            'missing_null',
+            'missing_skip',
+        ]
+
+    # VALUE fields with mismatched types -> field.to_representation() coercion
+    char_from_int = serializers.CharField(source='part_id', read_only=True)
+    int_from_bool = serializers.IntegerField(source='is_building', read_only=True)
+    float_from_decimal = serializers.FloatField(source='quantity', read_only=True)
+    readonly_value = serializers.ReadOnlyField(source='quantity')
+
+    # Callable source -> generic path (DRF calls the method)
+    callable_source = serializers.CharField(source='get_absolute_url', read_only=True)
+
+    # Missing attributes -> DRF default handling (None, or key omitted)
+    missing_null = serializers.CharField(
+        source='no_such_attribute', read_only=True, allow_null=True
+    )
+    missing_skip = serializers.CharField(source='no_such_attribute', read_only=True)
+
+
+class FastRepresentationTest(TestCase):
+    """Unit tests for InvenTree.fast_representation."""
+
+    fixtures = ['category', 'part', 'location', 'stock']
+
+    def compare(self, serializer_class, instances, **kwargs):
+        """Assert that fast and standard (DRF) output are identical."""
+        self.assertGreater(len(instances), 0)
+
+        try:
+            serializer_class.FAST_REPRESENTATION = False
+            expected = serializer_class(instances, many=True, **kwargs).data
+        finally:
+            serializer_class.FAST_REPRESENTATION = True
+
+        result = serializer_class(instances, many=True, **kwargs).data
+
+        self.assertEqual(len(result), len(expected))
+
+        for a, b in zip(result, expected, strict=True):
+            self.assertEqual(a, b)
+            # Key order must also be preserved
+            self.assertEqual(list(a.keys()), list(b.keys()))
+
+        return result
+
+    def test_plan_modes(self):
+        """Fields are classified into the expected fast-path modes."""
+        serializer = SampleStockSerializer()
+        plan = {
+            name: mode for name, _, mode, *_ in fr.build_plan(serializer, StockItem)
+        }
+
+        self.assertEqual(plan['batch'], fr.VALUE)
+        self.assertEqual(plan['delete_on_deplete'], fr.VALUE)
+        self.assertEqual(plan['readonly_value'], fr.VALUE)
+        self.assertEqual(plan['part'], fr.PK)
+        self.assertEqual(plan['location'], fr.PK)
+        self.assertEqual(plan['updated'], fr.DATETIME)
+        self.assertEqual(plan['creation_date'], fr.DATETIME)
+
+        # Custom datetime format -> not supported by fast path
+        field = serializers.DateTimeField(format='%Y')
+        field.bind('x', serializer)
+        self.assertEqual(fr.classify_field(field, StockItem)[0], fr.GENERIC)
+
+        # Dotted source -> generic
+        field = serializers.CharField(source='part.name')
+        field.bind('y', serializer)
+        self.assertEqual(fr.classify_field(field, StockItem)[0], fr.GENERIC)
+
+    def test_sample_serializer(self):
+        """Fast output matches DRF for all field modes and fallbacks."""
+        items = list(StockItem.objects.all())
+
+        # Include some edge-case values
+        items[0].batch = None
+        items[1].location = None
+        items[2].updated = None
+        items[3].serial = 12345  # Non-str value for a CharField
+
+        result = self.compare(SampleStockSerializer, items)
+
+        self.assertNotIn('missing_skip', result[0])
+        self.assertIsNone(result[0]['missing_null'])
+        self.assertEqual(result[3]['serial'], '12345')
+
+    def test_datetime_timezones(self):
+        """Datetime output matches DRF across timezone / format settings."""
+        items = list(StockItem.objects.all()[:10])
+
+        aware = datetime.datetime(2024, 6, 30, 23, 45, 12, tzinfo=datetime.UTC)
+        naive = datetime.datetime(2024, 1, 15, 3, 4, 5)
+
+        for idx, item in enumerate(items):
+            item.updated = aware if idx % 2 else naive
+            item.creation_date = naive if idx % 2 else aware
+
+        for use_tz in (True, False):
+            for tz in ('UTC', 'Australia/Sydney', 'America/New_York'):
+                for fmt in ('%Y-%m-%d %H:%M', 'iso-8601', None):
+                    with (
+                        self.subTest(use_tz=use_tz, tz=tz, fmt=fmt),
+                        override_settings(
+                            USE_TZ=use_tz,
+                            TIME_ZONE=tz,
+                            REST_FRAMEWORK={'DATETIME_FORMAT': fmt},
+                        ),
+                        timezone.override(ZoneInfo(tz)),
+                    ):
+                        self.compare(SampleStockSerializer, items)
+
+    def test_mro_guard(self):
+        """A to_representation() override after the base class disables the fast path."""
+
+        class LateMixin(serializers.Serializer):
+            def to_representation(self, instance):
+                data = super().to_representation(instance)
+                data['late'] = True
+                return data
+
+        class GuardedSerializer(InvenTreeModelSerializer, LateMixin):
+            class Meta:
+                model = StockItem
+                fields = ['pk', 'batch']
+
+        self.assertFalse(
+            fr.check_serializer_class(GuardedSerializer, InvenTreeModelSerializer)
+        )
+        self.assertTrue(
+            fr.check_serializer_class(SampleStockSerializer, InvenTreeModelSerializer)
+        )
+
+        data = GuardedSerializer(StockItem.objects.first()).data
+        self.assertTrue(data['late'])
+
+    def test_non_model_instance(self):
+        """Non-model instances (e.g. dicts) use the standard path."""
+
+        class DictSerializer(InvenTreeModelSerializer):
+            class Meta:
+                model = StockItem
+                fields = ['batch', 'serial']
+
+        data = DictSerializer({'batch': 'B1', 'serial': 7}).data
+        self.assertEqual(data, {'batch': 'B1', 'serial': '7'})
+
+
+class FastRepresentationApiTest(InvenTreeAPITestCase):
+    """Compare API list output with the fast path enabled vs disabled."""
+
+    fixtures = [
+        'category',
+        'part',
+        'location',
+        'stock',
+        'bom',
+        'company',
+        'supplier_part',
+        'manufacturer_part',
+        'params',
+        'test_templates',
+    ]
+
+    roles = 'all'
+    superuser = True
+
+    URLS = [
+        (
+            'api-stock-list',
+            'part_detail=true&location_detail=true&supplier_part_detail=true',
+        ),
+        ('api-part-list', 'category_detail=true&location_detail=true&parameters=true'),
+        ('api-part-category-list', 'path_detail=true'),
+        ('api-location-list', 'path_detail=true'),
+        ('api-company-list', ''),
+        (
+            'api-supplier-part-list',
+            'part_detail=true&supplier_detail=true&manufacturer_detail=true',
+        ),
+        ('api-manufacturer-part-list', 'part_detail=true&manufacturer_detail=true'),
+        ('api-bom-list', 'part_detail=true&sub_part_detail=true'),
+    ]
+
+    def test_api_equivalence(self):
+        """API responses are identical with and without the fast path."""
+        for name, query in self.URLS:
+            url = f'{reverse(name)}?{query}'
+
+            with self.subTest(url=url):
+                try:
+                    InvenTreeModelSerializer.FAST_REPRESENTATION = False
+                    expected = self.get(url, expected_code=200).data
+                finally:
+                    InvenTreeModelSerializer.FAST_REPRESENTATION = True
+
+                result = self.get(url, expected_code=200).data
+
+                self.assertGreater(len(result), 0)
+                self.assertEqual(result, expected)
+
+
+class MediaUrlTest(TestCase):
+    """Tests for InvenTree.helpers.get_media_url."""
+
+    NAMES = [
+        '',
+        'a.png',
+        '/a.png',
+        '//a.png',
+        'part_images/x.thumbnail.png',
+        'a/./b.png',
+        'a/../b.png',
+        '../x',
+        './x',
+        '.',
+        '..',
+        'a/..b/c',
+        'a//b',
+        'a?b#c;d:e',
+        'http://evil.com/x',
+        'C:\\x\\y.png',
+        'ünï cödé.png',
+        '%2e%2e/x',
+        'x/',
+    ]
+
+    def check_names(self, names):
+        """Assert get_media_url matches default_storage.url for all names."""
+        for name in names:
+            self.assertEqual(get_media_url(name), default_storage.url(name), name)
+
+    def test_equivalence(self):
+        """Output matches default_storage.url() for the filesystem backend."""
+        rnd = random.Random(0)
+        alphabet = "ab./\\?#;:%~!*()'é "
+        fuzz = [
+            ''.join(rnd.choice(alphabet) for _ in range(rnd.randint(0, 12)))
+            for _ in range(2000)
+        ]
+
+        for media_url in ('/media/', 'https://cdn.example.com/media/'):
+            with override_settings(MEDIA_URL=media_url):
+                self.check_names(self.NAMES + fuzz)
+
+    @override_settings(
+        STORAGES={
+            'default': {
+                'BACKEND': 'django.core.files.storage.InMemoryStorage',
+                'OPTIONS': {'base_url': '/memory/'},
+            },
+            'staticfiles': {
+                'BACKEND': 'django.contrib.staticfiles.storage.StaticFilesStorage'
+            },
+        }
+    )
+    def test_other_backend(self):
+        """Other storage backends are delegated to default_storage.url()."""
+        self.assertEqual(get_media_url('a/b.png'), '/memory/a/b.png')
+        self.check_names(self.NAMES)