refactor(backend): filtered endpoints - generic testing and small fixes (#10602)

* move filtering of serializer fields out of functions into mixin

* fix def

* temp fix

* rollback rollback

* more adoption

* fix many serializer behaviour

* optimize mro

* set many serializer

* adjust default filtering

* fix import

* add missed field

* make can_filter suppport more complex scenarios:
- different filtername from fieldname
- multiple fields with one filtername

* fix removal

* fix schema?

* add missing def

* add test

* fix schema fields

* fix another serializer issue

* further fixes

* extend tests

* also process strings

* fix serializer for schema

* ensure popped values are persisted

* move test around

* cleanup

* simplify tests

* fix typo

* fix another test

* var tests

* disable additional tests

* make application of PathScopedMixin more intentional -> more efficient

* make safer to use with various sanity checks

* fix list serializer

* add option to ignore special cases

* generalize addition

* remove generalize addition

* re-add missing schema generation exception

* remove new duplication

* fix style

* adjust naming and docs, add typing to clean stuff up

* simplify more

* fix ref calc

* Add generic test for serializer

* enable query based filtering

* enable previously disabled filters

* test failure modes

* reduce diff

* make check more robust

* add more INVE-I2 checks

* improve check

* make check and test more robust

* enable controlling query parameters per field

* ignore in coverage

* Remove project_code filter from BuildSerializer

Removed project_code filter from BuildSerializer.

* fix style

* Revert "Remove project_code filter from BuildSerializer"

This reverts commit 504eff0fd7.

* Revert "fix style"

This reverts commit 8e31db95d3.

src/backend/InvenTree/InvenTree/mixins.py

@@ -223,6 +223,20 @@ class OutputOptionsMixin:
         if getattr(cls, 'output_options', None) is not None:
             schema_for_view_output_options(cls)
 
+    def __init__(self) -> None:
+        """Initialize the mixin. Check that the serializer is compatible."""
+        super().__init__()
+
+        # Check that the serializer was defined
+        if (
+            hasattr(self, 'serializer_class')
+            and isinstance(self.serializer_class, type)
+            and (not issubclass(self.serializer_class, FilterableSerializerMixin))
+        ):
+            raise Exception(
+                'INVE-I2: `OutputOptionsMixin` can only be used with serializers that contain the `FilterableSerializerMixin` mixin'
+            )
+
     def get_serializer(self, *args, **kwargs):
         """Return serializer instance with output options applied."""
         if self.output_options and hasattr(self, 'request'):

src/backend/InvenTree/InvenTree/serializers.py

@@ -49,7 +49,10 @@ class FilterableSerializerField:
 
 
 def enable_filter(
-    func: Any, default_include: bool = False, filter_name: Optional[str] = None
+    func: Any,
+    default_include: bool = False,
+    filter_name: Optional[str] = None,
+    filter_by_query: bool = True,
 ):
     """Decorator for marking a serializer field as filterable.
 
@@ -59,6 +62,10 @@ def enable_filter(
         func: The serializer field to mark as filterable. Will automatically be passed when used as a decorator.
         default_include (bool): If True, the field will be included by default unless explicitly excluded. If False, the field will be excluded by default unless explicitly included.
         filter_name (str, optional): The name of the filter parameter to use in the URL. If None, the function name of the (decorated) function will be used.
+        filter_by_query (bool): If True, also look for filter parameters in the request query parameters.
+
+    Returns:
+        The decorated serializer field, marked as filterable.
     """
     # Ensure this function can be actually filteres
     if not issubclass(func.__class__, FilterableSerializerField):
@@ -71,6 +78,7 @@ def enable_filter(
     func._kwargs['is_filterable_vals'] = {
         'default': default_include,
         'filter_name': filter_name if filter_name else func.field_name,
+        'filter_by_query': filter_by_query,
     }
     return func
 
@@ -85,6 +93,9 @@ class FilterableSerializerMixin:
 
     _was_filtered = False
     no_filters = False
+    """If True, do not raise an exception if no filterable fields are found."""
+    filter_on_query = True
+    """If True, also look for filter parameters in the request query parameters."""
 
     def __init__(self, *args, **kwargs):
         """Initialization routine for the serializer. This gathers and applies filters through kwargs."""
@@ -113,12 +124,24 @@ class FilterableSerializerMixin:
             if getattr(a, 'is_filterable', None)
         }
 
+        # Gather query parameters from the request context
+        query_params = {}
+        if context := kwargs.get('context', {}):
+            query_params = dict(getattr(context.get('request', {}), 'query_params', {}))
+
         # Remove filter args from kwargs to avoid issues with super().__init__
         poped_kwargs = {}  # store popped kwargs as a arg might be reused for multiple fields
         tgs_vals: dict[str, bool] = {}
         for k, v in self.filter_targets.items():
             pop_ref = v['filter_name'] or k
             val = kwargs.pop(pop_ref, poped_kwargs.get(pop_ref))
+
+            # Optionally also look in query parameters
+            if val is None and self.filter_on_query and v.get('filter_by_query', True):
+                val = query_params.pop(pop_ref, None)
+                if isinstance(val, list) and len(val) == 1:
+                    val = val[0]
+
             if val:  # Save popped value for reuse
                 poped_kwargs[pop_ref] = val
             tgs_vals[k] = (

src/backend/InvenTree/InvenTree/test_serializers.py

@@ -0,0 +1,215 @@
+"""Low level tests for serializers."""
+
+from django.contrib import admin
+from django.contrib.auth.models import User
+from django.urls import path, reverse
+
+from rest_framework.serializers import SerializerMethodField
+
+import InvenTree.serializers
+from InvenTree.mixins import ListCreateAPI, OutputOptionsMixin
+from InvenTree.unit_test import InvenTreeAPITestCase
+from InvenTree.urls import backendpatterns
+
+
+class SampleSerializer(
+    InvenTree.serializers.FilterableSerializerMixin,
+    InvenTree.serializers.InvenTreeModelSerializer,
+):
+    """Sample serializer for testing FilterableSerializerMixin."""
+
+    class Meta:
+        """Meta options."""
+
+        model = User
+        fields = ['field_a', 'field_b', 'field_c', 'field_d', 'field_e', 'id']
+
+    field_a = SerializerMethodField(method_name='sample')
+    field_b = InvenTree.serializers.enable_filter(
+        InvenTree.serializers.FilterableSerializerMethodField(method_name='sample')
+    )
+    field_c = InvenTree.serializers.enable_filter(
+        InvenTree.serializers.FilterableSerializerMethodField(method_name='sample'),
+        True,
+        filter_name='crazy_name',
+    )
+    field_d = InvenTree.serializers.enable_filter(
+        InvenTree.serializers.FilterableSerializerMethodField(method_name='sample'),
+        True,
+        filter_name='crazy_name',
+    )
+    field_e = InvenTree.serializers.enable_filter(
+        InvenTree.serializers.FilterableSerializerMethodField(method_name='sample'),
+        filter_name='field_e',
+        filter_by_query=False,
+    )
+
+    def sample(self, obj):
+        """Sample method field."""
+        return 'sample123'
+
+
+class SampleList(OutputOptionsMixin, ListCreateAPI):
+    """List endpoint sample."""
+
+    serializer_class = SampleSerializer
+    queryset = User.objects.all()
+    permission_classes = []
+
+
+urlpatterns = [
+    path('', SampleList.as_view(), name='sample-list'),
+    path('admin/', admin.site.urls, name='inventree-admin'),
+]
+urlpatterns += backendpatterns
+
+
+class FilteredSerializers(InvenTreeAPITestCase):
+    """Tests for functionality of FilteredSerializerMixin / adjacent functions."""
+
+    def test_basic_setup(self):
+        """Test simple sample setup."""
+        with self.settings(
+            ROOT_URLCONF=__name__,
+            CSRF_TRUSTED_ORIGINS=['http://testserver'],
+            SITE_URL='http://testserver',
+        ):
+            url = reverse('sample-list', urlconf=__name__)
+
+            # Default request (no filters)
+            response = self.client.get(url)
+            self.assertContains(response, 'field_a')
+            self.assertNotContains(response, 'field_b')
+            self.assertContains(response, 'field_c')
+            self.assertContains(response, 'field_d')
+
+            # Request with filter for field_b
+            response = self.client.get(url, {'field_b': True})
+            self.assertContains(response, 'field_a')
+            self.assertContains(response, 'field_b')
+            self.assertContains(response, 'field_c')
+            self.assertContains(response, 'field_d')
+
+            self.assertEqual(response.data[0]['field_b'], 'sample123')
+
+            # Disable field_c using custom filter name
+            response = self.client.get(url, {'crazy_name': 'false'})
+            self.assertContains(response, 'field_a')
+            self.assertNotContains(response, 'field_b')
+            self.assertNotContains(response, 'field_c')
+            self.assertNotContains(response, 'field_d')
+
+            # Query parameters being turned off means it should not be enable-able
+            response = self.client.get(url, {'field_e': True})
+            self.assertContains(response, 'field_a')
+            self.assertNotContains(response, 'field_b')
+            self.assertContains(response, 'field_c')
+            self.assertContains(response, 'field_d')
+            self.assertNotContains(response, 'field_e')
+
+    def test_failiure_enable_filter(self):
+        """Test sanity check for enable_filter."""
+        # Allowed usage
+        field_b = InvenTree.serializers.enable_filter(  # noqa: F841
+            InvenTree.serializers.FilterableSerializerMethodField(method_name='sample')
+        )
+
+        # Disallowed usage
+        with self.assertRaises(Exception) as cm:
+            field_a = InvenTree.serializers.enable_filter(  # noqa: F841
+                SerializerMethodField(method_name='sample')
+            )
+        self.assertIn(
+            'INVE-I2: `enable_filter` can only be applied to serializer fields',
+            str(cm.exception),
+        )
+
+    def test_failiure_FilterableSerializerMixin(self):
+        """Test failure case for FilteredSerializerMixin."""
+
+        class BadSerializer(
+            InvenTree.serializers.FilterableSerializerMixin,
+            InvenTree.serializers.InvenTreeModelSerializer,
+        ):
+            """Bad serializer for testing FilterableSerializerMixin."""
+
+            class Meta:
+                """Meta options."""
+
+                model = User
+                fields = ['field_a', 'id']
+
+            field_a = SerializerMethodField(method_name='sample')
+
+            def sample(self, obj):
+                """Sample method field."""
+                return 'sample'  # pragma: no cover
+
+        with self.assertRaises(Exception) as cm:
+            _ = BadSerializer()
+        self.assertIn(
+            'INVE-I2: No filter targets found in fields, remove `PathScopedMixin`',
+            str(cm.exception),
+        )
+
+        # Test override
+        BadSerializer.no_filters = True
+        _ = BadSerializer()
+        self.assertTrue(True)  # Dummy assertion to ensure we reach here
+
+    def test_failiure_OutputOptionsMixin(self):
+        """Test failure case for OutputOptionsMixin."""
+
+        class BadSerializer(InvenTree.serializers.InvenTreeModelSerializer):
+            """Sample serializer."""
+
+            class Meta:
+                """Meta options."""
+
+                model = User
+                fields = ['id']
+
+            field_a = SerializerMethodField(method_name='sample')
+
+        # Bad implementation of OutputOptionsMixin
+        with self.assertRaises(Exception) as cm:
+
+            class BadList(OutputOptionsMixin, ListCreateAPI):
+                """Bad list endpoint for testing OutputOptionsMixin."""
+
+                serializer_class = BadSerializer
+                queryset = User.objects.all()
+                permission_classes = []
+
+            self.assertTrue(True)
+            _ = BadList()  # this should raise an exception
+        self.assertEqual(
+            str(cm.exception),
+            'INVE-I2: `OutputOptionsMixin` can only be used with serializers that contain the `FilterableSerializerMixin` mixin',
+        )
+
+        # More creative bad implementation
+        with self.assertRaises(Exception) as cm:
+
+            class BadList(OutputOptionsMixin, ListCreateAPI):
+                """Bad list endpoint for testing OutputOptionsMixin."""
+
+                queryset = User.objects.all()
+                permission_classes = []
+
+                def get_serializer(self, *args, **kwargs):
+                    """Get serializer override."""
+                    self.serializer_class = BadSerializer
+                    return super().get_serializer(*args, **kwargs)
+
+            view = BadList()
+            self.assertTrue(True)
+            # mock some stuff to allow get_serializer to run
+            view.request = self.client.request()
+            view.format_kwarg = {}
+            view.get_serializer()  # this should raise an exception
+
+        self.assertEqual(
+            str(cm.exception),
+            'INVE-I2: `OutputOptionsMixin` can only be used with serializers that contain the `FilterableSerializerMixin` mixin',
+        )

src/backend/InvenTree/InvenTree/urls.py

@@ -139,10 +139,7 @@ backendpatterns = [
         RedirectView.as_view(url=f'/{settings.FRONTEND_URL_BASE}', permanent=False),
         name='account_login',
     ),  # Add a redirect for login views
-    path('api/', include(apipatterns)),
+    path('anymail/', include('anymail.urls')),  # Emails
-    path('api-doc/', SpectacularRedocView.as_view(url_name='schema'), name='api-doc'),
-    # Emails
-    path('anymail/', include('anymail.urls')),
 ]
 
 urlpatterns = []
@@ -157,6 +154,10 @@ if settings.INVENTREE_ADMIN_ENABLED:
     ]
 
 urlpatterns += backendpatterns
+urlpatterns += [  # API URLs
+    path('api/', include(apipatterns)),
+    path('api-doc/', SpectacularRedocView.as_view(url_name='schema'), name='api-doc'),
+]
 urlpatterns += platform_urls
 
 # Append custom plugin URLs (if custom plugin support is enabled)

src/backend/InvenTree/build/serializers.py

@@ -33,6 +33,7 @@ from generic.states.fields import InvenTreeCustomStatusSerializerMixin
 from InvenTree.mixins import DataImportExportSerializerMixin
 from InvenTree.serializers import (
     FilterableCharField,
+    FilterableIntegerField,
     FilterableSerializerMixin,
     InvenTreeDecimalField,
     InvenTreeModelSerializer,
@@ -163,6 +164,17 @@ class BuildSerializer(
         filter_name='project_code_detail',
     )
 
+    project_code = enable_filter(
+        FilterableIntegerField(
+            allow_null=True,
+            required=False,
+            label=_('Project Code'),
+            help_text=_('Project code for this build order'),
+        ),
+        True,
+        filter_name='project_code_detail',
+    )
+
     @staticmethod
     def annotate_queryset(queryset):
         """Add custom annotations to the BuildSerializer queryset, performing database queries as efficiently as possible.

src/backend/InvenTree/build/test_api.py

@@ -1194,11 +1194,11 @@ class BuildListTest(BuildAPITest):
         self.run_output_test(
             self.url,
             [
-                'part_detail'
+                'part_detail',
-                # TODO re-enable ('project_code_detail', 'project_code'),
+                ('project_code_detail', 'project_code'),
-                # TODO re-enable 'project_code_detail',
+                'project_code_detail',
-                # TODO re-enable ('user_detail', 'responsible_detail'),
+                ('user_detail', 'responsible_detail'),
-                # TODO re-enable ('user_detail', 'issued_by_detail'),
+                ('user_detail', 'issued_by_detail'),
             ],
             additional_params={'limit': 1},
             assert_fnc=lambda x: x.data['results'][0],

src/backend/InvenTree/part/test_api.py

@@ -1404,8 +1404,8 @@ class PartAPITest(PartAPITestBase):
                 ('location_detail', 'default_location_detail'),
                 'parameters',
                 ('path_detail', 'category_path'),
-                # TODO re-enable ('pricing', 'pricing_min'),
+                ('pricing', 'pricing_min'),
-                # TODO re-enable ('pricing', 'pricing_updated'),
+                ('pricing', 'pricing_updated'),
             ],
             assert_subset=True,
         )
@@ -2761,8 +2761,8 @@ class BomItemTest(InvenTreeAPITestCase):
                 'can_build',
                 'part_detail',
                 'sub_part_detail',
-                # TODO re-enable 'substitutes',
+                'substitutes',
-                # TODO re-enable ('pricing', 'pricing_min'),
+                ('pricing', 'pricing_min'),
             ],
         )