Bulk create events (#12405)

* Add initial benchmark test

* Enable event batching for BulkdCreate API mixin

* Fix duplicate save event for Parameter

* Apply event batching for bulk_update

* Similar testing for bulk_delete

src/backend/InvenTree/InvenTree/api.py

@@ -31,6 +31,8 @@ from common.settings import get_global_setting
 from InvenTree import helpers, ready
 from InvenTree.auth_overrides import registration_enabled
 from InvenTree.mixins import ListCreateAPI
+from InvenTree.tasks import batch_offload_tasks
+from plugin.base.event.events import batch_events
 from plugin.serializers import MetadataSerializer
 from users.models import ApiToken
 from users.permissions import check_user_permission, prefetch_rule_sets
@@ -511,7 +513,7 @@ class BulkCreateMixin:
                 if has_unique_errors:
                     raise ValidationError(unique_errors)
 
-            with transaction.atomic():
+            with transaction.atomic(), batch_events(), batch_offload_tasks():
                 for item in data:
                     serializer = self.get_serializer(data=item)
                     if serializer.is_valid():
@@ -595,7 +597,7 @@ class BulkUpdateMixin(BulkOperationMixin):
 
         instance_data = []
 
-        with transaction.atomic():
+        with transaction.atomic(), batch_events(), batch_offload_tasks():
             # Perform object update
             # Note that we do not perform a bulk-update operation here,
             # as we want to trigger any custom post_save methods on the model
@@ -692,7 +694,7 @@ class CommonBulkDeleteMixin(BulkOperationMixin):
         # Keep track of how many items we deleted
         n_deleted = queryset.count()
 
-        with transaction.atomic():
+        with transaction.atomic(), batch_events(), batch_offload_tasks():
             # Perform object deletion
             # Note that we do not perform a bulk-delete operation here,
             # as we want to trigger any custom post_delete methods on the model

src/backend/InvenTree/common/serializers.py

@@ -947,9 +947,7 @@ class ParameterSerializer(
         if not target_model_class.check_related_permission('change', user):
             raise PermissionDenied(permission_error_msg)
 
-        instance = super().save(**kwargs)
+        instance = super().save(updated_by=user, **kwargs)
-        instance.updated_by = user
-        instance.save()
 
         return instance
 

src/backend/InvenTree/common/test_api.py

@@ -6,6 +6,7 @@ from django.core.exceptions import ValidationError
 from django.core.files.base import ContentFile
 from django.core.files.storage import default_storage
 from django.core.files.uploadedfile import SimpleUploadedFile
+from django.test.utils import override_settings
 from django.urls import reverse
 
 from PIL import Image
@@ -13,7 +14,8 @@ from taggit.models import Tag
 
 import common.models
 from common.models import SelectionList, SelectionListEntry
-from InvenTree.unit_test import InvenTreeAPITestCase
+from common.settings import set_global_setting
+from InvenTree.unit_test import InvenTreeAPITestCase, findOffloadedEvent
 
 
 class DataOutputAPITests(InvenTreeAPITestCase):
@@ -569,6 +571,96 @@ class ParameterAPITests(InvenTreeAPITestCase):
             common.models.Parameter.objects.filter(pk=parameter.pk).exists()
         )
 
+    @override_settings(
+        TESTING_TABLE_EVENTS=True,
+        PLUGIN_TESTING_EVENTS=True,
+        PLUGIN_TESTING_EVENTS_ASYNC=True,
+    )
+    def test_bulk_create_parameters(self):
+        """Test bulk creation of parameters via the API.
+
+        Test that:
+            - The correct number of items are created
+            - Instance creation events are offloaded to the background worker
+        """
+        from django_q.models import OrmQ
+
+        from part.models import Part
+
+        self.assignRole('part.add')
+
+        OrmQ.objects.all().delete()
+
+        set_global_setting('ENABLE_PLUGINS_EVENTS', True)
+
+        template = common.models.ParameterTemplate.objects.create(
+            name='Test Parameter',
+            description='A parameter template for testing bulk creation',
+            model_type=None,
+        )
+
+        # Generate a set of parts
+        parts = [
+            Part.objects.create(
+                name=f'Test Part {ii}', description='A part for testing'
+            )
+            for ii in range(50)
+        ]
+
+        N = common.models.Parameter.objects.count()
+
+        # Bulk-create parameters
+        response = self.post(
+            reverse('api-parameter-list'),
+            data=[
+                {
+                    'template': template.pk,
+                    'model_type': 'part.part',
+                    'model_id': part.pk,
+                    'data': f'Test data {part.pk}',
+                }
+                for part in parts
+            ],
+            benchmark=True,
+            max_query_count=500,
+            max_query_time=2.0,
+        )
+
+        self.assertEqual(len(response.data), 50)
+
+        # Check that the parameters have been created
+        self.assertEqual(common.models.Parameter.objects.count(), N + len(parts))
+
+        # We expect that 50 events have been offloaded to the background worker
+        self.assertGreaterEqual(OrmQ.objects.count(), len(parts))
+
+        # There should be a parameter for each part
+        for part in parts:
+            self.assertEqual(part.parameters.count(), 1)
+            parameter = part.parameters.first()
+            self.assertIsNotNone(parameter)
+            self.assertIsNotNone(parameter.updated)
+            self.assertIsNotNone(parameter.updated_by)
+            self.assertEqual(parameter.updated_by, self.user)
+
+            # Check that an associated event has been offloaded
+            self.assertIsNotNone(
+                findOffloadedEvent(
+                    'part_partparameter.created', matching_kwargs={'id': parameter.pk}
+                ),
+                f'No created event found for parameter {parameter.pk}',
+            )
+
+            # Check that an extra 'saved' event is *NOT* generated
+            self.assertIsNone(
+                findOffloadedEvent(
+                    'part_partparameter.saved', matching_kwargs={'id': parameter.pk}
+                ),
+                f'Unexpected saved event found for parameter {parameter.pk}',
+            )
+
+        set_global_setting('ENABLE_PLUGINS_EVENTS', False)
+
     def test_parameter_uniqueness(self):
         """Test the uniqueness options which can be applied to a ParameterTemplate."""
         from company.models import Company

src/backend/InvenTree/part/test_api.py

@@ -7,7 +7,7 @@ from random import randint
 
 from django.core.exceptions import ValidationError
 from django.db import connection
-from django.test.utils import CaptureQueriesContext
+from django.test.utils import CaptureQueriesContext, override_settings
 from django.urls import reverse
 
 import pytest
@@ -19,9 +19,15 @@ import company.models
 import order.models
 from build.status_codes import BuildStatus
 from common.models import InvenTreeSetting, ParameterTemplate
+from common.settings import set_global_setting
 from company.models import Company, SupplierPart
 from InvenTree.config import get_testfolder_dir
-from InvenTree.unit_test import InvenTreeAPIPerformanceTestCase, InvenTreeAPITestCase
+from InvenTree.unit_test import (
+    InvenTreeAPIPerformanceTestCase,
+    InvenTreeAPITestCase,
+    findOffloadedEvent,
+    findOffloadedTask,
+)
 from order.status_codes import PurchaseOrderStatusGroups
 from part.models import (
     BomItem,
@@ -1506,6 +1512,75 @@ class PartAPITest(PartAPITestBase):
             for field in ['name', 'description', 'structural']:
                 self.assertIn(field, category)
 
+    @override_settings(
+        TESTING_TABLE_EVENTS=True,
+        PLUGIN_TESTING_EVENTS=True,
+        PLUGIN_TESTING_EVENTS_ASYNC=True,
+    )
+    def test_bulk_update(self):
+        """Test that we can bulk-update a set of parts via the API.
+
+        Test that:
+            - All parts are updated correctly
+            - Instance saved events are offloaded to the background worker
+        """
+        from django_q.models import OrmQ
+
+        self.assignRole('part.change')
+
+        set_global_setting('ENABLE_PLUGINS_EVENTS', True)
+
+        # Create a bunch of parts
+        parts = [
+            Part.objects.create(
+                name=f'Bulk part {i}',
+                description='A part for bulk update testing',
+                category=PartCategory.objects.first(),
+                active=True,
+            )
+            for i in range(10)
+        ]
+
+        for part in parts:
+            self.assertTrue(part.active)
+
+        # Clear out event registry
+        OrmQ.objects.all().delete()
+
+        # Bulk update all parts to be inactive
+        response = self.patch(
+            reverse('api-part-list'),
+            {'active': False, 'items': [part.pk for part in parts]},
+            expected_code=200,
+            benchmark=True,
+            max_query_count=250,
+            max_query_time=2.0,
+        )
+
+        self.assertEqual(len(response.data['items']), len(parts))
+
+        # Check that events have been registered
+        self.assertGreaterEqual(OrmQ.objects.count(), 2 * len(parts))
+
+        # Check that 'active' parameter has been updated for all parts
+        for part in parts:
+            part.refresh_from_db()
+            self.assertFalse(part.active)
+
+            self.assertIsNotNone(
+                findOffloadedEvent('part_part.saved', matching_kwargs={'id': part.pk}),
+                f'part_part.saved event not found for part {part.pk} not found in offloaded events',
+            )
+
+            self.assertIsNotNone(
+                findOffloadedTask(
+                    'part.tasks.rebuild_supplier_parts', matching_args=[part.pk]
+                ),
+                f'rebuild_supplier_parts task not found for part {part.pk} not found in offloaded tasks',
+            )
+
+        set_global_setting('ENABLE_PLUGINS_EVENTS', False)
+
 
 class PartCreationTests(PartAPITestBase):
     """Tests for creating new Part instances via the API."""

src/backend/InvenTree/report/serializers.py

@@ -37,9 +37,7 @@ class ReportSerializerBase(InvenTreeModelSerializer):
                 _('User must be authenticated to save report templates')
             )
 
-        instance = super().save(**kwargs)
+        instance = super().save(updated_by=user, **kwargs)
-        instance.updated_by = user
-        instance.save(increment_revision=False)
 
         return instance
 

src/backend/InvenTree/stock/test_api.py

@@ -6,6 +6,7 @@ from datetime import datetime, timedelta
 
 from django.contrib.contenttypes.models import ContentType
 from django.core.exceptions import ValidationError
+from django.test.utils import override_settings
 from django.urls import reverse
 
 import pytest
@@ -18,7 +19,11 @@ import order.models
 import part.models
 from common.models import InvenTreeCustomUserStateModel, InvenTreeSetting
 from common.settings import set_global_setting
-from InvenTree.unit_test import InvenTreeAPIPerformanceTestCase, InvenTreeAPITestCase
+from InvenTree.unit_test import (
+    InvenTreeAPIPerformanceTestCase,
+    InvenTreeAPITestCase,
+    findOffloadedEvent,
+)
 from part.models import Part, PartTestTemplate
 from stock.models import (
     StockItem,
@@ -3695,12 +3700,21 @@ class StockTestResultTest(StockAPITestCase):
             # Check that an attachment has been uploaded
             self.assertIsNotNone(response.data['attachment'])
 
+    @override_settings(
+        TESTING_TABLE_EVENTS=True,
+        PLUGIN_TESTING_EVENTS=True,
+        PLUGIN_TESTING_EVENTS_ASYNC=True,
+    )
     def test_bulk_delete(self):
         """Test that the BulkDelete endpoint works for this model."""
+        from django_q.models import OrmQ
+
         n = StockItemTestResult.objects.count()
 
         tests = []
 
+        set_global_setting('ENABLE_PLUGINS_EVENTS', True)
+
         url = reverse('api-stock-test-result-list')
 
         stock_item = StockItem.objects.get(pk=1)
@@ -3740,11 +3754,32 @@ class StockTestResultTest(StockAPITestCase):
         # Attempt a delete without providing items
         self.delete(url, {}, expected_code=400)
 
+        OrmQ.objects.all().delete()
+
         # Now, let's delete all the newly created items with a single API request
-        response = self.delete(url, {'items': tests}, expected_code=200)
+        response = self.delete(
+            url,
+            {'items': tests},
+            expected_code=200,
+            max_query_count=100,
+            benchmark=True,
+            max_query_time=0.5,
+        )
 
         self.assertEqual(StockItemTestResult.objects.count(), n)
 
+        self.assertGreaterEqual(OrmQ.objects.count(), len(tests))
+
+        # Ensure that an associated 'deleted' event has been offloaded
+        for test in tests:
+            self.assertIsNotNone(
+                findOffloadedEvent(
+                    'stock_stockitemtestresult.deleted', matching_kwargs={'id': test}
+                )
+            )
+
+        set_global_setting('ENABLE_PLUGINS_EVENTS', False)
+
     def test_value_choices(self):
         """Test that the 'value' field is correctly validated."""
         url = reverse('api-stock-test-result-list')