Export records (#12887)

* Add 'bulkdumpdata' command

* Test --bulk option in CI

src/backend/InvenTree/InvenTree/management/commands/bulkdumpdata.py

@@ -0,0 +1,66 @@
+"""Custom management command to export fixtures faster, by caching natural-key lookups."""
+
+from contextlib import contextmanager
+
+from django.core.management.commands.dumpdata import Command as DumpDataCommand
+from django.core.serializers import python as serializers_python
+
+
+class Command(DumpDataCommand):
+    """Dump fixtures using a natural-key cache for improved performance.
+
+    Behaves identically to the built-in 'dumpdata' command, with one difference:
+
+    - When --natural-foreign is used, Django's serializer resolves each natural-key
+      FK by fetching the full related row (getattr(obj, field.name)) with no caching
+      of its own - see cached_handle_fk_field(). A large export can have many rows
+      referencing the same handful of natural-keyed objects (e.g. thousands of
+      stock.StockItem rows all pointing at a few part.Part records), which otherwise
+      costs one query per *row* instead of one query per distinct related object.
+      This mirrors the caching bulkloaddata.py already does on the import side.
+    """
+
+    def handle(self, *app_labels, **options):
+        """Wrap the base dumpdata command with a natural-key FK resolution cache."""
+        with self._cached_natural_keys():
+            super().handle(*app_labels, **options)
+
+    @contextmanager
+    def _cached_natural_keys(self):
+        """Cache natural-key foreign key resolutions for the duration of this block.
+
+        Keyed by (related model, raw FK id) rather than the natural key tuple
+        itself, so a cache hit never needs to touch the FK descriptor (and
+        therefore never issues a query) - only the raw id column already present
+        on the serialized object.
+        """
+        cache = {}
+        original = serializers_python.Serializer.handle_fk_field
+
+        def cached_handle_fk_field(serializer_self, obj, field):
+            if not (
+                serializer_self.use_natural_foreign_keys
+                and hasattr(field.remote_field.model, 'natural_key')
+            ):
+                return original(serializer_self, obj, field)
+
+            fk_id = getattr(obj, field.attname)
+
+            if fk_id is None:
+                serializer_self._current[field.name] = None
+                return
+
+            cache_key = (field.remote_field.model, fk_id)
+
+            if cache_key in cache:
+                serializer_self._current[field.name] = cache[cache_key]
+                return
+
+            original(serializer_self, obj, field)
+            cache[cache_key] = serializer_self._current[field.name]
+
+        serializers_python.Serializer.handle_fk_field = cached_handle_fk_field
+        try:
+            yield
+        finally:
+            serializers_python.Serializer.handle_fk_field = original

src/backend/InvenTree/InvenTree/ready.py

@@ -49,7 +49,10 @@ def isWaitingForDatabase():
 
 def isImportingData():
     """Returns True if the database is currently importing (or exporting) data, e.g. 'loaddata' command is performed."""
-    return any(x in sys.argv for x in ['flush', 'loaddata', 'bulkloaddata', 'dumpdata'])
+    return any(
+        x in sys.argv
+        for x in ['flush', 'loaddata', 'bulkloaddata', 'dumpdata', 'bulkdumpdata']
+    )
 
 
 def isRunningMigrations():

src/backend/InvenTree/InvenTree/test_commands.py

@@ -161,6 +161,74 @@ class CommandTestCase(TestCase):
             ContentType.objects.filter(pk__in=pks).delete()
             tmp_file.unlink(missing_ok=True)
 
+    def test_bulkdumpdata_natural_key_caching(self):
+        """Test that bulkdumpdata caches natural-key FK resolution during serialization."""
+        from django.contrib.admin.models import ADDITION, LogEntry
+        from django.contrib.contenttypes.models import ContentType
+        from django.core import serializers
+        from django.db import connection
+        from django.test.utils import CaptureQueriesContext
+
+        from InvenTree.management.commands.bulkdumpdata import (
+            Command as BulkDumpDataCommand,
+        )
+
+        user = User.objects.create_user(username='bulkdumpdata_test_user')
+        content_type = ContentType.objects.create(
+            app_label='bulkdumpdata_test', model='dummymodel'
+        )
+
+        entries = [
+            LogEntry.objects.create(
+                user=user,
+                content_type=content_type,
+                object_id=str(i),
+                object_repr=f'Object {i}',
+                action_flag=ADDITION,
+                change_message='Created',
+            )
+            for i in range(20)
+        ]
+        pks = [e.pk for e in entries]
+
+        def make_queryset():
+            # A fresh queryset each time, so FK descriptor caching on the
+            # instances themselves can't mask whether *our* cache is doing
+            # the work
+            return LogEntry.objects.filter(pk__in=pks).order_by('pk')
+
+        try:
+            with CaptureQueriesContext(connection) as uncached:
+                uncached_data = serializers.serialize(
+                    'json', make_queryset(), use_natural_foreign_keys=True
+                )
+
+            with CaptureQueriesContext(connection) as cached:
+                with BulkDumpDataCommand()._cached_natural_keys():
+                    cached_data = serializers.serialize(
+                        'json', make_queryset(), use_natural_foreign_keys=True
+                    )
+
+            # Same output either way - caching must not change what gets exported
+            self.assertEqual(uncached_data, cached_data)
+
+            # Without caching: one extra query per row for each repeated
+            # natural-keyed FK (content_type and user are both natural-keyed
+            # here, so up to 2 extra queries per row -> 40, plus the main select)
+            self.assertGreaterEqual(len(uncached.captured_queries), 40)
+
+            # With caching: only the first reference to each distinct related
+            # object (one content_type, one user) issues a query - every other
+            # row is served from cache
+            self.assertLessEqual(len(cached.captured_queries), 4)
+            self.assertLess(
+                len(cached.captured_queries), len(uncached.captured_queries)
+            )
+        finally:
+            LogEntry.objects.filter(pk__in=pks).delete()
+            content_type.delete()
+            user.delete()
+
     def test_backup_metadata(self):
         """Test the backup metadata functions."""
         from InvenTree.backup import (