From 8ec61aca0adc06b3d9c079b1c6e1a6aaa88472cc Mon Sep 17 00:00:00 2001
From: Oliver <oliver.henry.walters@gmail.com>
Date: Wed, 25 Mar 2026 00:35:08 +1100
Subject: [PATCH] Update DataExport functionality (#11604)

* Update DataExport functionality

- Chunk queryset into memory

* Allow larger number of queries for chunked database fetching

* Handle possible exception in unit testing
---
 src/backend/InvenTree/common/tests.py         | 11 +++++--
 .../plugin/base/integration/DataExport.py     | 30 ++++++++++++++++---
 src/backend/InvenTree/stock/test_api.py       |  2 +-
 3 files changed, 35 insertions(+), 8 deletions(-)

diff --git a/src/backend/InvenTree/common/tests.py b/src/backend/InvenTree/common/tests.py
index e6c850e0c3..a0e5bb198f 100644
--- a/src/backend/InvenTree/common/tests.py
+++ b/src/backend/InvenTree/common/tests.py
@@ -1575,9 +1575,14 @@ class CurrencyAPITests(InvenTreeAPITestCase):
 
         # Updating via the external exchange may not work every time
         for _idx in range(5):
-            self.post(
-                reverse('api-currency-refresh'), expected_code=200, max_query_time=30
-            )
+            try:
+                self.post(
+                    reverse('api-currency-refresh'),
+                    expected_code=200,
+                    max_query_time=30,
+                )
+            except Exception:
+                continue
 
             # There should be some new exchange rate objects now
             if Rate.objects.all().exists():
diff --git a/src/backend/InvenTree/plugin/base/integration/DataExport.py b/src/backend/InvenTree/plugin/base/integration/DataExport.py
index 97d38ec493..15649e78dc 100644
--- a/src/backend/InvenTree/plugin/base/integration/DataExport.py
+++ b/src/backend/InvenTree/plugin/base/integration/DataExport.py
@@ -22,6 +22,9 @@ class DataExportMixin:
 
     ExportOptionsSerializer = None
 
+    # How many rows to fetch into memory at once when exporting data?
+    EXPORT_CHUNK_SIZE: int = 250
+
     class MixinMeta:
         """Meta options for this mixin."""
 
@@ -108,10 +111,29 @@ class DataExportMixin:
 
         Returns: The exported data (a list of dict objects)
         """
-        # The default implementation simply serializes the queryset
-        return serializer_class(
-            queryset, many=True, exporting=True, context=serializer_context or {}
-        ).data
+        output.refresh_from_db()
+        N = queryset.count()
+
+        rows = []
+
+        offset = 0
+
+        while offset < N:
+            chunk = queryset[offset : offset + self.EXPORT_CHUNK_SIZE]
+
+            chunk_rows = serializer_class(
+                chunk, many=True, exporting=True, context=serializer_context or {}
+            ).data
+
+            rows.extend(chunk_rows)
+
+            offset += self.EXPORT_CHUNK_SIZE
+
+            # Update the export progress
+            output.progress += len(chunk_rows)
+            output.save()
+
+        return rows
 
     def get_export_options_serializer(self, **kwargs) -> serializers.Serializer | None:
         """Return a serializer class with dynamic export options for this plugin.
diff --git a/src/backend/InvenTree/stock/test_api.py b/src/backend/InvenTree/stock/test_api.py
index 1434a0e3f0..e2a7b22b0c 100644
--- a/src/backend/InvenTree/stock/test_api.py
+++ b/src/backend/InvenTree/stock/test_api.py
@@ -1038,7 +1038,7 @@ class StockItemListTest(StockAPITestCase):
         # Note: While the export is quick on pgsql, it is still quite slow on sqlite3
         with self.export_data(
             self.list_url,
-            max_query_count=50,
+            max_query_count=100,
             max_query_time=12.0,  # Test time increased due to worker variability
         ) as data_file:
             data = self.process_csv(data_file)