From 94692563fe5ad83f4c2cee59b6ba4bae4188351c Mon Sep 17 00:00:00 2001 From: Oliver Date: Mon, 21 Sep 2026 13:05:21 +1000 Subject: [PATCH] [bug] Bulk import fix (#12898) * Handle auto_now and auto_now_add fields for bulk import - Prevent them from being set to the current date * Add regression testing --- .../management/commands/bulkloaddata.py | 48 +++++++++- .../InvenTree/InvenTree/test_commands.py | 95 +++++++++++++++++++ 2 files changed, 138 insertions(+), 5 deletions(-) diff --git a/src/backend/InvenTree/InvenTree/management/commands/bulkloaddata.py b/src/backend/InvenTree/InvenTree/management/commands/bulkloaddata.py index ec7f20e21d..d5e8f43274 100644 --- a/src/backend/InvenTree/InvenTree/management/commands/bulkloaddata.py +++ b/src/backend/InvenTree/InvenTree/management/commands/bulkloaddata.py @@ -22,6 +22,38 @@ DEFAULT_BATCH_SIZE = 500 PROGRESS_UPDATE_INTERVAL = 100 +@contextmanager +def _suspend_auto_now(model): + """Temporarily disable auto_now / auto_now_add on a model's fields. + + Normal (non-bulk) fixture loading saves each object with raw=True, which + tells Field.pre_save() to leave auto_now/auto_now_add fields alone and + use the fixture's own value. bulk_create() has no equivalent of that + raw=True path - it always calls Field.pre_save(), which unconditionally + stamps such fields with timezone.now(), discarding whatever value the + fixture provided (e.g. stock.StockItemTracking.date). Clearing the flags + for the duration of the bulk_create() call restores the raw=True + behaviour for this model. + """ + affected = [ + field + for field in model._meta.local_fields + if getattr(field, 'auto_now', False) or getattr(field, 'auto_now_add', False) + ] + saved = [(field, field.auto_now, field.auto_now_add) for field in affected] + + for field in affected: + field.auto_now = False + field.auto_now_add = False + + try: + yield + finally: + for field, auto_now, auto_now_add in saved: + field.auto_now = auto_now + field.auto_now_add = auto_now_add + + class Command(LoadDataCommand): """Load fixtures using bulk_create() for improved performance. @@ -30,6 +62,11 @@ class Command(LoadDataCommand): - pre_save / post_save signals are not sent, and Model.save() / full_clean() are bypassed entirely (this is a Django bulk_create() limitation). + auto_now / auto_now_add fields are a partial exception: they are + temporarily disabled around the bulk_create() call (see + _suspend_auto_now()) so that fixture-provided values for fields such as + stock.StockItemTracking.date survive the import, matching what a + normal, non-bulk fixture load does via raw=True. - Multi-table inheritance is not supported by bulk_create() and will fail loudly rather than being silently mishandled. Natural-key foreign key / many-to-many resolution *is* supported (falling back to an individual, @@ -181,11 +218,12 @@ class Command(LoadDataCommand): """Bulk-create every object buffered so far, grouped by model.""" for model, objs in self.pending_objs.items(): try: - model._default_manager.db_manager(self.using).bulk_create( - [obj.object for obj in objs], - batch_size=self.batch_size, - ignore_conflicts=self.ignore_conflicts, - ) + with _suspend_auto_now(model): + model._default_manager.db_manager(self.using).bulk_create( + [obj.object for obj in objs], + batch_size=self.batch_size, + ignore_conflicts=self.ignore_conflicts, + ) except (DatabaseError, IntegrityError, ValueError) as e: e.args = ( f'Could not bulk-create {len(objs)} object(s) of {model._meta.label}: {e}', diff --git a/src/backend/InvenTree/InvenTree/test_commands.py b/src/backend/InvenTree/InvenTree/test_commands.py index b36cf9ef32..5414c6bde9 100644 --- a/src/backend/InvenTree/InvenTree/test_commands.py +++ b/src/backend/InvenTree/InvenTree/test_commands.py @@ -161,6 +161,101 @@ class CommandTestCase(TestCase): ContentType.objects.filter(pk__in=pks).delete() tmp_file.unlink(missing_ok=True) + def test_bulkloaddata_preserves_auto_now_add(self): + """bulk_create() must not overwrite fixture values for auto_now_add fields. + + Covers both an auto_now_add DateTimeField (BarcodeScanResult.timestamp, + StockItem.creation_date, StockItemTracking.date) and an auto_now_add + DateField (Part.creation_date) - StockItem/StockItemTracking are also + the models the bug was originally reported against. + """ + import datetime + + from django.core import serializers + from django.utils import timezone + + from common.models import BarcodeScanResult + from part.models import Part + from stock.models import StockItem, StockItemTracking + from stock.status_codes import StockHistoryCode + + # JSON fixtures only round-trip datetimes to millisecond precision + # (DjangoJSONEncoder truncates microseconds) - use a value already at + # that precision so the round-trip comparisons below are exact. + original_timestamp = (timezone.now() - datetime.timedelta(days=30)).replace( + microsecond=123000 + ) + original_date = original_timestamp.date() + + entry = BarcodeScanResult.objects.create(data='test-barcode') + entry.timestamp = original_timestamp + entry.save() + entry.refresh_from_db() + self.assertEqual(entry.timestamp, original_timestamp) + + part = Part.objects.create( + name='Bulkload test part', description='Bulkload test part' + ) + part.creation_date = original_date + part.save() + part.refresh_from_db() + self.assertEqual(part.creation_date, original_date) + + item = StockItem.objects.create(part=part, quantity=10) + item.creation_date = original_timestamp + item.save() + item.refresh_from_db() + self.assertEqual(item.creation_date, original_timestamp) + + tracking = StockItemTracking.objects.create( + item=item, tracking_type=StockHistoryCode.CREATED + ) + tracking.date = original_timestamp + tracking.save() + tracking.refresh_from_db() + self.assertEqual(tracking.date, original_timestamp) + + pks = { + 'barcode': entry.pk, + 'part': part.pk, + 'item': item.pk, + 'tracking': tracking.pk, + } + + # Serialize parent-before-child, so bulkloaddata's per-model + # bulk_create() calls happen in an order that satisfies FK constraints. + data = serializers.serialize('json', [entry, part, item, tracking]) + + # Use queryset deletes - Part.delete() refuses to delete an active part + StockItemTracking.objects.filter(pk=tracking.pk).delete() + StockItem.objects.filter(pk=item.pk).delete() + Part.objects.filter(pk=part.pk).delete() + BarcodeScanResult.objects.filter(pk=entry.pk).delete() + + tmp_file = get_testfolder_dir().joinpath('bulkloaddata_auto_now_test.json') + tmp_file.write_text(data, encoding='utf-8') + + try: + call_command('bulkloaddata', str(tmp_file), verbosity=0) + + reloaded_entry = BarcodeScanResult.objects.get(pk=pks['barcode']) + self.assertEqual(reloaded_entry.timestamp, original_timestamp) + + reloaded_part = Part.objects.get(pk=pks['part']) + self.assertEqual(reloaded_part.creation_date, original_date) + + reloaded_item = StockItem.objects.get(pk=pks['item']) + self.assertEqual(reloaded_item.creation_date, original_timestamp) + + reloaded_tracking = StockItemTracking.objects.get(pk=pks['tracking']) + self.assertEqual(reloaded_tracking.date, original_timestamp) + finally: + StockItemTracking.objects.filter(pk=pks['tracking']).delete() + StockItem.objects.filter(pk=pks['item']).delete() + Part.objects.filter(pk=pks['part']).delete() + BarcodeScanResult.objects.filter(pk=pks['barcode']).delete() + tmp_file.unlink(missing_ok=True) + def test_bulkdumpdata_natural_key_caching(self): """Test that bulkdumpdata caches natural-key FK resolution during serialization.""" from django.contrib.admin.models import ADDITION, LogEntry