[bug] Bulk import fix (#12898)

* Handle auto_now and auto_now_add fields for bulk import

- Prevent them from being set to the current date

* Add regression testing
This commit is contained in:
Oliver
2026-09-21 13:05:21 +10:00
committed by GitHub
parent bb195a77b8
commit 94692563fe
2 changed files with 138 additions and 5 deletions
@@ -22,6 +22,38 @@ DEFAULT_BATCH_SIZE = 500
PROGRESS_UPDATE_INTERVAL = 100 PROGRESS_UPDATE_INTERVAL = 100
@contextmanager
def _suspend_auto_now(model):
"""Temporarily disable auto_now / auto_now_add on a model's fields.
Normal (non-bulk) fixture loading saves each object with raw=True, which
tells Field.pre_save() to leave auto_now/auto_now_add fields alone and
use the fixture's own value. bulk_create() has no equivalent of that
raw=True path - it always calls Field.pre_save(), which unconditionally
stamps such fields with timezone.now(), discarding whatever value the
fixture provided (e.g. stock.StockItemTracking.date). Clearing the flags
for the duration of the bulk_create() call restores the raw=True
behaviour for this model.
"""
affected = [
field
for field in model._meta.local_fields
if getattr(field, 'auto_now', False) or getattr(field, 'auto_now_add', False)
]
saved = [(field, field.auto_now, field.auto_now_add) for field in affected]
for field in affected:
field.auto_now = False
field.auto_now_add = False
try:
yield
finally:
for field, auto_now, auto_now_add in saved:
field.auto_now = auto_now
field.auto_now_add = auto_now_add
class Command(LoadDataCommand): class Command(LoadDataCommand):
"""Load fixtures using bulk_create() for improved performance. """Load fixtures using bulk_create() for improved performance.
@@ -30,6 +62,11 @@ class Command(LoadDataCommand):
- pre_save / post_save signals are not sent, and Model.save() / full_clean() - pre_save / post_save signals are not sent, and Model.save() / full_clean()
are bypassed entirely (this is a Django bulk_create() limitation). are bypassed entirely (this is a Django bulk_create() limitation).
auto_now / auto_now_add fields are a partial exception: they are
temporarily disabled around the bulk_create() call (see
_suspend_auto_now()) so that fixture-provided values for fields such as
stock.StockItemTracking.date survive the import, matching what a
normal, non-bulk fixture load does via raw=True.
- Multi-table inheritance is not supported by bulk_create() and will fail - Multi-table inheritance is not supported by bulk_create() and will fail
loudly rather than being silently mishandled. Natural-key foreign key / loudly rather than being silently mishandled. Natural-key foreign key /
many-to-many resolution *is* supported (falling back to an individual, many-to-many resolution *is* supported (falling back to an individual,
@@ -181,11 +218,12 @@ class Command(LoadDataCommand):
"""Bulk-create every object buffered so far, grouped by model.""" """Bulk-create every object buffered so far, grouped by model."""
for model, objs in self.pending_objs.items(): for model, objs in self.pending_objs.items():
try: try:
model._default_manager.db_manager(self.using).bulk_create( with _suspend_auto_now(model):
[obj.object for obj in objs], model._default_manager.db_manager(self.using).bulk_create(
batch_size=self.batch_size, [obj.object for obj in objs],
ignore_conflicts=self.ignore_conflicts, batch_size=self.batch_size,
) ignore_conflicts=self.ignore_conflicts,
)
except (DatabaseError, IntegrityError, ValueError) as e: except (DatabaseError, IntegrityError, ValueError) as e:
e.args = ( e.args = (
f'Could not bulk-create {len(objs)} object(s) of {model._meta.label}: {e}', f'Could not bulk-create {len(objs)} object(s) of {model._meta.label}: {e}',
@@ -161,6 +161,101 @@ class CommandTestCase(TestCase):
ContentType.objects.filter(pk__in=pks).delete() ContentType.objects.filter(pk__in=pks).delete()
tmp_file.unlink(missing_ok=True) tmp_file.unlink(missing_ok=True)
def test_bulkloaddata_preserves_auto_now_add(self):
"""bulk_create() must not overwrite fixture values for auto_now_add fields.
Covers both an auto_now_add DateTimeField (BarcodeScanResult.timestamp,
StockItem.creation_date, StockItemTracking.date) and an auto_now_add
DateField (Part.creation_date) - StockItem/StockItemTracking are also
the models the bug was originally reported against.
"""
import datetime
from django.core import serializers
from django.utils import timezone
from common.models import BarcodeScanResult
from part.models import Part
from stock.models import StockItem, StockItemTracking
from stock.status_codes import StockHistoryCode
# JSON fixtures only round-trip datetimes to millisecond precision
# (DjangoJSONEncoder truncates microseconds) - use a value already at
# that precision so the round-trip comparisons below are exact.
original_timestamp = (timezone.now() - datetime.timedelta(days=30)).replace(
microsecond=123000
)
original_date = original_timestamp.date()
entry = BarcodeScanResult.objects.create(data='test-barcode')
entry.timestamp = original_timestamp
entry.save()
entry.refresh_from_db()
self.assertEqual(entry.timestamp, original_timestamp)
part = Part.objects.create(
name='Bulkload test part', description='Bulkload test part'
)
part.creation_date = original_date
part.save()
part.refresh_from_db()
self.assertEqual(part.creation_date, original_date)
item = StockItem.objects.create(part=part, quantity=10)
item.creation_date = original_timestamp
item.save()
item.refresh_from_db()
self.assertEqual(item.creation_date, original_timestamp)
tracking = StockItemTracking.objects.create(
item=item, tracking_type=StockHistoryCode.CREATED
)
tracking.date = original_timestamp
tracking.save()
tracking.refresh_from_db()
self.assertEqual(tracking.date, original_timestamp)
pks = {
'barcode': entry.pk,
'part': part.pk,
'item': item.pk,
'tracking': tracking.pk,
}
# Serialize parent-before-child, so bulkloaddata's per-model
# bulk_create() calls happen in an order that satisfies FK constraints.
data = serializers.serialize('json', [entry, part, item, tracking])
# Use queryset deletes - Part.delete() refuses to delete an active part
StockItemTracking.objects.filter(pk=tracking.pk).delete()
StockItem.objects.filter(pk=item.pk).delete()
Part.objects.filter(pk=part.pk).delete()
BarcodeScanResult.objects.filter(pk=entry.pk).delete()
tmp_file = get_testfolder_dir().joinpath('bulkloaddata_auto_now_test.json')
tmp_file.write_text(data, encoding='utf-8')
try:
call_command('bulkloaddata', str(tmp_file), verbosity=0)
reloaded_entry = BarcodeScanResult.objects.get(pk=pks['barcode'])
self.assertEqual(reloaded_entry.timestamp, original_timestamp)
reloaded_part = Part.objects.get(pk=pks['part'])
self.assertEqual(reloaded_part.creation_date, original_date)
reloaded_item = StockItem.objects.get(pk=pks['item'])
self.assertEqual(reloaded_item.creation_date, original_timestamp)
reloaded_tracking = StockItemTracking.objects.get(pk=pks['tracking'])
self.assertEqual(reloaded_tracking.date, original_timestamp)
finally:
StockItemTracking.objects.filter(pk=pks['tracking']).delete()
StockItem.objects.filter(pk=pks['item']).delete()
Part.objects.filter(pk=pks['part']).delete()
BarcodeScanResult.objects.filter(pk=pks['barcode']).delete()
tmp_file.unlink(missing_ok=True)
def test_bulkdumpdata_natural_key_caching(self): def test_bulkdumpdata_natural_key_caching(self):
"""Test that bulkdumpdata caches natural-key FK resolution during serialization.""" """Test that bulkdumpdata caches natural-key FK resolution during serialization."""
from django.contrib.admin.models import ADDITION, LogEntry from django.contrib.admin.models import ADDITION, LogEntry