mirror of
https://github.com/inventree/InvenTree.git
synced 2026-09-27 14:16:02 +00:00
[bug] Bulk import fix (#12898)
* Handle auto_now and auto_now_add fields for bulk import - Prevent them from being set to the current date * Add regression testing
This commit is contained in:
@@ -22,6 +22,38 @@ DEFAULT_BATCH_SIZE = 500
|
||||
PROGRESS_UPDATE_INTERVAL = 100
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _suspend_auto_now(model):
|
||||
"""Temporarily disable auto_now / auto_now_add on a model's fields.
|
||||
|
||||
Normal (non-bulk) fixture loading saves each object with raw=True, which
|
||||
tells Field.pre_save() to leave auto_now/auto_now_add fields alone and
|
||||
use the fixture's own value. bulk_create() has no equivalent of that
|
||||
raw=True path - it always calls Field.pre_save(), which unconditionally
|
||||
stamps such fields with timezone.now(), discarding whatever value the
|
||||
fixture provided (e.g. stock.StockItemTracking.date). Clearing the flags
|
||||
for the duration of the bulk_create() call restores the raw=True
|
||||
behaviour for this model.
|
||||
"""
|
||||
affected = [
|
||||
field
|
||||
for field in model._meta.local_fields
|
||||
if getattr(field, 'auto_now', False) or getattr(field, 'auto_now_add', False)
|
||||
]
|
||||
saved = [(field, field.auto_now, field.auto_now_add) for field in affected]
|
||||
|
||||
for field in affected:
|
||||
field.auto_now = False
|
||||
field.auto_now_add = False
|
||||
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
for field, auto_now, auto_now_add in saved:
|
||||
field.auto_now = auto_now
|
||||
field.auto_now_add = auto_now_add
|
||||
|
||||
|
||||
class Command(LoadDataCommand):
|
||||
"""Load fixtures using bulk_create() for improved performance.
|
||||
|
||||
@@ -30,6 +62,11 @@ class Command(LoadDataCommand):
|
||||
|
||||
- pre_save / post_save signals are not sent, and Model.save() / full_clean()
|
||||
are bypassed entirely (this is a Django bulk_create() limitation).
|
||||
auto_now / auto_now_add fields are a partial exception: they are
|
||||
temporarily disabled around the bulk_create() call (see
|
||||
_suspend_auto_now()) so that fixture-provided values for fields such as
|
||||
stock.StockItemTracking.date survive the import, matching what a
|
||||
normal, non-bulk fixture load does via raw=True.
|
||||
- Multi-table inheritance is not supported by bulk_create() and will fail
|
||||
loudly rather than being silently mishandled. Natural-key foreign key /
|
||||
many-to-many resolution *is* supported (falling back to an individual,
|
||||
@@ -181,11 +218,12 @@ class Command(LoadDataCommand):
|
||||
"""Bulk-create every object buffered so far, grouped by model."""
|
||||
for model, objs in self.pending_objs.items():
|
||||
try:
|
||||
model._default_manager.db_manager(self.using).bulk_create(
|
||||
[obj.object for obj in objs],
|
||||
batch_size=self.batch_size,
|
||||
ignore_conflicts=self.ignore_conflicts,
|
||||
)
|
||||
with _suspend_auto_now(model):
|
||||
model._default_manager.db_manager(self.using).bulk_create(
|
||||
[obj.object for obj in objs],
|
||||
batch_size=self.batch_size,
|
||||
ignore_conflicts=self.ignore_conflicts,
|
||||
)
|
||||
except (DatabaseError, IntegrityError, ValueError) as e:
|
||||
e.args = (
|
||||
f'Could not bulk-create {len(objs)} object(s) of {model._meta.label}: {e}',
|
||||
|
||||
@@ -161,6 +161,101 @@ class CommandTestCase(TestCase):
|
||||
ContentType.objects.filter(pk__in=pks).delete()
|
||||
tmp_file.unlink(missing_ok=True)
|
||||
|
||||
def test_bulkloaddata_preserves_auto_now_add(self):
|
||||
"""bulk_create() must not overwrite fixture values for auto_now_add fields.
|
||||
|
||||
Covers both an auto_now_add DateTimeField (BarcodeScanResult.timestamp,
|
||||
StockItem.creation_date, StockItemTracking.date) and an auto_now_add
|
||||
DateField (Part.creation_date) - StockItem/StockItemTracking are also
|
||||
the models the bug was originally reported against.
|
||||
"""
|
||||
import datetime
|
||||
|
||||
from django.core import serializers
|
||||
from django.utils import timezone
|
||||
|
||||
from common.models import BarcodeScanResult
|
||||
from part.models import Part
|
||||
from stock.models import StockItem, StockItemTracking
|
||||
from stock.status_codes import StockHistoryCode
|
||||
|
||||
# JSON fixtures only round-trip datetimes to millisecond precision
|
||||
# (DjangoJSONEncoder truncates microseconds) - use a value already at
|
||||
# that precision so the round-trip comparisons below are exact.
|
||||
original_timestamp = (timezone.now() - datetime.timedelta(days=30)).replace(
|
||||
microsecond=123000
|
||||
)
|
||||
original_date = original_timestamp.date()
|
||||
|
||||
entry = BarcodeScanResult.objects.create(data='test-barcode')
|
||||
entry.timestamp = original_timestamp
|
||||
entry.save()
|
||||
entry.refresh_from_db()
|
||||
self.assertEqual(entry.timestamp, original_timestamp)
|
||||
|
||||
part = Part.objects.create(
|
||||
name='Bulkload test part', description='Bulkload test part'
|
||||
)
|
||||
part.creation_date = original_date
|
||||
part.save()
|
||||
part.refresh_from_db()
|
||||
self.assertEqual(part.creation_date, original_date)
|
||||
|
||||
item = StockItem.objects.create(part=part, quantity=10)
|
||||
item.creation_date = original_timestamp
|
||||
item.save()
|
||||
item.refresh_from_db()
|
||||
self.assertEqual(item.creation_date, original_timestamp)
|
||||
|
||||
tracking = StockItemTracking.objects.create(
|
||||
item=item, tracking_type=StockHistoryCode.CREATED
|
||||
)
|
||||
tracking.date = original_timestamp
|
||||
tracking.save()
|
||||
tracking.refresh_from_db()
|
||||
self.assertEqual(tracking.date, original_timestamp)
|
||||
|
||||
pks = {
|
||||
'barcode': entry.pk,
|
||||
'part': part.pk,
|
||||
'item': item.pk,
|
||||
'tracking': tracking.pk,
|
||||
}
|
||||
|
||||
# Serialize parent-before-child, so bulkloaddata's per-model
|
||||
# bulk_create() calls happen in an order that satisfies FK constraints.
|
||||
data = serializers.serialize('json', [entry, part, item, tracking])
|
||||
|
||||
# Use queryset deletes - Part.delete() refuses to delete an active part
|
||||
StockItemTracking.objects.filter(pk=tracking.pk).delete()
|
||||
StockItem.objects.filter(pk=item.pk).delete()
|
||||
Part.objects.filter(pk=part.pk).delete()
|
||||
BarcodeScanResult.objects.filter(pk=entry.pk).delete()
|
||||
|
||||
tmp_file = get_testfolder_dir().joinpath('bulkloaddata_auto_now_test.json')
|
||||
tmp_file.write_text(data, encoding='utf-8')
|
||||
|
||||
try:
|
||||
call_command('bulkloaddata', str(tmp_file), verbosity=0)
|
||||
|
||||
reloaded_entry = BarcodeScanResult.objects.get(pk=pks['barcode'])
|
||||
self.assertEqual(reloaded_entry.timestamp, original_timestamp)
|
||||
|
||||
reloaded_part = Part.objects.get(pk=pks['part'])
|
||||
self.assertEqual(reloaded_part.creation_date, original_date)
|
||||
|
||||
reloaded_item = StockItem.objects.get(pk=pks['item'])
|
||||
self.assertEqual(reloaded_item.creation_date, original_timestamp)
|
||||
|
||||
reloaded_tracking = StockItemTracking.objects.get(pk=pks['tracking'])
|
||||
self.assertEqual(reloaded_tracking.date, original_timestamp)
|
||||
finally:
|
||||
StockItemTracking.objects.filter(pk=pks['tracking']).delete()
|
||||
StockItem.objects.filter(pk=pks['item']).delete()
|
||||
Part.objects.filter(pk=pks['part']).delete()
|
||||
BarcodeScanResult.objects.filter(pk=pks['barcode']).delete()
|
||||
tmp_file.unlink(missing_ok=True)
|
||||
|
||||
def test_bulkdumpdata_natural_key_caching(self):
|
||||
"""Test that bulkdumpdata caches natural-key FK resolution during serialization."""
|
||||
from django.contrib.admin.models import ADDITION, LogEntry
|
||||
|
||||
Reference in New Issue
Block a user