mirror of
https://github.com/inventree/InvenTree.git
synced 2026-09-27 14:16:02 +00:00
[bug] Bulk import fix (#12898)
* Handle auto_now and auto_now_add fields for bulk import - Prevent them from being set to the current date * Add regression testing
This commit is contained in:
@@ -22,6 +22,38 @@ DEFAULT_BATCH_SIZE = 500
|
|||||||
PROGRESS_UPDATE_INTERVAL = 100
|
PROGRESS_UPDATE_INTERVAL = 100
|
||||||
|
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def _suspend_auto_now(model):
|
||||||
|
"""Temporarily disable auto_now / auto_now_add on a model's fields.
|
||||||
|
|
||||||
|
Normal (non-bulk) fixture loading saves each object with raw=True, which
|
||||||
|
tells Field.pre_save() to leave auto_now/auto_now_add fields alone and
|
||||||
|
use the fixture's own value. bulk_create() has no equivalent of that
|
||||||
|
raw=True path - it always calls Field.pre_save(), which unconditionally
|
||||||
|
stamps such fields with timezone.now(), discarding whatever value the
|
||||||
|
fixture provided (e.g. stock.StockItemTracking.date). Clearing the flags
|
||||||
|
for the duration of the bulk_create() call restores the raw=True
|
||||||
|
behaviour for this model.
|
||||||
|
"""
|
||||||
|
affected = [
|
||||||
|
field
|
||||||
|
for field in model._meta.local_fields
|
||||||
|
if getattr(field, 'auto_now', False) or getattr(field, 'auto_now_add', False)
|
||||||
|
]
|
||||||
|
saved = [(field, field.auto_now, field.auto_now_add) for field in affected]
|
||||||
|
|
||||||
|
for field in affected:
|
||||||
|
field.auto_now = False
|
||||||
|
field.auto_now_add = False
|
||||||
|
|
||||||
|
try:
|
||||||
|
yield
|
||||||
|
finally:
|
||||||
|
for field, auto_now, auto_now_add in saved:
|
||||||
|
field.auto_now = auto_now
|
||||||
|
field.auto_now_add = auto_now_add
|
||||||
|
|
||||||
|
|
||||||
class Command(LoadDataCommand):
|
class Command(LoadDataCommand):
|
||||||
"""Load fixtures using bulk_create() for improved performance.
|
"""Load fixtures using bulk_create() for improved performance.
|
||||||
|
|
||||||
@@ -30,6 +62,11 @@ class Command(LoadDataCommand):
|
|||||||
|
|
||||||
- pre_save / post_save signals are not sent, and Model.save() / full_clean()
|
- pre_save / post_save signals are not sent, and Model.save() / full_clean()
|
||||||
are bypassed entirely (this is a Django bulk_create() limitation).
|
are bypassed entirely (this is a Django bulk_create() limitation).
|
||||||
|
auto_now / auto_now_add fields are a partial exception: they are
|
||||||
|
temporarily disabled around the bulk_create() call (see
|
||||||
|
_suspend_auto_now()) so that fixture-provided values for fields such as
|
||||||
|
stock.StockItemTracking.date survive the import, matching what a
|
||||||
|
normal, non-bulk fixture load does via raw=True.
|
||||||
- Multi-table inheritance is not supported by bulk_create() and will fail
|
- Multi-table inheritance is not supported by bulk_create() and will fail
|
||||||
loudly rather than being silently mishandled. Natural-key foreign key /
|
loudly rather than being silently mishandled. Natural-key foreign key /
|
||||||
many-to-many resolution *is* supported (falling back to an individual,
|
many-to-many resolution *is* supported (falling back to an individual,
|
||||||
@@ -181,6 +218,7 @@ class Command(LoadDataCommand):
|
|||||||
"""Bulk-create every object buffered so far, grouped by model."""
|
"""Bulk-create every object buffered so far, grouped by model."""
|
||||||
for model, objs in self.pending_objs.items():
|
for model, objs in self.pending_objs.items():
|
||||||
try:
|
try:
|
||||||
|
with _suspend_auto_now(model):
|
||||||
model._default_manager.db_manager(self.using).bulk_create(
|
model._default_manager.db_manager(self.using).bulk_create(
|
||||||
[obj.object for obj in objs],
|
[obj.object for obj in objs],
|
||||||
batch_size=self.batch_size,
|
batch_size=self.batch_size,
|
||||||
|
|||||||
@@ -161,6 +161,101 @@ class CommandTestCase(TestCase):
|
|||||||
ContentType.objects.filter(pk__in=pks).delete()
|
ContentType.objects.filter(pk__in=pks).delete()
|
||||||
tmp_file.unlink(missing_ok=True)
|
tmp_file.unlink(missing_ok=True)
|
||||||
|
|
||||||
|
def test_bulkloaddata_preserves_auto_now_add(self):
|
||||||
|
"""bulk_create() must not overwrite fixture values for auto_now_add fields.
|
||||||
|
|
||||||
|
Covers both an auto_now_add DateTimeField (BarcodeScanResult.timestamp,
|
||||||
|
StockItem.creation_date, StockItemTracking.date) and an auto_now_add
|
||||||
|
DateField (Part.creation_date) - StockItem/StockItemTracking are also
|
||||||
|
the models the bug was originally reported against.
|
||||||
|
"""
|
||||||
|
import datetime
|
||||||
|
|
||||||
|
from django.core import serializers
|
||||||
|
from django.utils import timezone
|
||||||
|
|
||||||
|
from common.models import BarcodeScanResult
|
||||||
|
from part.models import Part
|
||||||
|
from stock.models import StockItem, StockItemTracking
|
||||||
|
from stock.status_codes import StockHistoryCode
|
||||||
|
|
||||||
|
# JSON fixtures only round-trip datetimes to millisecond precision
|
||||||
|
# (DjangoJSONEncoder truncates microseconds) - use a value already at
|
||||||
|
# that precision so the round-trip comparisons below are exact.
|
||||||
|
original_timestamp = (timezone.now() - datetime.timedelta(days=30)).replace(
|
||||||
|
microsecond=123000
|
||||||
|
)
|
||||||
|
original_date = original_timestamp.date()
|
||||||
|
|
||||||
|
entry = BarcodeScanResult.objects.create(data='test-barcode')
|
||||||
|
entry.timestamp = original_timestamp
|
||||||
|
entry.save()
|
||||||
|
entry.refresh_from_db()
|
||||||
|
self.assertEqual(entry.timestamp, original_timestamp)
|
||||||
|
|
||||||
|
part = Part.objects.create(
|
||||||
|
name='Bulkload test part', description='Bulkload test part'
|
||||||
|
)
|
||||||
|
part.creation_date = original_date
|
||||||
|
part.save()
|
||||||
|
part.refresh_from_db()
|
||||||
|
self.assertEqual(part.creation_date, original_date)
|
||||||
|
|
||||||
|
item = StockItem.objects.create(part=part, quantity=10)
|
||||||
|
item.creation_date = original_timestamp
|
||||||
|
item.save()
|
||||||
|
item.refresh_from_db()
|
||||||
|
self.assertEqual(item.creation_date, original_timestamp)
|
||||||
|
|
||||||
|
tracking = StockItemTracking.objects.create(
|
||||||
|
item=item, tracking_type=StockHistoryCode.CREATED
|
||||||
|
)
|
||||||
|
tracking.date = original_timestamp
|
||||||
|
tracking.save()
|
||||||
|
tracking.refresh_from_db()
|
||||||
|
self.assertEqual(tracking.date, original_timestamp)
|
||||||
|
|
||||||
|
pks = {
|
||||||
|
'barcode': entry.pk,
|
||||||
|
'part': part.pk,
|
||||||
|
'item': item.pk,
|
||||||
|
'tracking': tracking.pk,
|
||||||
|
}
|
||||||
|
|
||||||
|
# Serialize parent-before-child, so bulkloaddata's per-model
|
||||||
|
# bulk_create() calls happen in an order that satisfies FK constraints.
|
||||||
|
data = serializers.serialize('json', [entry, part, item, tracking])
|
||||||
|
|
||||||
|
# Use queryset deletes - Part.delete() refuses to delete an active part
|
||||||
|
StockItemTracking.objects.filter(pk=tracking.pk).delete()
|
||||||
|
StockItem.objects.filter(pk=item.pk).delete()
|
||||||
|
Part.objects.filter(pk=part.pk).delete()
|
||||||
|
BarcodeScanResult.objects.filter(pk=entry.pk).delete()
|
||||||
|
|
||||||
|
tmp_file = get_testfolder_dir().joinpath('bulkloaddata_auto_now_test.json')
|
||||||
|
tmp_file.write_text(data, encoding='utf-8')
|
||||||
|
|
||||||
|
try:
|
||||||
|
call_command('bulkloaddata', str(tmp_file), verbosity=0)
|
||||||
|
|
||||||
|
reloaded_entry = BarcodeScanResult.objects.get(pk=pks['barcode'])
|
||||||
|
self.assertEqual(reloaded_entry.timestamp, original_timestamp)
|
||||||
|
|
||||||
|
reloaded_part = Part.objects.get(pk=pks['part'])
|
||||||
|
self.assertEqual(reloaded_part.creation_date, original_date)
|
||||||
|
|
||||||
|
reloaded_item = StockItem.objects.get(pk=pks['item'])
|
||||||
|
self.assertEqual(reloaded_item.creation_date, original_timestamp)
|
||||||
|
|
||||||
|
reloaded_tracking = StockItemTracking.objects.get(pk=pks['tracking'])
|
||||||
|
self.assertEqual(reloaded_tracking.date, original_timestamp)
|
||||||
|
finally:
|
||||||
|
StockItemTracking.objects.filter(pk=pks['tracking']).delete()
|
||||||
|
StockItem.objects.filter(pk=pks['item']).delete()
|
||||||
|
Part.objects.filter(pk=pks['part']).delete()
|
||||||
|
BarcodeScanResult.objects.filter(pk=pks['barcode']).delete()
|
||||||
|
tmp_file.unlink(missing_ok=True)
|
||||||
|
|
||||||
def test_bulkdumpdata_natural_key_caching(self):
|
def test_bulkdumpdata_natural_key_caching(self):
|
||||||
"""Test that bulkdumpdata caches natural-key FK resolution during serialization."""
|
"""Test that bulkdumpdata caches natural-key FK resolution during serialization."""
|
||||||
from django.contrib.admin.models import ADDITION, LogEntry
|
from django.contrib.admin.models import ADDITION, LogEntry
|
||||||
|
|||||||
Reference in New Issue
Block a user