diff --git a/src/backend/InvenTree/importer/migrations/0008_alter_dataimportrow_data_alter_dataimportrow_errors_and_more.py b/src/backend/InvenTree/importer/migrations/0008_alter_dataimportrow_data_alter_dataimportrow_errors_and_more.py new file mode 100644 index 0000000000..3688af1d03 --- /dev/null +++ b/src/backend/InvenTree/importer/migrations/0008_alter_dataimportrow_data_alter_dataimportrow_errors_and_more.py @@ -0,0 +1,88 @@ +# Generated by Django 5.2.16 on 2026-08-08 01:36 + +import django.core.serializers.json +import importer.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("importer", "0007_dataimportsession_completed_row_count_history_and_more"), + ] + + operations = [ + migrations.AlterField( + model_name="dataimportrow", + name="data", + field=models.JSONField( + blank=True, + encoder=django.core.serializers.json.DjangoJSONEncoder, + null=True, + verbose_name="Data", + ), + ), + migrations.AlterField( + model_name="dataimportrow", + name="errors", + field=models.JSONField( + blank=True, + encoder=django.core.serializers.json.DjangoJSONEncoder, + null=True, + verbose_name="Errors", + ), + ), + migrations.AlterField( + model_name="dataimportrow", + name="row_data", + field=models.JSONField( + blank=True, + encoder=django.core.serializers.json.DjangoJSONEncoder, + null=True, + verbose_name="Original row data", + ), + ), + migrations.AlterField( + model_name="dataimportsession", + name="columns", + field=models.JSONField( + blank=True, + encoder=django.core.serializers.json.DjangoJSONEncoder, + null=True, + verbose_name="Columns", + ), + ), + migrations.AlterField( + model_name="dataimportsession", + name="field_defaults", + field=models.JSONField( + blank=True, + encoder=django.core.serializers.json.DjangoJSONEncoder, + null=True, + validators=[importer.validators.validate_field_defaults], + verbose_name="Field Defaults", + ), + ), + migrations.AlterField( + model_name="dataimportsession", + name="field_filters", + field=models.JSONField( + blank=True, + encoder=django.core.serializers.json.DjangoJSONEncoder, + null=True, + validators=[importer.validators.validate_field_defaults], + verbose_name="Field Filters", + ), + ), + migrations.AlterField( + model_name="dataimportsession", + name="field_overrides", + field=models.JSONField( + blank=True, + encoder=django.core.serializers.json.DjangoJSONEncoder, + null=True, + validators=[importer.validators.validate_field_defaults], + verbose_name="Field Overrides", + ), + ), + ] diff --git a/src/backend/InvenTree/importer/models.py b/src/backend/InvenTree/importer/models.py index 9aba90a5da..6c6f1f6f34 100644 --- a/src/backend/InvenTree/importer/models.py +++ b/src/backend/InvenTree/importer/models.py @@ -8,6 +8,7 @@ from typing import Optional from django.contrib.auth.models import User from django.core.exceptions import FieldDoesNotExist from django.core.exceptions import ValidationError as DjangoValidationError +from django.core.serializers.json import DjangoJSONEncoder from django.core.validators import FileExtensionValidator from django.db import models, transaction from django.urls import reverse @@ -82,7 +83,9 @@ class DataImportSession(models.Model): ], ) - columns = models.JSONField(blank=True, null=True, verbose_name=_('Columns')) + columns = models.JSONField( + blank=True, null=True, verbose_name=_('Columns'), encoder=DjangoJSONEncoder + ) model_type = models.CharField( blank=False, @@ -107,6 +110,7 @@ class DataImportSession(models.Model): null=True, verbose_name=_('Field Defaults'), validators=[importer.validators.validate_field_defaults], + encoder=DjangoJSONEncoder, ) field_overrides = models.JSONField( @@ -114,6 +118,7 @@ class DataImportSession(models.Model): null=True, verbose_name=_('Field Overrides'), validators=[importer.validators.validate_field_defaults], + encoder=DjangoJSONEncoder, ) field_filters = models.JSONField( @@ -121,6 +126,7 @@ class DataImportSession(models.Model): null=True, verbose_name=_('Field Filters'), validators=[importer.validators.validate_field_defaults], + encoder=DjangoJSONEncoder, ) update_records = models.BooleanField( @@ -668,12 +674,19 @@ class DataImportRow(models.Model): row_index = models.PositiveIntegerField(default=0, verbose_name=_('Row Index')) row_data = models.JSONField( - blank=True, null=True, verbose_name=_('Original row data') + blank=True, + null=True, + verbose_name=_('Original row data'), + encoder=DjangoJSONEncoder, ) - data = models.JSONField(blank=True, null=True, verbose_name=_('Data')) + data = models.JSONField( + blank=True, null=True, verbose_name=_('Data'), encoder=DjangoJSONEncoder + ) - errors = models.JSONField(blank=True, null=True, verbose_name=_('Errors')) + errors = models.JSONField( + blank=True, null=True, verbose_name=_('Errors'), encoder=DjangoJSONEncoder + ) valid = models.BooleanField(default=False, verbose_name=_('Valid')) @@ -762,7 +775,11 @@ class DataImportRow(models.Model): field, value, lookup_field=field_lookup_mapping.get(field) ) except DjangoValidationError as exc: - extract_errors[field] = exc.message + # exc.message only exists if the error was raised with a single + # message string - lookup_related_field may also raise with a + # dict (message_dict) or list (message), so use exc.messages, + # which normalizes any construction to a flat list of strings. + extract_errors[field] = '; '.join(exc.messages) continue # Use the default value, if provided diff --git a/src/backend/InvenTree/importer/tests.py b/src/backend/InvenTree/importer/tests.py index 8077566989..cfc354b581 100644 --- a/src/backend/InvenTree/importer/tests.py +++ b/src/backend/InvenTree/importer/tests.py @@ -1,5 +1,6 @@ """Unit tests for the 'importer' app.""" +import datetime import os import threading from unittest import mock @@ -81,6 +82,37 @@ class ImporterTest(ImporterMixin, InvenTreeTestCase): # Check that the new companies have been created self.assertEqual(n + 12, Company.objects.count()) + def test_row_data_datetime_serialization(self): + """Test that row data containing native datetime values can be saved. + + Regression test for a Sentry crash: "Object of type datetime is not JSON + serializable when serializing dict item 'COMMENT'". Excel imports (via + tablib/openpyxl) return date-formatted cells as native datetime.datetime + objects rather than strings. These flow unmodified into row_data, which + is then persisted via DataImportRow.objects.bulk_create(). + + DataImportRow.row_data / data / errors (and the related JSONFields on + DataImportSession) previously had no `encoder=DjangoJSONEncoder`, so + Django's JSONField fell back to the plain stdlib json.JSONEncoder, which + cannot serialize datetime objects - raising a TypeError on save/bulk_create. + """ + data_file = self.helper_file('companies.csv') + + session = DataImportSession.objects.create( + data_file=data_file, model_type='company' + ) + + row = DataImportRow( + session=session, + row_index=0, + row_data={'COMMENT': datetime.datetime(2024, 5, 1, 12, 30)}, + ) + + # This must not raise: TypeError: Object of type datetime is not JSON serializable + DataImportRow.objects.bulk_create([row]) + row = DataImportRow.objects.get(session=session, row_index=0) + self.assertEqual(row.row_data['COMMENT'], '2024-05-01T12:30:00') + def test_import_header_whitespace(self): """Test that column headers with leading/trailing whitespace are handled correctly. @@ -338,6 +370,45 @@ class ImporterTest(ImporterMixin, InvenTreeTestCase): result = row.lookup_related_field('part', 'AMBIG-001', lookup_field='name') self.assertNotEqual(result, part_a.pk) + def test_extract_data_related_field_validation_error(self): + """Test that extract_data() handles a dict-constructed ValidationError. + + Regression test: django.core.exceptions.ValidationError only exposes a + `.message` attribute when it was raised with a single message string. + lookup_related_field can also raise with a dict (e.g. "no related model + found for field") or a list, in which case `.message` does not exist and + accessing it raises: AttributeError: 'ValidationError' object has no + attribute 'message'. extract_data() must use `.messages` instead, which + normalizes any construction to a flat list of strings. + """ + from django.core.exceptions import ValidationError as DjangoValidationError + + data_file = self.helper_file('companies.csv') + session = DataImportSession.objects.create( + data_file=data_file, model_type='stockitem' + ) + + row = DataImportRow(session=session, row_data={'Website': 'foo'}) + + field_mapping = {'part': 'Website'} + available_fields = {'part': {'type': 'related field'}} + + with mock.patch.object( + DataImportRow, + 'lookup_related_field', + side_effect=DjangoValidationError({ + 'session': 'No related model found for field: part' + }), + ): + # Must not raise: AttributeError: 'ValidationError' object has no attribute 'message' + row.extract_data( + field_mapping=field_mapping, + available_fields=available_fields, + commit=False, + ) + + self.assertEqual(row.errors, {'part': 'No related model found for field: part'}) + def test_lookup_field_validation(self): """Test that DataImportColumnMap.clean() validates the lookup_field value.""" from django.core.exceptions import ValidationError as DjangoValidationError