diff --git a/src/backend/InvenTree/importer/operations.py b/src/backend/InvenTree/importer/operations.py index 7334f2badf..fec9cc1567 100644 --- a/src/backend/InvenTree/importer/operations.py +++ b/src/backend/InvenTree/importer/operations.py @@ -44,7 +44,15 @@ def load_data_file(data_file, file_format=None): # Excel formats expect binary data if file_format not in ['xls', 'xlsx']: - data = data.decode() + # Attempt to decode the file data as UTF-8 first, falling back to + # cp1252 (common for CSV files exported from Excel on Windows) + try: + data = data.decode('utf-8-sig') + except UnicodeDecodeError: + try: + data = data.decode('cp1252') + except UnicodeDecodeError: + raise ValidationError(_('Could not decode data file as text')) try: data = tablib.Dataset().load(data, headers=True, format=file_format) diff --git a/src/backend/InvenTree/importer/tests.py b/src/backend/InvenTree/importer/tests.py index 6472a30d58..8077566989 100644 --- a/src/backend/InvenTree/importer/tests.py +++ b/src/backend/InvenTree/importer/tests.py @@ -155,6 +155,46 @@ class ImporterTest(ImporterMixin, InvenTreeTestCase): # Check that the new companies have been created self.assertEqual(n + 12, Company.objects.count()) + def test_import_encoding(self): + """Test that non-UTF-8 encoded data files are handled correctly. + + Regression test: CSV files exported from Excel on Windows are commonly + encoded as cp1252 rather than utf-8. Previously, an unhandled + UnicodeDecodeError was raised (surfaced to the user as a server error) + instead of a well-formed ValidationError. + """ + from django.core.exceptions import ValidationError as DjangoValidationError + + raw = self.helper_file('companies.csv').read() + + # Encode the file data as cp1252, including a character not valid in utf-8 + lines = raw.splitlines() + lines[1] = lines[1].replace('Arrow Electronics', 'Arrow Electronics \xb2') + cp1252_data = '\n'.join(lines).encode('cp1252') + + data_file = ContentFile(cp1252_data, 'companies_cp1252.csv') + + session = DataImportSession.objects.create( + data_file=data_file, model_type='company' + ) + + session.extract_columns() + session.import_data() + + arrow_row = next( + row for row in session.rows.all() if row.data.get('name') == 'Arrow' + ) + self.assertEqual(arrow_row.data.get('description'), 'Arrow Electronics ²') + + # A file containing bytes which are not valid in either utf-8 or cp1252 + # should raise a ValidationError, rather than an unhandled exception + invalid_data = ContentFile(b'name,website\r\n\x81\x8d,test\r\n', 'invalid.csv') + + with self.assertRaises(DjangoValidationError): + DataImportSession.objects.create( + data_file=invalid_data, model_type='company' + ).extract_columns() + def test_field_defaults(self): """Test default field values.