diff --git a/docs/docs/assets/images/admin/import_select_id.png b/docs/docs/assets/images/admin/import_select_id.png new file mode 100644 index 0000000000..38d8b797a3 Binary files /dev/null and b/docs/docs/assets/images/admin/import_select_id.png differ diff --git a/docs/docs/assets/images/admin/import_session_create_update.png b/docs/docs/assets/images/admin/import_session_create_update.png new file mode 100644 index 0000000000..31e0aca2d8 Binary files /dev/null and b/docs/docs/assets/images/admin/import_session_create_update.png differ diff --git a/docs/docs/assets/images/admin/import_update_process.png b/docs/docs/assets/images/admin/import_update_process.png new file mode 100644 index 0000000000..93c178bd55 Binary files /dev/null and b/docs/docs/assets/images/admin/import_update_process.png differ diff --git a/docs/docs/settings/import.md b/docs/docs/settings/import.md index 71c3052fe2..c712216217 100644 --- a/docs/docs/settings/import.md +++ b/docs/docs/settings/import.md @@ -76,3 +76,37 @@ Each individual row can be imported, or removed (deleted) by the user. Once all ### Import Completed Once all records have been processed, the import session is considered complete. The import session can be closed, and the imported records are now stored in the database. + +## Updating Existing Records + +The data import process can also be used to update existing records in the database. This requires that the imported data file contains a unique identifier for each record, which can be used to match the records in the database. + +The basic outline of this process is: + +1. Export the existing records to a CSV file. +2. Modify the CSV file to update the records as required. +3. Upload the modified CSV file to the import session. + +!!! note "Mixing Creation and Update" + It is not possible to mix the creation of new records with the updating of existing records in a single import session. If you wish to create new records, you must create a separate import session for that purpose. + +### Create Import Session + +!!! note "Admin Center" + Updating existing records can only be performed when creating a new import session from the [Admin Center](./admin.md#admin-center). + +Create a new import session, and ensure that the *Update Existing Records* option is selected. This will allow the import session to update existing records in the database. + +{{ image("admin/import_session_create_update.png", "Update existing records") }} + +### Map Data Fields + +When mapping the data fields, ensure that the `ID` field is correctly mapped to the corresponding column in the file: + +{{ image("admin/import_select_id.png", "Update existing records") }} + +### Process Data + +When processing the data, each row will be matched against an existing record in the database. If a match is found, the existing record will be updated with the new data from the imported file. + +{{ image("admin/import_update_process.png", "Update existing records") }} diff --git a/src/backend/InvenTree/InvenTree/api_version.py b/src/backend/InvenTree/InvenTree/api_version.py index ba2be6f219..ead7da69fa 100644 --- a/src/backend/InvenTree/InvenTree/api_version.py +++ b/src/backend/InvenTree/InvenTree/api_version.py @@ -1,11 +1,14 @@ """InvenTree API version information.""" # InvenTree API version -INVENTREE_API_VERSION = 386 +INVENTREE_API_VERSION = 387 """Increment this API version number whenever there is a significant change to the API that any clients need to know about.""" INVENTREE_API_TEXT = """ +v387 -> 2025-08-19 : https://github.com/inventree/InvenTree/pull/10188 + - Adds "update_records" field to the DataImportSession API + v386 -> 2025-08-11 : https://github.com/inventree/InvenTree/pull/8191 - Adds "consumed" field to the BuildItem API - Adds API endpoint to consume stock against a BuildOrder diff --git a/src/backend/InvenTree/importer/migrations/0005_dataimportsession_update_records.py b/src/backend/InvenTree/importer/migrations/0005_dataimportsession_update_records.py new file mode 100644 index 0000000000..8e18cb3944 --- /dev/null +++ b/src/backend/InvenTree/importer/migrations/0005_dataimportsession_update_records.py @@ -0,0 +1,22 @@ +# Generated by Django 4.2.23 on 2025-08-18 13:57 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("importer", "0004_alter_dataimportsession_model_type"), + ] + + operations = [ + migrations.AddField( + model_name="dataimportsession", + name="update_records", + field=models.BooleanField( + default=False, + help_text="If enabled, existing records will be updated with new data", + verbose_name="Update Existing Records", + ), + ), + ] diff --git a/src/backend/InvenTree/importer/models.py b/src/backend/InvenTree/importer/models.py index f1c21ca86d..7dd533d269 100644 --- a/src/backend/InvenTree/importer/models.py +++ b/src/backend/InvenTree/importer/models.py @@ -1,6 +1,7 @@ """Model definitions for the 'importer' app.""" import json +from collections import OrderedDict from typing import Optional from django.contrib.auth.models import User @@ -39,6 +40,8 @@ class DataImportSession(models.Model): field_filters: JSONField for field filter values - optional field API filters """ + ID_FIELD_LABEL = 'id' + class ModelChoices(RenderChoices): """Model choices for data import sessions.""" @@ -118,6 +121,12 @@ class DataImportSession(models.Model): validators=[importer.validators.validate_field_defaults], ) + update_records = models.BooleanField( + default=False, + verbose_name=_('Update Existing Records'), + help_text=_('If enabled, existing records will be updated with new data'), + ) + @property def field_mapping(self) -> dict: """Construct a dict of field mappings for this import session. @@ -351,13 +360,25 @@ class DataImportSession(models.Model): metadata = InvenTreeMetadata() + fields = OrderedDict() + + if self.update_records: + # If we are updating records, ensure the ID field is included + fields[self.ID_FIELD_LABEL] = { + 'label': _('ID'), + 'help_text': _('Existing database identifier for the record'), + 'type': 'integer', + 'required': True, + 'read_only': False, + } + if serializer_class := self.serializer_class: serializer = serializer_class(data={}, importing=True) - fields = metadata.get_serializer_info(serializer) - else: - fields = {} + fields.update(metadata.get_serializer_info(serializer)) + # Cache the available fields against this instance self._available_fields = fields + return fields def required_fields(self) -> dict: @@ -370,6 +391,10 @@ class DataImportSession(models.Model): if info.get('required', False): required[field] = info + elif self.update_records and field == self.ID_FIELD_LABEL: + # If we are updating records, the ID field is required + required[field] = info + return required @@ -630,11 +655,13 @@ class DataImportRow(models.Model): return data - def construct_serializer(self, request=None): + def construct_serializer(self, instance=None, request=None): """Construct a serializer object for this row.""" if serializer_class := self.session.serializer_class: return serializer_class( - data=self.serializer_data(), context={'request': request} + instance=instance, + data=self.serializer_data(), + context={'request': request}, ) def validate(self, commit=False, request=None) -> bool: @@ -654,7 +681,26 @@ class DataImportRow(models.Model): # Row has already been completed return True - serializer = self.construct_serializer(request=request) + if self.session.update_records: + # Extract the ID field from the data + instance_id = self.data.get(self.session.ID_FIELD_LABEL, None) + + if not instance_id: + raise DjangoValidationError( + _('ID is required for updating existing records.') + ) + + try: + instance = self.session.model_class.objects.get(pk=instance_id) + except self.session.model_class.DoesNotExist: + raise DjangoValidationError(_('No record found with the provided ID.')) + except ValueError: + raise DjangoValidationError(_('Invalid ID format provided.')) + + serializer = self.construct_serializer(instance=instance, request=request) + + else: + serializer = self.construct_serializer(request=request) if not serializer: self.errors = { diff --git a/src/backend/InvenTree/importer/serializers.py b/src/backend/InvenTree/importer/serializers.py index ad33d6e6a1..2e428b7e9b 100644 --- a/src/backend/InvenTree/importer/serializers.py +++ b/src/backend/InvenTree/importer/serializers.py @@ -41,6 +41,7 @@ class DataImportSessionSerializer(InvenTreeModelSerializer): 'pk', 'timestamp', 'data_file', + 'update_records', 'model_type', 'available_fields', 'status', diff --git a/src/backend/InvenTree/importer/test_data/part_categories.csv b/src/backend/InvenTree/importer/test_data/part_categories.csv new file mode 100644 index 0000000000..836b3a07a9 --- /dev/null +++ b/src/backend/InvenTree/importer/test_data/part_categories.csv @@ -0,0 +1,6 @@ +ID,Name,Description,Default Location,Default keywords,Level,Parent Category,Parts,Subcategories,Path,Starred,Structural,Icon,Parent default location +23,Category 0,"Part category, level 1",,,0,,0,5,Category 0,False,False,, +1,Electronics,Electronic components and systems,,,0,,135,12,Electronics,False,False,, +17,Furniture,Furniture and associated things,,,0,,22,2,Furniture,False,False,, +2,Mechanical,Mechanical components,,,0,,263,3,Mechanical,False,False,, +20,Paint,"Paints, inks, etc",,,0,,5,0,Paint,False,False,, diff --git a/src/backend/InvenTree/importer/tests.py b/src/backend/InvenTree/importer/tests.py index 2393feb1cf..97619c70a3 100644 --- a/src/backend/InvenTree/importer/tests.py +++ b/src/backend/InvenTree/importer/tests.py @@ -3,25 +3,23 @@ import os from django.core.files.base import ContentFile +from django.urls import reverse from importer.models import DataImportRow, DataImportSession -from InvenTree.unit_test import AdminTestCase, InvenTreeTestCase +from InvenTree.unit_test import AdminTestCase, InvenTreeAPITestCase, InvenTreeTestCase class ImporterMixin: """Helpers for import tests.""" - def helper_file(self): + def helper_file(self, fn: str) -> ContentFile: """Return test data.""" - fn = os.path.join(os.path.dirname(__file__), 'test_data', 'companies.csv') + file_path = os.path.join(os.path.dirname(__file__), 'test_data', fn) - with open(fn, encoding='utf-8') as input_file: + with open(file_path, encoding='utf-8') as input_file: data = input_file.read() - return data - def helper_content(self): - """Return content file.""" - return ContentFile(self.helper_file(), 'companies.csv') + return ContentFile(data, fn) class ImporterTest(ImporterMixin, InvenTreeTestCase): @@ -33,8 +31,10 @@ class ImporterTest(ImporterMixin, InvenTreeTestCase): n = Company.objects.count() + data_file = self.helper_file('companies.csv') + session = DataImportSession.objects.create( - data_file=self.helper_content(), model_type='company' + data_file=data_file, model_type='company' ) session.extract_columns() @@ -74,13 +74,116 @@ class ImporterTest(ImporterMixin, InvenTreeTestCase): """Test default field values.""" +class ImportAPITest(ImporterMixin, InvenTreeAPITestCase): + """End-to-end tests for the importer API.""" + + def test_import(self): + """Test full import process via the API.""" + from part.models import PartCategory + + N = PartCategory.objects.count() + + url = reverse('api-importer-session-list') + + # Load data file + data_file = self.helper_file('part_categories.csv') + + data = self.post( + url, + {'model_type': 'partcategory', 'data_file': data_file}, + format='multipart', + ).data + + self.assertFalse(data['update_records']) + self.assertEqual(data['model_type'], 'partcategory') + + # No data has been imported yet + self.assertEqual(data['row_count'], 0) + self.assertEqual(data['completed_row_count'], 0) + + field_names = data['available_fields'].keys() + + for fn in ['name', 'default_location', 'description']: + self.assertIn(fn, field_names) + + self.assertEqual(len(data['columns']), 14) + for col in ['Name', 'Parent Category', 'Path']: + self.assertIn(col, data['columns']) + + session_id = data['pk'] + + # Accept the field mappings + url = reverse('api-import-session-accept-fields', kwargs={'pk': session_id}) + + # Initially the user does not have the right permissions + self.post(url, expected_code=403) + + # Assign correct permission to user + self.assignRole('part_category.add') + + self.post(url, expected_code=200) + + session = self.get( + reverse('api-import-session-detail', kwargs={'pk': session_id}) + ).data + + self.assertEqual(session['row_count'], 5) + self.assertEqual(session['completed_row_count'], 0) + + # Fetch each row, and validate it + rows = self.get( + reverse('api-importer-row-list'), data={'session': session_id} + ).data + + self.assertEqual(len(rows), 5) + + row_ids = [] + + for row in rows: + row_ids.append(row['pk']) + self.assertEqual(row['session'], session_id) + self.assertTrue(row['valid']) + self.assertFalse(row['complete']) + + # Validate the rows + url = reverse('api-import-session-accept-rows', kwargs={'pk': session_id}) + + self.post( + url, + { + 'rows': row_ids[1:] # Validate all but the first row + }, + ) + + # Update session information + session = self.get( + reverse('api-import-session-detail', kwargs={'pk': session_id}) + ).data + + self.assertEqual(session['row_count'], 5) + self.assertEqual(session['completed_row_count'], 4) + + for idx, row in enumerate(row_ids): + detail = self.get( + reverse('api-importer-row-detail', kwargs={'pk': row}) + ).data + + self.assertEqual(detail['session'], session_id) + self.assertEqual(detail['complete'], idx > 0) + + # Check that there are new database records + self.assertEqual(PartCategory.objects.count(), N + 4) + + class AdminTest(ImporterMixin, AdminTestCase): """Tests for the admin interface integration.""" def test_admin(self): """Test the admin URL.""" + data_file = self.helper_file('companies.csv') + session = self.helper( model=DataImportSession, - model_kwargs={'data_file': self.helper_content(), 'model_type': 'company'}, + model_kwargs={'data_file': data_file, 'model_type': 'company'}, ) self.helper(model=DataImportRow, model_kwargs={'session_id': session.id}) diff --git a/src/frontend/playwright.config.ts b/src/frontend/playwright.config.ts index f73c704958..cb2aeaf47d 100644 --- a/src/frontend/playwright.config.ts +++ b/src/frontend/playwright.config.ts @@ -31,7 +31,6 @@ const BASE_URL: string = IS_CI : 'http://localhost:5173'; console.log('Running Playwright Tests:'); -console.log(`- CI Mode: ${IS_CI}`); console.log('- Base URL:', BASE_URL); export default defineConfig({ diff --git a/src/frontend/src/components/importer/ImportDataSelector.tsx b/src/frontend/src/components/importer/ImportDataSelector.tsx index ada7eac1b2..db0edc3fa3 100644 --- a/src/frontend/src/components/importer/ImportDataSelector.tsx +++ b/src/frontend/src/components/importer/ImportDataSelector.tsx @@ -153,6 +153,10 @@ export default function ImporterDataSelector({ }; } + if (field == 'id') { + continue; // Skip the ID field + } + fields[field] = { ...fieldDef, field_type: fieldDef.type, @@ -225,6 +229,10 @@ export default function ImporterDataSelector({ const editCell = useCallback( (row: any, col: any) => { + if (col.field == 'id') { + return; // Cannot edit the ID field + } + setSelectedRow(row); setSelectedFieldNames([col.field]); editRow.open(); diff --git a/src/frontend/src/components/importer/ImporterColumnSelector.tsx b/src/frontend/src/components/importer/ImporterColumnSelector.tsx index 7f3aa487b0..9ad87056d9 100644 --- a/src/frontend/src/components/importer/ImporterColumnSelector.tsx +++ b/src/frontend/src/components/importer/ImporterColumnSelector.tsx @@ -61,6 +61,7 @@ function ImporterColumn({ return (