Import update (#10188)

* Add field to "update" existing records

* Ensure the ID is first

* Prevent editing of "ID" field

* Extract db instance

* Bump API version

* Prevent edit of "id" field

* Refactoring

* Enhanced playwright tests for data importing

* Update docs

* Update src/backend/InvenTree/importer/models.py

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update src/frontend/src/forms/ImporterForms.tsx

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Fix silly AI mistake

* Fix for table pagination

- Ensure page does not exceed available records

* Bug fix for playwright test

* Add end-to-end API testing

* Fix unit tests

* Adjust table page logic

* Ensure sensible page size

* Simplify playwright test

* Simplify test again

* Tweak unit test

- Importing has invalidated the BOM?

* Adjust playwright tests

* Further playwright fixes

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

src/frontend/playwright.config.ts

@@ -31,7 +31,6 @@ const BASE_URL: string = IS_CI
   : 'http://localhost:5173';
 
 console.log('Running Playwright Tests:');
-console.log(`- CI Mode: ${IS_CI}`);
 console.log('- Base URL:', BASE_URL);
 
 export default defineConfig({

src/frontend/src/components/importer/ImportDataSelector.tsx

@@ -153,6 +153,10 @@ export default function ImporterDataSelector({
           };
         }
 
+        if (field == 'id') {
+          continue; // Skip the ID field
+        }
+
         fields[field] = {
           ...fieldDef,
           field_type: fieldDef.type,
@@ -225,6 +229,10 @@ export default function ImporterDataSelector({
 
   const editCell = useCallback(
     (row: any, col: any) => {
+      if (col.field == 'id') {
+        return; // Cannot edit the ID field
+      }
+
       setSelectedRow(row);
       setSelectedFieldNames([col.field]);
       editRow.open();

src/frontend/src/components/importer/ImporterColumnSelector.tsx

@@ -61,6 +61,7 @@ function ImporterColumn({
 
   return (
     <Select
+      aria-label={`import-column-map-${column.field}`}
       error={errorMessage}
       clearable
       searchable

src/frontend/src/forms/ImporterForms.tsx

@@ -1,9 +1,23 @@
+import type { ModelType } from '@lib/enums/ModelType';
 import type { ApiFormFieldSet } from '@lib/types/Forms';
 
-export function dataImporterSessionFields(): ApiFormFieldSet {
+export function dataImporterSessionFields({
+  modelType,
+  allowUpdate = false
+}: {
+  modelType?: ModelType | string;
+  allowUpdate?: boolean;
+}): ApiFormFieldSet {
   return {
     data_file: {},
-    model_type: {},
+    model_type: {
+      value: modelType,
+      hidden: modelType != undefined
+    },
+    update_records: {
+      hidden: allowUpdate !== true,
+      value: allowUpdate ? undefined : false
+    },
     field_defaults: {
       hidden: true,
       value: {}

src/frontend/src/tables/InvenTreeTable.tsx

@@ -340,7 +340,28 @@ export function InvenTreeTable<T extends Record<string, any>>({
   // Reset the pagination state when the search term changes
   useEffect(() => {
     tableState.setPage(1);
-  }, [tableState.searchTerm]);
+  }, [
+    tableState.searchTerm,
+    tableState.filterSet.activeFilters,
+    tableState.queryFilters
+  ]);
+
+  // Account for invalid page offsets
+  useEffect(() => {
+    if (
+      tableState.page > 1 &&
+      pageSize * tableState.page > tableState.recordCount
+    ) {
+      tableState.setPage(1);
+    } else if (tableState.page < 1) {
+      tableState.setPage(1);
+    }
+
+    if (pageSize < 10) {
+      // Default page size
+      setPageSize(25);
+    }
+  }, [tableState.records, tableState.page, pageSize]);
 
   // Data Sorting
   const [sortStatus, setSortStatus] = useState<DataTableSortStatus<T>>({
@@ -705,7 +726,7 @@ export function InvenTreeTable<T extends Record<string, any>>({
         ..._params,
         totalRecords: tableState.recordCount,
         recordsPerPage: tablePageSize,
-        page: tableState.page,
+        page: Math.max(1, tableState.page),
         onPageChange: tableState.setPage,
         recordsPerPageOptions: PAGE_SIZES,
         onRecordsPerPageChange: updatePageSize

src/frontend/src/tables/bom/BomTable.tsx

@@ -451,10 +451,9 @@ export function BomTable({
   const [selectedBomItem, setSelectedBomItem] = useState<any>({});
 
   const importSessionFields = useMemo(() => {
-    const fields = dataImporterSessionFields();
-
-    fields.model_type.hidden = true;
-    fields.model_type.value = 'bomitem';
+    const fields = dataImporterSessionFields({
+      modelType: 'bomitem'
+    });
 
     fields.field_overrides.value = {
       part: partId

src/frontend/src/tables/purchasing/PurchaseOrderLineItemTable.tsx

@@ -79,10 +79,9 @@ export function PurchaseOrderLineItemTable({
   );
 
   const importSessionFields = useMemo(() => {
-    const fields = dataImporterSessionFields();
-
-    fields.model_type.hidden = true;
-    fields.model_type.value = ModelType.purchaseorderlineitem;
+    const fields = dataImporterSessionFields({
+      modelType: ModelType.purchaseorderlineitem
+    });
 
     // Specify override values for import
     fields.field_overrides.value = {

src/frontend/src/tables/settings/ImportSessionTable.tsx

@@ -42,7 +42,9 @@ export default function ImportSessionTable() {
   const newImportSession = useCreateApiFormModal({
     url: ApiEndpoints.import_session_list,
     title: t`Create Import Session`,
-    fields: dataImporterSessionFields(),
+    fields: dataImporterSessionFields({
+      allowUpdate: true
+    }),
     onFormSuccess: (response: any) => {
       setSelectedSession(response.pk);
       setOpened(true);

src/frontend/tests/fixtures/bom_data.csv

@@ -1,5 +1,4 @@
-Assembly,Component,Reference,Quantity,Overage,Allow Variants,Gets inherited,Optional,Consumable,Note,ID,Pricing min,Pricing max,Pricing min total,Pricing max total,Pricing updated,Component.Ipn,Component.Name,Component.Description,Validated,Available Stock,Available substitute stock,Available variant stock,External stock,On Order,In Production,Can Build
-87,66,Screws,4.0,,False,False,False,False,,16,0.28,0.648622,1.12,2.594488,2024-08-08 06:55,,M3x8 Torx,"Torx head screw, M3 thread, 8.0mm",True,485.0,0.0,0.0,0.0,0.0,0.0,121.25
-87,67,Large screw,1.0,,False,False,False,False,,17,0.574802,0.574802,0.574802,0.574802,2024-07-27 05:13,,M3x10 Torx,"Torx head screw, M3 thread, 10.0mm",True,1450.0,0.0,0.0,0.0,0.0,0.0,1450.0
-87,82,Enclosure,1.0,,False,False,False,False,,15,,,,,2024-07-27 05:08,,1551ABK,"Small plastic enclosure, black",True,165.0,223.0,0.0,0.0,0.0,0.0,388.0
-87,88,PCBA,1.0,,True,False,False,False,Assembled board,23,80.431083,129.328176,80.431083,129.328176,2024-12-27 23:14,002.01-PCBA,Widget Board (assembled),Assembled PCB for converting electricity into magic smoke,True,55.0,0.0,0.0,0.0,0.0,0.0,55.0
+Assembly,Component,Reference,Quantity,Allow Variants,Gets inherited,Optional,Consumable,Setup quantity,Attrition,Rounding multiple,Note,ID,Pricing min,Pricing max,Pricing min total,Pricing max total,Pricing updated,Component.Ipn,Component.Name,Component.Description,Validated,Available Stock,Available substitute stock,Available variant stock,External stock,On Order,In Production,Can Build
+106,98,screws,5,FALSE,TRUE,FALSE,TRUE,0,0,0,,39,0.075,0.1,0.375,0.5,23/07/2025 9:12,,Wood Screw,Screw for fixing wood to other wood,TRUE,1604,0,0,0,0,0,320.8
+106,95,legs,4,FALSE,TRUE,FALSE,FALSE,0,0,0,,40,10.6,12.75,42.4,51,23/07/2025 9:12,,Leg,Leg for a chair or a table,TRUE,317,0,0,0,0,0,79.25
+109,92,paint,0.125,FALSE,FALSE,FALSE,FALSE,0,0,0,,43,1.403886,14.389836,0.175486,1.79873,23/07/2025 9:12,,Green Paint,Green Paint,TRUE,110.125,0,0,0,0,0,881

src/frontend/tests/pages/pui_part.spec.ts

@@ -92,8 +92,6 @@ test('Parts - BOM', async ({ browser }) => {
   await setTableChoiceFilter(page, 'active', 'Yes');
   await setTableChoiceFilter(page, 'BOM Valid', 'Yes');
 
-  await page.getByText('1 - 12 / 12').waitFor();
-
   // Navigate to BOM for a particular assembly
   await navigate(page, 'part/87/bom');
   await loadTab(page, 'Bill of Materials');
@@ -620,8 +618,11 @@ test('Parts - Bulk Edit', async ({ browser }) => {
   await page.getByLabel('Select record 2', { exact: true }).click();
   await page.getByLabel('action-menu-part-actions').click();
   await page.getByLabel('action-menu-part-actions-set-category').click();
+
   await page.getByLabel('related-field-category').fill('rnitu');
-  await page.getByRole('option', { name: '- Furniture/Chairs' }).click;
+  await page.waitForTimeout(250);
+
+  await page.getByRole('option', { name: '- Furniture/Chairs' }).click();
   await page.getByRole('button', { name: 'Update' }).click();
   await page.getByText('Items Updated').waitFor();
 });

src/frontend/tests/pui_importing.spec.ts

@@ -14,6 +14,14 @@ test('Importing - Admin Center', async ({ browser }) => {
 
   const fileInput = await page.locator('input[type="file"]');
   await fileInput.setInputFiles('./tests/fixtures/bom_data.csv');
+
+  await page
+    .locator('label')
+    .filter({ hasText: 'Update Existing RecordsIf' })
+    .locator('div')
+    .first()
+    .click();
+
   await page.getByRole('button', { name: 'Submit' }).click();
 
   // Submitting without selecting model type, should show error
@@ -22,21 +30,54 @@ test('Importing - Admin Center', async ({ browser }) => {
 
   await page
     .getByRole('textbox', { name: 'choice-field-model_type' })
-    .fill('Cat');
-  await page
-    .getByRole('option', { name: 'Part Category', exact: true })
-    .click();
+    .fill('bom');
+  await page.getByRole('option', { name: 'BOM Item', exact: true }).click();
   await page.getByRole('button', { name: 'Submit' }).click();
 
-  await page.getByText('Description (optional)').waitFor();
-  await page.getByText('Parent Category').waitFor();
+  await page.getByText('Select the parent assembly').waitFor();
+  await page.getByText('Select the component part').waitFor();
+  await page.getByText('Existing database identifier for the record').waitFor();
+
+  await page
+    .getByRole('textbox', { name: 'import-column-map-reference' })
+    .click();
+  await page.getByRole('option', { name: 'Ignore this field' }).click();
+
+  await page.getByRole('button', { name: 'Accept Column Mapping' }).click();
+
+  // Check for expected ID values
+  for (const itemId of ['16', '17', '15', '23']) {
+    await page.getByRole('cell', { name: itemId, exact: true });
+  }
+
+  // Import all the records
+  await page
+    .getByRole('row', { name: 'Select all records Row Not' })
+    .getByLabel('Select all records')
+    .click();
+  await page
+    .getByRole('button', { name: 'action-button-import-selected' })
+    .click();
+
+  await page.getByText('Data has been imported successfully').waitFor();
+  await page.getByRole('button', { name: 'Close' }).click();
+
+  // Confirmation of full import success
+  await page.getByRole('cell', { name: '3 / 3' }).first().waitFor();
+
+  // Manually delete records
+  await page.getByRole('checkbox', { name: 'Select all records' }).click();
+  await page
+    .getByRole('button', { name: 'action-button-delete-selected' })
+    .click();
+  await page.getByRole('button', { name: 'Delete', exact: true }).click();
 });
 
 test('Importing - BOM', async ({ browser }) => {
   const page = await doCachedLogin(browser, {
     username: 'steven',
     password: 'wizardstaff',
-    url: 'part/87/bom'
+    url: 'part/109/bom'
   });
 
   await page
@@ -53,10 +94,10 @@ test('Importing - BOM', async ({ browser }) => {
   await page.waitForTimeout(500);
 
   await page.getByText('Importing Data').waitFor();
-  await page.getByText('0 / 4').waitFor();
+  await page.getByText('0 / 3').waitFor();
 
-  await page.getByText('Torx head screw, M3 thread, 10.0mm').first().waitFor();
-  await page.getByText('Small plastic enclosure, black').first().waitFor();
+  await page.getByText('Screw for fixing wood').first().waitFor();
+  await page.getByText('Leg for a chair or a table').first().waitFor();
 
   // Select some rows
   await page
@@ -90,15 +131,16 @@ test('Importing - BOM', async ({ browser }) => {
   await page.getByRole('button', { name: 'Submit' }).click();
   await page.waitForTimeout(250);
 
-  await page.getByText('0 / 2', { exact: true }).waitFor();
+  await page.getByText('0 / 1', { exact: true }).waitFor();
 
   // Submit a row
   await page
     .getByRole('row', { name: 'Select record 1 2 Thumbnail' })
     .getByLabel('row-action-menu-')
     .click();
+
   await page.getByRole('menuitem', { name: 'Accept' }).click();
-  await page.getByText('1 / 2', { exact: true }).waitFor();
+  await page.getByText('0 / 1', { exact: true }).waitFor();
 });
 
 test('Importing - Purchase Order', async ({ browser }) => {