[Enhancement] Import by natural key fields (#11288)

* Data import flexibility

- Allow specification of alternative lookup fields for data import

* Observe field filters during data import

* Add alternative import fields for Part models

* More options for IMPORT_ID_FIELDS

* Update src/backend/InvenTree/importer/models.py

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Bump CHANGELOG

* Handle empty input values

* Add IMPORT_ID_FIELDS for more models

* PK field takes highest priority

* Update import docs

* Tweak return type

* Handle multiple date formats

* Add playwright testing

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

src/backend/InvenTree/build/models.py

@@ -111,6 +111,7 @@ class Build(
     """
 
     STATUS_CLASS = BuildStatus
+    IMPORT_ID_FIELDS = ['reference']
 
     class Meta:
         """Metaclass options for the BuildOrder model."""

src/backend/InvenTree/common/models.py

@@ -148,6 +148,8 @@ class UpdatedUserMixin(models.Model):
 class ProjectCode(InvenTree.models.InvenTreeMetadataModel):
     """A ProjectCode is a unique identifier for a project."""
 
+    IMPORT_ID_FIELDS = ['code']
+
     class Meta:
         """Class options for the ProjectCode model."""
 
@@ -2396,6 +2398,8 @@ class ParameterTemplate(
         enabled: Is this template enabled?
     """
 
+    IMPORT_ID_FIELDS = ['name']
+
     class Meta:
         """Metaclass options for the ParameterTemplate model."""
 

src/backend/InvenTree/company/models.py

@@ -111,6 +111,7 @@ class Company(
     """
 
     IMAGE_RENAME = rename_company_image
+    IMPORT_ID_FIELDS = ['name']
 
     class Meta:
         """Metaclass defines extra model options."""
@@ -297,6 +298,8 @@ class Contact(InvenTree.models.InvenTreeMetadataModel):
         role: position in company
     """
 
+    IMPORT_ID_FIELDS = ['name', 'email']
+
     class Meta:
         """Metaclass defines extra model options."""
 
@@ -494,6 +497,8 @@ class ManufacturerPart(
         description: Descriptive notes field
     """
 
+    IMPORT_ID_FIELDS = ['MPN']
+
     class Meta:
         """Metaclass defines extra model options."""
 
@@ -620,6 +625,8 @@ class SupplierPart(
         updated: Date that the SupplierPart was last updated
     """
 
+    IMPORT_ID_FIELDS = ['SKU']
+
     class Meta:
         """Metaclass defines extra model options."""
 

src/backend/InvenTree/importer/models.py

@@ -2,6 +2,7 @@
 
 import json
 from collections import OrderedDict
+from datetime import datetime
 from typing import Optional
 
 from django.contrib.auth.models import User
@@ -151,6 +152,27 @@ class DataImportSession(models.Model):
 
         return supported_models().get(self.model_type, None)
 
+    def get_related_model(self, field_name: str) -> models.Model:
+        """Return the related model for a given field name.
+
+        Arguments:
+            field_name: The name of the field to check
+
+        Returns:
+            The related model class, if one exists, or None otherwise
+        """
+        model_class = self.model_class
+
+        if not model_class:
+            return None
+
+        try:
+            related_field = model_class._meta.get_field(field_name)
+            model = related_field.remote_field.model
+            return model
+        except (AttributeError, models.FieldDoesNotExist):
+            return None
+
     def extract_columns(self) -> None:
         """Run initial column extraction and mapping.
 
@@ -597,6 +619,8 @@ class DataImportRow(models.Model):
 
         data = {}
 
+        self.related_field_map = {}
+
         # We have mapped column (file) to field (serializer) already
         for field, col in field_mapping.items():
             # Data override (force value and skip any further checks)
@@ -622,7 +646,9 @@ class DataImportRow(models.Model):
             if field_type == 'boolean':
                 value = InvenTree.helpers.str2bool(value)
             elif field_type == 'date':
-                value = value or None
+                value = self.convert_date_field(value)
+            elif field_type == 'related field':
+                value = self.lookup_related_field(field, value)
 
             # Use the default value, if provided
             if value is None and field in default_values:
@@ -667,6 +693,93 @@ class DataImportRow(models.Model):
         if commit:
             self.save()
 
+    def convert_date_field(self, value: str) -> str:
+        """Convert an incoming date field to the correct format for the database."""
+        if value in [None, '']:
+            return None
+
+        # Attempt conversion using accepted formats
+        date_formats = ['%Y-%m-%d', '%d/%m/%Y', '%m/%d/%Y', '%Y/%m/%d']
+
+        for fmt in date_formats:
+            try:
+                dt = datetime.strptime(value.strip(), fmt)
+
+                # If the date is valid, convert it to the standard format and return
+                return dt.strftime('%Y-%m-%d')
+            except ValueError:
+                continue
+
+        # If none of the formats matched, return the original value
+        return value
+
+    def lookup_related_field(self, field_name: str, value: str) -> Optional[int]:
+        """Try to perform lookup against a related field.
+
+        - This is used to convert a human-readable value (e.g. a supplier name) into a database reference (e.g. supplier ID).
+        - Reference the value against the related model's allowable import fields
+
+        Arguments:
+            field_name: The name of the field to perform the lookup against
+            value: The value to be looked up
+
+        Returns:
+            A primary key value
+        """
+        if value is None or value == '':
+            return value
+
+        if field_name is None or field_name == '':
+            return value
+
+        if field_name in self.related_field_map:
+            model = self.related_field_map[field_name]
+        else:
+            # Cache the related model for this field name
+            model = self.related_field_map[field_name] = self.session.get_related_model(
+                field_name
+            )
+
+        if not model:
+            raise DjangoValidationError({
+                'session': f'No related model found for field: {field_name}'
+            })
+
+        valid_items = set()
+
+        base_filters = (
+            self.session.field_filters.get(field_name, {})
+            if self.session.field_filters
+            else {}
+        )
+
+        # First priority is the PK (primary key) field
+        id_fields = ['pk']
+
+        if custom_id_fields := getattr(model, 'IMPORT_ID_FIELDS', None):
+            id_fields += custom_id_fields
+
+        # Iterate through the provided list - if any of the values match, we can perform the lookup
+        for id_field in id_fields:
+            try:
+                queryset = model.objects.filter(**{id_field: value}, **base_filters)
+            except ValueError:
+                continue
+
+            # Evaluate at most two results to determine if there is exactly one match
+            results = list(queryset[:2])
+            if len(results) == 1:
+                # We have a single match against this field
+                valid_items.add(results[0].pk)
+
+        if len(valid_items) == 1:
+            # We found a single valid match against the related model - return this value
+            return valid_items.pop()
+
+        # We found either zero or multiple values matching against the related model
+        # Return the original value and let the serializer validation handle any errors against this field
+        return value
+
     def serializer_data(self):
         """Construct data object to be sent to the serializer.
 

src/backend/InvenTree/order/models.py

@@ -299,6 +299,7 @@ class Order(
 
     REQUIRE_RESPONSIBLE_SETTING = None
     UNLOCK_SETTING = None
+    IMPORT_ID_FIELDS = ['reference']
 
     class Meta:
         """Metaclass options. Abstract ensures no database table is created."""

src/backend/InvenTree/part/models.py

@@ -84,8 +84,8 @@ class PartCategory(
     """
 
     ITEM_PARENT_KEY = 'category'
-
     EXTRA_PATH_FIELDS = ['icon']
+    IMPORT_ID_FIELDS = ['pathstring', 'name']
 
     class Meta:
         """Metaclass defines extra model properties."""
@@ -517,6 +517,7 @@ class Part(
 
     NODE_PARENT_KEY = 'variant_of'
     IMAGE_RENAME = rename_part_image
+    IMPORT_ID_FIELDS = ['IPN', 'name']
 
     objects = TreeManager()
 
@@ -3635,6 +3636,8 @@ class PartTestTemplate(InvenTree.models.InvenTreeMetadataModel):
     run on the model (refer to the validate_unique function).
     """
 
+    IMPORT_ID_FIELDS = ['key']
+
     class Meta:
         """Metaclass options for the PartTestTemplate model."""
 

src/backend/InvenTree/stock/models.py

@@ -65,6 +65,8 @@ class StockLocationType(InvenTree.models.MetadataMixin, models.Model):
         icon: icon class
     """
 
+    IMPORT_ID_FIELDS = ['name']
+
     class Meta:
         """Metaclass defines extra model properties."""
 
@@ -134,8 +136,8 @@ class StockLocation(
     """
 
     ITEM_PARENT_KEY = 'location'
-
     EXTRA_PATH_FIELDS = ['icon']
+    IMPORT_ID_FIELDS = ['pathstring', 'name']
 
     objects = TreeManager()
 
@@ -434,6 +436,7 @@ class StockItem(
         packaging: Description of how the StockItem is packaged (e.g. "reel", "loose", "tape" etc)
     """
 
+    IMPORT_ID_FIELDS = ['serial']
     STATUS_CLASS = StockStatus
 
     class Meta: