Update data migration file (#4596)

* Update data migration file

- Do not call "save()" on the ORM model as it has some other hooks
- Manually calculate checksum to determine which line items are "valid"
- Update BomItem checksum calculation

* Update BomItem hashing function

- Ensure it remains the same after saving
- Must use normalize(quantity) otherwise weird issues

InvenTree/part/migrations/0102_auto_20230314_0112.py

@@ -1,21 +1,94 @@
 # Generated by Django 3.2.18 on 2023-03-14 01:12
 
+import hashlib
 import logging
 
 from django.db import migrations
+from jinja2 import Template
+
+from InvenTree.helpers import normalize
 
 
 logger = logging.getLogger('inventree')
 
 
 def update_bom_item(apps, schema_editor):
-    """Update all existing BomItem instances"""
+    """Update all existing BomItem instances, and cache the 'validated' field.
 
-    from part.models import BomItem
+    The 'validated' field denotes whether this individual BomItem has been validated,
+    which previously was calculated on the fly (which was very expensive).
+    """
 
-    if n := BomItem.objects.count():
+    BomItem = apps.get_model('part', 'bomitem')
+    InvenTreeSetting = apps.get_model('common', 'inventreesetting')
+
+    n = BomItem.objects.count()
+
+    if n > 0:
 
         for item in BomItem.objects.all():
+            """For each item, we need to re-calculate the "checksum", based on the *old* routine.
+            Note that as we cannot access the ORM models, we have to do this "by hand"
+            """
+
+            # Construct the 'full_name' for the sub_part (this is no longer required, but *was* required at point of migration)
+            try:
+                setting = InvenTreeSetting.objects.get(key='PART_NAME_FORMAT')
+                full_name_pattern = str(setting.value)
+            except Exception:
+                full_name_pattern = "{{ part.IPN if part.IPN }}{{ ' | ' if part.IPN }}{{ part.name }}{{ ' | ' if part.revision }}{{ part.revision if part.revision }}"
+
+            template = Template(full_name_pattern)
+
+            full_name = template.render({'part': item.sub_part})
+
+            # Calculate the OLD checksum manually for this BomItem
+            old_hash = hashlib.md5(str(item.pk).encode())
+            old_hash.update(str(item.sub_part.pk).encode())
+            old_hash.update(str(full_name).encode())
+            old_hash.update(str(item.quantity).encode())
+            old_hash.update(str(item.note).encode())
+            old_hash.update(str(item.reference).encode())
+            old_hash.update(str(item.optional).encode())
+            old_hash.update(str(item.inherited).encode())
+
+            if item.consumable:
+                old_hash.update(str(item.consumable).encode())
+
+            if item.allow_variants:
+                old_hash.update(str(item.allow_variants).encode())
+
+            checksum = str(old_hash.digest())
+
+            # Now, update the 'validated' field based on whether the checksum is 'valid' or not
+            item.validated = item.checksum == checksum
+
+            """Next, we need to update the item with a "new" hash, with the following differences:
+            - Uses the PK of the 'part', not the BomItem itself,
+            - Does not use the 'full_name' of the linked 'sub_part'
+            - Does not use the 'note' field
+            """
+
+            if item.validated:
+
+                new_hash = hashlib.md5(''.encode())
+
+                components = [
+                    item.part.pk,
+                    item.sub_part.pk,
+                    normalize(item.quantity),
+                    item.reference,
+                    item.optional,
+                    item.inherited,
+                    item.consumable,
+                    item.allow_variants
+                ]
+
+                for component in components:
+                    new_hash.update(str(component).encode())
+
+                item.checksum = str(new_hash.digest())
+
             item.save()
 
         logger.info(f"Updated 'validated' flag for {n} BomItem objects")

InvenTree/part/models.py

@@ -3631,32 +3631,32 @@ class BomItem(DataImportMixin, MetadataMixin, models.Model):
         """Calculate the checksum hash of this BOM line item.
 
         The hash is calculated from the following fields:
-        - Part.full_name (if the part name changes, the BOM checksum is invalidated)
+        - part.pk
-        - Quantity
+        - sub_part.pk
-        - Reference field
+        - quantity
-        - Note field
+        - reference
-        - Optional field
+        - optional
-        - Inherited field
+        - inherited
+        - consumable
+        - allow_variants
         """
         # Seed the hash with the ID of this BOM item
-        result_hash = hashlib.md5(str(self.id).encode())
+        result_hash = hashlib.md5(''.encode())
 
-        # Update the hash based on line information
+        # The following components are used to calculate the checksum
-        result_hash.update(str(self.sub_part.id).encode())
+        components = [
-        result_hash.update(str(self.sub_part.full_name).encode())
+            self.part.pk,
-        result_hash.update(str(self.quantity).encode())
+            self.sub_part.pk,
-        result_hash.update(str(self.note).encode())
+            normalize(self.quantity),
-        result_hash.update(str(self.reference).encode())
+            self.reference,
-        result_hash.update(str(self.optional).encode())
+            self.optional,
-        result_hash.update(str(self.inherited).encode())
+            self.inherited,
+            self.consumable,
+            self.allow_variants
+        ]
 
-        # Optionally encoded for backwards compatibility
+        for component in components:
-        if self.consumable:
+            result_hash.update(str(component).encode())
-            result_hash.update(str(self.consumable).encode())
-
-        # Optionally encoded for backwards compatibility
-        if self.allow_variants:
-            result_hash.update(str(self.allow_variants).encode())
 
         return str(result_hash.digest())
 
@@ -3667,7 +3667,7 @@ class BomItem(DataImportMixin, MetadataMixin, models.Model):
             valid: If true, validate the hash, otherwise invalidate it (default = True)
         """
         if valid:
-            self.checksum = str(self.get_item_hash())
+            self.checksum = self.get_item_hash()
         else:
             self.checksum = ''
 

InvenTree/part/test_migrations.py

@@ -46,3 +46,39 @@ class TestForwardMigrations(MigratorTestCase):
         for name in ['A', 'C', 'E']:
             part = Part.objects.get(name=name)
             self.assertEqual(part.description, f"My part {name}")
+
+
+class TestBomItemMigrations(MigratorTestCase):
+    """Tests for BomItem migrations"""
+
+    migrate_from = ('part', '0002_auto_20190520_2204')
+    migrate_to = ('part', helpers.getNewestMigrationFile('part'))
+
+    def prepare(self):
+        """Create intial dataset"""
+
+        Part = self.old_state.apps.get_model('part', 'part')
+        BomItem = self.old_state.apps.get_model('part', 'bomitem')
+
+        a = Part.objects.create(name='Part A', description='My part A')
+        b = Part.objects.create(name='Part B', description='My part B')
+        c = Part.objects.create(name='Part C', description='My part C')
+
+        BomItem.objects.create(part=a, sub_part=b, quantity=1)
+        BomItem.objects.create(part=a, sub_part=c, quantity=1)
+
+        self.assertEqual(BomItem.objects.count(), 2)
+
+        # Initially we don't have the 'validated' field
+        with self.assertRaises(AttributeError):
+            print(b.validated)
+
+    def test_validated_field(self):
+        """Test that the 'validated' field is added to the BomItem objects"""
+
+        BomItem = self.new_state.apps.get_model('part', 'bomitem')
+
+        self.assertEqual(BomItem.objects.count(), 2)
+
+        for bom_item in BomItem.objects.all():
+            self.assertFalse(bom_item.validated)