Tree query improvements (#3443)

* Allow part category table to be ordered by part count

* Add queryset annotation for part-category part-count

- Uses subquery to annotate the part-count for sub-categories
- Huge reduction in number of queries

* Update 'pathstring' property of PartCategory and StockLocation

- No longer a dynamically calculated value
- Constructed when the model is saved, and then written to the database
- Limited to 250 characters

* Data migration to re-construct pathstring for PartCategory objects

* Fix for tree model save() method

* Add unit tests for pathstring construction

* Data migration for StockLocation pathstring values

* Update part API

- Add new annotation to PartLocationDetail view

* Update API version

* Apply similar annotation to StockLocation API endpoints

* Extra tests for PartCategory API

* Unit test fixes

* Allow PartCategory and StockLocation lists to be sorted by 'pathstring'

* Further unit test fixes

InvenTree/stock/api.py

@@ -224,6 +224,13 @@ class StockLocationList(ListCreateAPI):
     queryset = StockLocation.objects.all()
     serializer_class = StockSerializers.LocationSerializer
 
+    def get_queryset(self, *args, **kwargs):
+        """Return annotated queryset for the StockLocationList endpoint"""
+
+        queryset = super().get_queryset(*args, **kwargs)
+        queryset = StockSerializers.LocationSerializer.annotate_queryset(queryset)
+        return queryset
+
     def filter_queryset(self, queryset):
         """Custom filtering: - Allow filtering by "null" parent to retrieve top-level stock locations."""
         queryset = super().filter_queryset(queryset)
@@ -293,6 +300,7 @@ class StockLocationList(ListCreateAPI):
 
     ordering_fields = [
         'name',
+        'pathstring',
         'items',
         'level',
         'tree_id',
@@ -1340,6 +1348,13 @@ class LocationDetail(RetrieveUpdateDestroyAPI):
     queryset = StockLocation.objects.all()
     serializer_class = StockSerializers.LocationSerializer
 
+    def get_queryset(self, *args, **kwargs):
+        """Return annotated queryset for the StockLocationList endpoint"""
+
+        queryset = super().get_queryset(*args, **kwargs)
+        queryset = StockSerializers.LocationSerializer.annotate_queryset(queryset)
+        return queryset
+
 
 stock_api_urls = [
     re_path(r'^location/', include([

InvenTree/stock/filters.py

@@ -0,0 +1,36 @@
+"""Custom query filters for the Stock models"""
+
+from django.db.models import F, Func, IntegerField, OuterRef, Q, Subquery
+from django.db.models.functions import Coalesce
+
+import stock.models
+
+
+def annotate_location_items(filter: Q = None):
+    """Construct a queryset annotation which returns the number of stock items in a particular location.
+
+    - Includes items in subcategories also
+    - Requires subquery to perform annotation
+    """
+
+    # Construct a subquery to provide all items in this location and any sublocations
+    subquery = stock.models.StockItem.objects.exclude(location=None).filter(
+        location__tree_id=OuterRef('tree_id'),
+        location__lft__gte=OuterRef('lft'),
+        location__rght__lte=OuterRef('rght'),
+        location__level__gte=OuterRef('level'),
+    )
+
+    # Optionally apply extra filter to returned results
+    if filter is not None:
+        subquery = subquery.filter(filter)
+
+    return Coalesce(
+        Subquery(
+            subquery.annotate(
+                total=Func(F('pk'), function='COUNT', output_field=IntegerField())
+            ).values('total')
+        ),
+        0,
+        output_field=IntegerField()
+    )

InvenTree/stock/migrations/0080_stocklocation_pathstring.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.14 on 2022-07-31 23:54
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('stock', '0079_alter_stocklocation_name'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='stocklocation',
+            name='pathstring',
+            field=models.CharField(blank=True, help_text='Path', max_length=250, verbose_name='Path'),
+        ),
+    ]

InvenTree/stock/migrations/0081_auto_20220801_0044.py

@@ -0,0 +1,55 @@
+# Generated by Django 3.2.14 on 2022-08-01 00:44
+
+from django.db import migrations
+
+from InvenTree.helpers import constructPathString
+
+
+def update_pathstring(apps, schema_editor):
+    """Construct pathstring for all existing StockLocation objects"""
+
+    StockLocation = apps.get_model('stock', 'stocklocation')
+
+    n = StockLocation.objects.count()
+
+    if n > 0:
+
+        for loc in StockLocation.objects.all():
+
+            # Construct complete path for category
+            path = [loc.name]
+
+            parent = loc.parent
+
+            # Iterate up the tree
+            while parent is not None:
+                path = [parent.name] + path
+                parent = parent.parent
+
+            pathstring = constructPathString(path)
+
+            loc.pathstring = pathstring
+            loc.save()
+
+        print(f"\n--- Updated 'pathstring' for {n} StockLocation objects ---\n")
+
+
+def nupdate_pathstring(apps, schema_editor):
+    """Empty function for reverse migration compatibility"""
+
+    pass
+
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('stock', '0080_stocklocation_pathstring'),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            update_pathstring,
+            reverse_code=nupdate_pathstring
+        )
+    ]

InvenTree/stock/serializers.py

@@ -18,6 +18,7 @@ import company.models
 import InvenTree.helpers
 import InvenTree.serializers
 import part.models as part_models
+import stock.filters
 from common.settings import currency_code_default, currency_code_mappings
 from company.serializers import SupplierPartSerializer
 from InvenTree.models import extract_int
@@ -575,9 +576,20 @@ class LocationTreeSerializer(InvenTree.serializers.InvenTreeModelSerializer):
 class LocationSerializer(InvenTree.serializers.InvenTreeModelSerializer):
     """Detailed information about a stock location."""
 
+    @staticmethod
+    def annotate_queryset(queryset):
+        """Annotate extra information to the queryset"""
+
+        # Annotate the number of stock items which exist in this category (including subcategories)
+        queryset = queryset.annotate(
+            items=stock.filters.annotate_location_items()
+        )
+
+        return queryset
+
     url = serializers.CharField(source='get_absolute_url', read_only=True)
 
-    items = serializers.IntegerField(source='item_count', read_only=True)
+    items = serializers.IntegerField(read_only=True)
 
     level = serializers.IntegerField(read_only=True)
 

InvenTree/stock/tests.py

@@ -125,6 +125,10 @@ class StockTest(InvenTreeTestCase):
 
     def test_parent_locations(self):
         """Test parent."""
+
+        # Ensure pathstring gets updated
+        self.drawer3.save()
+
         self.assertEqual(self.office.parent, None)
         self.assertEqual(self.drawer1.parent, self.office)
         self.assertEqual(self.drawer2.parent, self.office)