Queryset annotation refactor (#3117)

* Refactor out 'ordering' serializer annotation field

* Refactor BomItem serializer annotations

* Factor out MPTT OuterRef query

* Add 'available_stock' annotation to SalesOrderLineItem serializer

- Allows for better rendering of stock availability in sales order table

* Improve 'available quantity' rendering of salesorderlineitem table

* Bump API version

* Add docstring

InvenTree/InvenTree/api_version.py

@@ -2,12 +2,16 @@
 
 
 # InvenTree API version
-INVENTREE_API_VERSION = 53
+INVENTREE_API_VERSION = 54
 
 """
 Increment this API version number whenever there is a significant change to the API that any clients need to know about
 
-v52 -> 2022-06-01 : https://github.com/inventree/InvenTree/pull/3110
+v54 -> 2022-06-02 : https://github.com/inventree/InvenTree/pull/3117
+    - Adds 'available_stock' annotation on the SalesOrderLineItem API
+    - Adds (well, fixes) 'overdue' annotation on the SalesOrderLineItem API
+
+v53 -> 2022-06-01 : https://github.com/inventree/InvenTree/pull/3110
     - Adds extra search fields to the BuildOrder list API endpoint
 
 v52 -> 2022-05-31 : https://github.com/inventree/InvenTree/pull/3103

InvenTree/order/api.py

@@ -788,6 +788,8 @@ class SalesOrderLineItemList(generics.ListCreateAPIView):
             'order__stock_items',
         )
 
+        queryset = serializers.SalesOrderLineItemSerializer.annotate_queryset(queryset)
+
         return queryset
 
     filter_backends = [
@@ -835,6 +837,14 @@ class SalesOrderLineItemDetail(generics.RetrieveUpdateDestroyAPIView):
     queryset = models.SalesOrderLineItem.objects.all()
     serializer_class = serializers.SalesOrderLineItemSerializer
 
+    def get_queryset(self, *args, **kwargs):
+        """Return annotated queryset for this endpoint"""
+        queryset = super().get_queryset(*args, **kwargs)
+
+        queryset = serializers.SalesOrderLineItemSerializer.annotate_queryset(queryset)
+
+        return queryset
+
 
 class SalesOrderContextMixin:
     """Mixin to add sales order object as serializer context variable."""

InvenTree/order/models.py

@@ -887,14 +887,6 @@ class OrderLineItem(models.Model):
         target_date: An (optional) date for expected shipment of this line item.
     """
 
-    """
-    Query filter for determining if an individual line item is "overdue":
-    - Amount received is less than the required quantity
-    - Target date is not None
-    - Target date is in the past
-    """
-    OVERDUE_FILTER = Q(received__lt=F('quantity')) & ~Q(target_date=None) & Q(target_date__lt=datetime.now().date())
-
     class Meta:
         """Metaclass options. Abstract ensures no database table is created."""
 
@@ -953,6 +945,9 @@ class PurchaseOrderLineItem(OrderLineItem):
         order: Reference to a PurchaseOrder object
     """
 
+    # Filter for determining if a particular PurchaseOrderLineItem is overdue
+    OVERDUE_FILTER = Q(received__lt=F('quantity')) & ~Q(target_date=None) & Q(target_date__lt=datetime.now().date())
+
     @staticmethod
     def get_api_url():
         """Return the API URL associated with the PurchaseOrderLineItem model"""
@@ -1076,6 +1071,9 @@ class SalesOrderLineItem(OrderLineItem):
         shipped: The number of items which have actually shipped against this line item
     """
 
+    # Filter for determining if a particular SalesOrderLineItem is overdue
+    OVERDUE_FILTER = Q(shipped__lt=F('quantity')) & ~Q(target_date=None) & Q(target_date__lt=datetime.now().date())
+
     @staticmethod
     def get_api_url():
         """Return the API URL associated with the SalesOrderLineItem model"""

InvenTree/order/serializers.py

@@ -14,6 +14,7 @@ from rest_framework.serializers import ValidationError
 from sql_util.utils import SubqueryCount
 
 import order.models
+import part.filters
 import stock.models
 import stock.serializers
 from common.settings import currency_code_mappings
@@ -248,7 +249,7 @@ class PurchaseOrderLineItemSerializer(InvenTreeModelSerializer):
         queryset = queryset.annotate(
             overdue=Case(
                 When(
-                    Q(order__status__in=PurchaseOrderStatus.OPEN) & order.models.OrderLineItem.OVERDUE_FILTER, then=Value(True, output_field=BooleanField())
+                    Q(order__status__in=PurchaseOrderStatus.OPEN) & order.models.PurchaseOrderLineItem.OVERDUE_FILTER, then=Value(True, output_field=BooleanField())
                 ),
                 default=Value(False, output_field=BooleanField()),
             )
@@ -790,17 +791,36 @@ class SalesOrderLineItemSerializer(InvenTreeModelSerializer):
     def annotate_queryset(queryset):
         """Add some extra annotations to this queryset:
 
-        - "Overdue" status (boolean field)
+        - "overdue" status (boolean field)
+        - "available_quantity"
         """
+
         queryset = queryset.annotate(
             overdue=Case(
                 When(
-                    Q(order__status__in=SalesOrderStatus.OPEN) & order.models.OrderLineItem.OVERDUE_FILTER, then=Value(True, output_field=BooleanField()),
+                    Q(order__status__in=SalesOrderStatus.OPEN) & order.models.SalesOrderLineItem.OVERDUE_FILTER, then=Value(True, output_field=BooleanField()),
                 ),
                 default=Value(False, output_field=BooleanField()),
             )
         )
 
+        # Annotate each line with the available stock quantity
+        # To do this, we need to look at the total stock and any allocations
+        queryset = queryset.alias(
+            total_stock=part.filters.annotate_total_stock(reference='part__'),
+            allocated_to_sales_orders=part.filters.annotate_sales_order_allocations(reference='part__'),
+            allocated_to_build_orders=part.filters.annotate_build_order_allocations(reference='part__'),
+        )
+
+        queryset = queryset.annotate(
+            available_stock=ExpressionWrapper(
+                F('total_stock') - F('allocated_to_sales_orders') - F('allocated_to_build_orders'),
+                output_field=models.DecimalField()
+            )
+        )
+
+        return queryset
+
     def __init__(self, *args, **kwargs):
         """Initializion routine for the serializer:
 
@@ -825,7 +845,9 @@ class SalesOrderLineItemSerializer(InvenTreeModelSerializer):
     part_detail = PartBriefSerializer(source='part', many=False, read_only=True)
     allocations = SalesOrderAllocationSerializer(many=True, read_only=True, location_detail=True)
 
+    # Annotated fields
     overdue = serializers.BooleanField(required=False, read_only=True)
+    available_stock = serializers.FloatField(read_only=True)
 
     quantity = InvenTreeDecimalField()
 
@@ -853,6 +875,7 @@ class SalesOrderLineItemSerializer(InvenTreeModelSerializer):
             'pk',
             'allocated',
             'allocations',
+            'available_stock',
             'quantity',
             'reference',
             'notes',

InvenTree/part/filters.py

@@ -0,0 +1,141 @@
+"""Custom query filters for the Part model
+
+The code here makes heavy use of subquery annotations!
+
+Useful References:
+
+- https://hansonkd.medium.com/the-dramatic-benefits-of-django-subqueries-and-annotations-4195e0dafb16
+- https://pypi.org/project/django-sql-utils/
+- https://docs.djangoproject.com/en/4.0/ref/models/expressions/
+- https://stackoverflow.com/questions/42543978/django-1-11-annotating-a-subquery-aggregate
+
+Relevant PRs:
+
+- https://github.com/inventree/InvenTree/pull/2797/
+- https://github.com/inventree/InvenTree/pull/2827
+
+"""
+
+from decimal import Decimal
+
+from django.db import models
+from django.db.models import OuterRef, Q
+from django.db.models.functions import Coalesce
+
+from sql_util.utils import SubquerySum
+
+import stock.models
+from InvenTree.status_codes import (BuildStatus, PurchaseOrderStatus,
+                                    SalesOrderStatus)
+
+
+def annotate_on_order_quantity(reference: str = ''):
+    """Annotate the 'on order' quantity for each part in a queryset"""
+
+    # Filter only 'active' purhase orders
+    order_filter = Q(order__status__in=PurchaseOrderStatus.OPEN)
+
+    return Coalesce(
+        SubquerySum(f'{reference}supplier_parts__purchase_order_line_items__quantity', filter=order_filter),
+        Decimal(0),
+        output_field=models.DecimalField()
+    ) - Coalesce(
+        SubquerySum(f'{reference}supplier_parts__purchase_order_line_items__received', filter=order_filter),
+        Decimal(0),
+        output_field=models.DecimalField(),
+    )
+
+
+def annotate_total_stock(reference: str = ''):
+    """Annotate 'total stock' quantity against a queryset:
+
+    - This function calculates the 'total stock' for a given part
+    - Finds all stock items associated with each part (using the provided filter)
+    - Aggregates the 'quantity' of each relevent stock item
+
+    Args:
+        reference: The relationship reference of the part from the current model e.g. 'part'
+        stock_filter: Q object which defines how to filter the stock items
+    """
+
+    # Stock filter only returns 'in stock' items
+    stock_filter = stock.models.StockItem.IN_STOCK_FILTER
+
+    return Coalesce(
+        SubquerySum(
+            f'{reference}stock_items__quantity',
+            filter=stock_filter,
+        ),
+        Decimal(0),
+        output_field=models.DecimalField(),
+    )
+
+
+def annotate_build_order_allocations(reference: str = ''):
+    """Annotate the total quantity of each part allocated to build orders:
+
+    - This function calculates the total part quantity allocated to open build orders
+    - Finds all build order allocations for each part (using the provided filter)
+    - Aggregates the 'allocated quantity' for each relevent build order allocation item
+
+    Args:
+        reference: The relationship reference of the part from the current model
+        build_filter: Q object which defines how to filter the allocation items
+    """
+
+    # Build filter only returns 'active' build orders
+    build_filter = Q(build__status__in=BuildStatus.ACTIVE_CODES)
+
+    return Coalesce(
+        SubquerySum(
+            f'{reference}stock_items__allocations__quantity',
+            filter=build_filter,
+        ),
+        Decimal(0),
+        output_field=models.DecimalField(),
+    )
+
+
+def annotate_sales_order_allocations(reference: str = ''):
+    """Annotate the total quantity of each part allocated to sales orders:
+
+    - This function calculates the total part quantity allocated to open sales orders"
+    - Finds all sales order allocations for each part (using the provided filter)
+    - Aggregates the 'allocated quantity' for each relevent sales order allocation item
+
+    Args:
+        reference: The relationship reference of the part from the current model
+        order_filter: Q object which defines how to filter the allocation items
+    """
+
+    # Order filter only returns incomplete shipments for open orders
+    order_filter = Q(
+        line__order__status__in=SalesOrderStatus.OPEN,
+        shipment__shipment_date=None,
+    )
+
+    return Coalesce(
+        SubquerySum(
+            f'{reference}stock_items__sales_order_allocations__quantity',
+            filter=order_filter,
+        ),
+        Decimal(0),
+        output_field=models.DecimalField(),
+    )
+
+
+def variant_stock_query(reference: str = '', filter: Q = stock.models.StockItem.IN_STOCK_FILTER):
+    """Create a queryset to retrieve all stock items for variant parts under the specified part
+
+    - Useful for annotating a queryset with aggregated information about variant parts
+
+    Args:
+        reference: The relationship reference of the part from the current model
+        filter: Q object which defines how to filter the returned StockItem instances
+    """
+
+    return stock.models.StockItem.objects.filter(
+        part__tree_id=OuterRef(f'{reference}tree_id'),
+        part__lft__gt=OuterRef(f'{reference}lft'),
+        part__rght__lt=OuterRef(f'{reference}rght'),
+    ).filter(filter)

InvenTree/part/serializers.py

@@ -4,8 +4,8 @@ import imghdr
 from decimal import Decimal
 
 from django.db import models, transaction
-from django.db.models import (ExpressionWrapper, F, FloatField, Func, OuterRef,
+from django.db.models import (ExpressionWrapper, F, FloatField, Func, Q,
-                              Q, Subquery)
+                              Subquery)
 from django.db.models.functions import Coalesce
 from django.urls import reverse_lazy
 from django.utils.translation import gettext_lazy as _
@@ -14,6 +14,7 @@ from djmoney.contrib.django_rest_framework import MoneyField
 from rest_framework import serializers
 from sql_util.utils import SubqueryCount, SubquerySum
 
+import part.filters
 from common.settings import currency_code_default, currency_code_mappings
 from InvenTree.serializers import (DataFileExtractSerializer,
                                    DataFileUploadSerializer,
@@ -23,9 +24,7 @@ from InvenTree.serializers import (DataFileExtractSerializer,
                                    InvenTreeImageSerializerField,
                                    InvenTreeModelSerializer,
                                    InvenTreeMoneySerializer)
-from InvenTree.status_codes import (BuildStatus, PurchaseOrderStatus,
+from InvenTree.status_codes import BuildStatus
-                                    SalesOrderStatus)
-from stock.models import StockItem
 
 from .models import (BomItem, BomItemSubstitute, Part, PartAttachment,
                      PartCategory, PartCategoryParameterTemplate,
@@ -311,14 +310,6 @@ class PartSerializer(InvenTreeModelSerializer):
 
         Performing database queries as efficiently as possible, to reduce database trips.
         """
-        # Annotate with the total 'in stock' quantity
-        queryset = queryset.annotate(
-            in_stock=Coalesce(
-                SubquerySum('stock_items__quantity', filter=StockItem.IN_STOCK_FILTER),
-                Decimal(0),
-                output_field=models.DecimalField(),
-            ),
-        )
 
         # Annotate with the total number of stock items
         queryset = queryset.annotate(
@@ -326,11 +317,7 @@ class PartSerializer(InvenTreeModelSerializer):
         )
 
         # Annotate with the total variant stock quantity
-        variant_query = StockItem.objects.filter(
+        variant_query = part.filters.variant_stock_query()
-            part__tree_id=OuterRef('tree_id'),
-            part__lft__gt=OuterRef('lft'),
-            part__rght__lt=OuterRef('rght'),
-        ).filter(StockItem.IN_STOCK_FILTER)
 
         queryset = queryset.annotate(
             variant_stock=Coalesce(
@@ -357,24 +344,6 @@ class PartSerializer(InvenTreeModelSerializer):
             )
         )
 
-        # Filter to limit orders to "open"
-        order_filter = Q(
-            order__status__in=PurchaseOrderStatus.OPEN
-        )
-
-        # Annotate with the total 'on order' quantity
-        queryset = queryset.annotate(
-            ordering=Coalesce(
-                SubquerySum('supplier_parts__purchase_order_line_items__quantity', filter=order_filter),
-                Decimal(0),
-                output_field=models.DecimalField(),
-            ) - Coalesce(
-                SubquerySum('supplier_parts__purchase_order_line_items__received', filter=order_filter),
-                Decimal(0),
-                output_field=models.DecimalField(),
-            )
-        )
-
         # Annotate with the number of 'suppliers'
         queryset = queryset.annotate(
             suppliers=Coalesce(
@@ -384,40 +353,11 @@ class PartSerializer(InvenTreeModelSerializer):
             ),
         )
 
-        """
-        Annotate with the number of stock items allocated to sales orders.
-        This annotation is modeled on Part.sales_order_allocations() method:
-
-        - Only look for "open" orders
-        - Stock items have not been "shipped"
-        """
-        so_allocation_filter = Q(
-            line__order__status__in=SalesOrderStatus.OPEN,  # LineItem points to an OPEN order
-            shipment__shipment_date=None,  # Allocated item has *not* been shipped out
-        )
-
         queryset = queryset.annotate(
-            allocated_to_sales_orders=Coalesce(
+            ordering=part.filters.annotate_on_order_quantity(),
-                SubquerySum('stock_items__sales_order_allocations__quantity', filter=so_allocation_filter),
+            in_stock=part.filters.annotate_total_stock(),
-                Decimal(0),
+            allocated_to_sales_orders=part.filters.annotate_sales_order_allocations(),
-                output_field=models.DecimalField(),
+            allocated_to_build_orders=part.filters.annotate_build_order_allocations(),
-            )
-        )
-
-        """
-        Annotate with the number of stock items allocated to build orders.
-        This annotation is modeled on Part.build_order_allocations() method
-        """
-        bo_allocation_filter = Q(
-            build__status__in=BuildStatus.ACTIVE_CODES,
-        )
-
-        queryset = queryset.annotate(
-            allocated_to_build_orders=Coalesce(
-                SubquerySum('stock_items__allocations__quantity', filter=bo_allocation_filter),
-                Decimal(0),
-                output_field=models.DecimalField(),
-            )
         )
 
         # Annotate with the total 'available stock' quantity
@@ -659,40 +599,16 @@ class BomItemSerializer(InvenTreeModelSerializer):
         Construct an "available stock" quantity:
         available_stock = total_stock - build_order_allocations - sales_order_allocations
         """
-        build_order_filter = Q(build__status__in=BuildStatus.ACTIVE_CODES)
+
-        sales_order_filter = Q(
+        ref = 'sub_part__'
-            line__order__status__in=SalesOrderStatus.OPEN,
-            shipment__shipment_date=None,
-        )
 
         # Calculate "total stock" for the referenced sub_part
         # Calculate the "build_order_allocations" for the sub_part
         # Note that these fields are only aliased, not annotated
         queryset = queryset.alias(
-            total_stock=Coalesce(
+            total_stock=part.filters.annotate_total_stock(reference=ref),
-                SubquerySum(
+            allocated_to_sales_orders=part.filters.annotate_sales_order_allocations(reference=ref),
-                    'sub_part__stock_items__quantity',
+            allocated_to_build_orders=part.filters.annotate_build_order_allocations(reference=ref),
-                    filter=StockItem.IN_STOCK_FILTER
-                ),
-                Decimal(0),
-                output_field=models.DecimalField(),
-            ),
-            allocated_to_sales_orders=Coalesce(
-                SubquerySum(
-                    'sub_part__stock_items__sales_order_allocations__quantity',
-                    filter=sales_order_filter,
-                ),
-                Decimal(0),
-                output_field=models.DecimalField(),
-            ),
-            allocated_to_build_orders=Coalesce(
-                SubquerySum(
-                    'sub_part__stock_items__allocations__quantity',
-                    filter=build_order_filter,
-                ),
-                Decimal(0),
-                output_field=models.DecimalField(),
-            ),
         )
 
         # Calculate 'available_stock' based on previously annotated fields
@@ -703,32 +619,13 @@ class BomItemSerializer(InvenTreeModelSerializer):
             )
         )
 
+        ref = 'substitutes__part__'
+
         # Extract similar information for any 'substitute' parts
         queryset = queryset.alias(
-            substitute_stock=Coalesce(
+            substitute_stock=part.filters.annotate_total_stock(reference=ref),
-                SubquerySum(
+            substitute_build_allocations=part.filters.annotate_build_order_allocations(reference=ref),
-                    'substitutes__part__stock_items__quantity',
+            substitute_sales_allocations=part.filters.annotate_sales_order_allocations(reference=ref)
-                    filter=StockItem.IN_STOCK_FILTER,
-                ),
-                Decimal(0),
-                output_field=models.DecimalField(),
-            ),
-            substitute_build_allocations=Coalesce(
-                SubquerySum(
-                    'substitutes__part__stock_items__allocations__quantity',
-                    filter=build_order_filter,
-                ),
-                Decimal(0),
-                output_field=models.DecimalField(),
-            ),
-            substitute_sales_allocations=Coalesce(
-                SubquerySum(
-                    'substitutes__part__stock_items__sales_order_allocations__quantity',
-                    filter=sales_order_filter,
-                ),
-                Decimal(0),
-                output_field=models.DecimalField(),
-            ),
         )
 
         # Calculate 'available_substitute_stock' field
@@ -740,11 +637,7 @@ class BomItemSerializer(InvenTreeModelSerializer):
         )
 
         # Annotate the queryset with 'available variant stock' information
-        variant_stock_query = StockItem.objects.filter(
+        variant_stock_query = part.filters.variant_stock_query(reference='sub_part__')
-            part__tree_id=OuterRef('sub_part__tree_id'),
-            part__lft__gt=OuterRef('sub_part__lft'),
-            part__rght__lt=OuterRef('sub_part__rght'),
-        ).filter(StockItem.IN_STOCK_FILTER)
 
         queryset = queryset.alias(
             variant_stock_total=Coalesce(

InvenTree/templates/js/translated/order.js

@@ -3540,9 +3540,35 @@ function loadSalesOrderLineItemTable(table, options={}) {
         columns.push(
             {
                 field: 'stock',
-                title: '{% trans "In Stock" %}',
+                title: '{% trans "Available Stock" %}',
                 formatter: function(value, row) {
-                    return row.part_detail.stock;
+                    var available = row.available_stock;
+                    var total = row.part_detail.stock;
+                    var required = Math.max(row.quantity - row.allocated - row.shipped, 0);
+
+                    var html = '';
+
+                    if (total > 0) {
+                        var url = `/part/${row.part}/?display=part-stock`;
+
+                        var text = available;
+
+                        if (total != available) {
+                            text += ` / ${total}`;
+                        }
+
+                        html = renderLink(text, url);
+                    } else {
+                        html += `<span class='badge rounded-pill bg-danger'>{% trans "No Stock Available" %}</span>`;
+                    }
+
+                    if (available >= required) {
+                        html += `<span class='fas fa-check-circle icon-green float-right' title='{% trans "Sufficient stock available" %}'></span>`;
+                    } else {
+                        html += `<span class='fas fa-times-circle icon-red float-right' title='{% trans "Insufficient stock available" %}'></span>`;
+                    }
+
+                    return html;
                 },
             },
         );