Fix: Treegrid is loading an eternity for huge amounts of data (#3451)

* Added default max depth and lazy loading to StorageLocation

* Added default max depth and lazy loading to PartCategory

* Update API version

* lint: fix

* Added INVENTREE_TREE_DEPTH setting

* Refactored int conversion into own helper function

* Added tests

InvenTree/InvenTree/api_version.py

@@ -2,11 +2,15 @@
 
 
 # InvenTree API version
-INVENTREE_API_VERSION = 69
+INVENTREE_API_VERSION = 70
 
 """
 Increment this API version number whenever there is a significant change to the API that any clients need to know about
 
+v70 -> 2022-08-02 : https://github.com/inventree/InvenTree/pull/3451
+    - Adds a 'depth' parameter to the PartCategory list API
+    - Adds a 'depth' parameter to the StockLocation list API
+
 v69 -> 2022-08-01 : https://github.com/inventree/InvenTree/pull/3443
     - Updates the PartCategory list API:
         - Improve query efficiency: O(n) becomes O(1)

InvenTree/InvenTree/helpers.py

@@ -283,6 +283,22 @@ def str2bool(text, test=True):
         return str(text).lower() in ['0', 'n', 'no', 'none', 'f', 'false', 'off', ]
 
 
+def str2int(text, default=None):
+    """Convert a string to int if possible
+
+    Args:
+        text: Int like string
+        default: Return value if str is no int like
+
+    Returns:
+        Converted int value
+    """
+    try:
+        return int(text)
+    except Exception:
+        return default
+
+
 def is_bool(text):
     """Determine if a string value 'looks' like a boolean."""
     if str2bool(text, True):

InvenTree/common/models.py

@@ -876,6 +876,16 @@ class InvenTreeSetting(BaseInvenTreeSetting):
             'default': True,
         },
 
+        'INVENTREE_TREE_DEPTH': {
+            'name': _('Tree Depth'),
+            'description': _('Default tree depth for treeview. Deeper levels can be lazy loaded as they are needed.'),
+            'default': 1,
+            'validator': [
+                int,
+                MinValueValidator(0),
+            ]
+        },
+
         'BARCODE_ENABLE': {
             'name': _('Barcode Support'),
             'description': _('Enable barcode scanner support'),

InvenTree/part/api.py

@@ -25,7 +25,8 @@ from company.models import Company, ManufacturerPart, SupplierPart
 from InvenTree.api import (APIDownloadMixin, AttachmentMixin,
                            ListCreateDestroyAPIView)
 from InvenTree.filters import InvenTreeOrderingFilter
-from InvenTree.helpers import DownloadFile, increment, isNull, str2bool
+from InvenTree.helpers import (DownloadFile, increment, isNull, str2bool,
+                               str2int)
 from InvenTree.mixins import (CreateAPI, ListAPI, ListCreateAPI, RetrieveAPI,
                               RetrieveUpdateAPI, RetrieveUpdateDestroyAPI,
                               UpdateAPI)
@@ -85,6 +86,8 @@ class CategoryList(ListCreateAPI):
 
         cascade = str2bool(params.get('cascade', False))
 
+        depth = str2int(params.get('depth', None))
+
         # Do not filter by category
         if cat_id is None:
             pass
@@ -94,12 +97,18 @@ class CategoryList(ListCreateAPI):
             if not cascade:
                 queryset = queryset.filter(parent=None)
 
+            if cascade and depth is not None:
+                queryset = queryset.filter(level__lte=depth)
+
         else:
             try:
                 category = PartCategory.objects.get(pk=cat_id)
 
                 if cascade:
                     parents = category.get_descendants(include_self=True)
+                    if depth is not None:
+                        parents = parents.filter(level__lte=category.level + depth)
+
                     parent_ids = [p.id for p in parents]
 
                     queryset = queryset.filter(parent__in=parent_ids)

InvenTree/part/test_api.py

@@ -49,33 +49,25 @@ class PartCategoryAPITest(InvenTreeAPITestCase):
         """Test the PartCategoryList API endpoint"""
         url = reverse('api-part-category-list')
 
-        response = self.get(url, expected_code=200)
+        test_cases = [
+            ({}, 8, 'no parameters'),
+            ({'parent': 1, 'cascade': False}, 3, 'Filter by parent, no cascading'),
+            ({'parent': 1, 'cascade': True}, 5, 'Filter by parent, cascading'),
+            ({'cascade': True, 'depth': 0}, 8, 'Cascade with no parent, depth=0'),
+            ({'cascade': False, 'depth': 10}, 8, 'Cascade with no parent, depth=0'),
+            ({'parent': 'null', 'cascade': True, 'depth': 0}, 2, 'Cascade with null parent, depth=0'),
+            ({'parent': 'null', 'cascade': True, 'depth': 10}, 8, 'Cascade with null parent and bigger depth'),
+            ({'parent': 'null', 'cascade': False, 'depth': 10}, 2, 'No cascade even with depth specified with null parent'),
+            ({'parent': 1, 'cascade': False, 'depth': 0}, 3, 'Dont cascade with depth=0 and parent'),
+            ({'parent': 1, 'cascade': True, 'depth': 0}, 3, 'Cascade with depth=0 and parent'),
+            ({'parent': 1, 'cascade': False, 'depth': 1}, 3, 'Dont cascade even with depth=1 specified with parent'),
+            ({'parent': 1, 'cascade': True, 'depth': 1}, 5, 'Cascade with depth=1 with parent'),
+            ({'parent': 1, 'cascade': True, 'depth': 'abcdefg'}, 5, 'Cascade with invalid depth and parent'),
+        ]
 
-        self.assertEqual(len(response.data), 8)
+        for params, res_len, description in test_cases:
-
+            response = self.get(url, params, expected_code=200)
-        # Filter by parent, depth=1
+            self.assertEqual(len(response.data), res_len, description)
-        response = self.get(
-            url,
-            {
-                'parent': 1,
-                'cascade': False,
-            },
-            expected_code=200
-        )
-
-        self.assertEqual(len(response.data), 3)
-
-        # Filter by parent, cascading
-        response = self.get(
-            url,
-            {
-                'parent': 1,
-                'cascade': True,
-            },
-            expected_code=200,
-        )
-
-        self.assertEqual(len(response.data), 5)
 
         # Check that the required fields are present
         fields = [
@@ -90,6 +82,7 @@ class PartCategoryAPITest(InvenTreeAPITestCase):
             'url'
         ]
 
+        response = self.get(url, expected_code=200)
         for result in response.data:
             for f in fields:
                 self.assertIn(f, result)

InvenTree/stock/api.py

@@ -26,7 +26,7 @@ from InvenTree.api import (APIDownloadMixin, AttachmentMixin,
                            ListCreateDestroyAPIView)
 from InvenTree.filters import InvenTreeOrderingFilter
 from InvenTree.helpers import (DownloadFile, extract_serial_numbers, isNull,
-                               str2bool)
+                               str2bool, str2int)
 from InvenTree.mixins import (CreateAPI, ListAPI, ListCreateAPI, RetrieveAPI,
                               RetrieveUpdateAPI, RetrieveUpdateDestroyAPI)
 from order.models import PurchaseOrder, SalesOrder, SalesOrderAllocation
@@ -241,6 +241,8 @@ class StockLocationList(ListCreateAPI):
 
         cascade = str2bool(params.get('cascade', False))
 
+        depth = str2int(params.get('depth', None))
+
         # Do not filter by location
         if loc_id is None:
             pass
@@ -251,6 +253,9 @@ class StockLocationList(ListCreateAPI):
             if not cascade:
                 queryset = queryset.filter(parent=None)
 
+            if cascade and depth is not None:
+                queryset = queryset.filter(level__lte=depth)
+
         else:
 
             try:
@@ -259,6 +264,9 @@ class StockLocationList(ListCreateAPI):
                 # All sub-locations to be returned too?
                 if cascade:
                     parents = location.get_descendants(include_self=True)
+                    if depth is not None:
+                        parents = parents.filter(level__lte=location.level + depth)
+
                     parent_ids = [p.id for p in parents]
                     queryset = queryset.filter(parent__in=parent_ids)
 

InvenTree/stock/test_api.py

@@ -54,11 +54,44 @@ class StockLocationTest(StockAPITestCase):
         StockLocation.objects.create(name='top', description='top category')
 
     def test_list(self):
-        """Test StockLocation list."""
+        """Test the StockLocationList API endpoint"""
-        # Check that we can request the StockLocation list
+        test_cases = [
-        response = self.client.get(self.list_url, format='json')
+            ({}, 8, 'no parameters'),
-        self.assertEqual(response.status_code, status.HTTP_200_OK)
+            ({'parent': 1, 'cascade': False}, 2, 'Filter by parent, no cascading'),
-        self.assertGreaterEqual(len(response.data), 1)
+            ({'parent': 1, 'cascade': True}, 2, 'Filter by parent, cascading'),
+            ({'cascade': True, 'depth': 0}, 8, 'Cascade with no parent, depth=0'),
+            ({'cascade': False, 'depth': 10}, 8, 'Cascade with no parent, depth=0'),
+            ({'parent': 'null', 'cascade': True, 'depth': 0}, 7, 'Cascade with null parent, depth=0'),
+            ({'parent': 'null', 'cascade': True, 'depth': 10}, 8, 'Cascade with null parent and bigger depth'),
+            ({'parent': 'null', 'cascade': False, 'depth': 10}, 3, 'No cascade even with depth specified with null parent'),
+            ({'parent': 1, 'cascade': False, 'depth': 0}, 2, 'Dont cascade with depth=0 and parent'),
+            ({'parent': 1, 'cascade': True, 'depth': 0}, 2, 'Cascade with depth=0 and parent'),
+            ({'parent': 1, 'cascade': False, 'depth': 1}, 2, 'Dont cascade even with depth=1 specified with parent'),
+            ({'parent': 1, 'cascade': True, 'depth': 1}, 2, 'Cascade with depth=1 with parent'),
+            ({'parent': 1, 'cascade': True, 'depth': 'abcdefg'}, 2, 'Cascade with invalid depth and parent'),
+        ]
+
+        for params, res_len, description in test_cases:
+            response = self.get(self.list_url, params, expected_code=200)
+            self.assertEqual(len(response.data), res_len, description)
+
+        # Check that the required fields are present
+        fields = [
+            'pk',
+            'name',
+            'description',
+            'level',
+            'parent',
+            'items',
+            'pathstring',
+            'owner',
+            'url'
+        ]
+
+        response = self.get(self.list_url, expected_code=200)
+        for result in response.data:
+            for f in fields:
+                self.assertIn(f, result)
 
     def test_add(self):
         """Test adding StockLocation."""

InvenTree/templates/InvenTree/settings/global.html

@@ -22,6 +22,7 @@
         {% include "InvenTree/settings/setting.html" with key="INVENTREE_DOWNLOAD_FROM_URL" icon="fa-cloud-download-alt" %}
         {% include "InvenTree/settings/setting.html" with key="INVENTREE_DOWNLOAD_IMAGE_MAX_SIZE" icon="fa-server" %}
         {% include "InvenTree/settings/setting.html" with key="INVENTREE_REQUIRE_CONFIRM" icon="fa-check" %}
+        {% include "InvenTree/settings/setting.html" with key="INVENTREE_TREE_DEPTH" icon="fa-sitemap" %}
     </tbody>
 </table>
 

InvenTree/templates/js/translated/part.js

@@ -1750,6 +1750,7 @@ function loadPartCategoryTable(table, options) {
 
     if (tree_view) {
         params.cascade = true;
+        params.depth = global_settings.INVENTREE_TREE_DEPTH;
     }
 
     var original = {};
@@ -1761,6 +1762,35 @@ function loadPartCategoryTable(table, options) {
 
     setupFilterList(filterKey, table, filterListElement);
 
+    // Function to request sub-category items
+    function requestSubItems(parent_pk) {
+        inventreeGet(
+            options.url || '{% url "api-part-category-list" %}',
+            {
+                parent: parent_pk,
+            },
+            {
+                success: function(response) {
+                    // Add the returned sub-items to the table
+                    for (var idx = 0; idx < response.length; idx++) {
+                        response[idx].parent = parent_pk;
+                    }
+
+                    const row = $(table).bootstrapTable('getRowByUniqueId', parent_pk);
+                    row.subReceived = true;
+
+                    $(table).bootstrapTable('updateByUniqueId', parent_pk, row, true);
+
+                    table.bootstrapTable('append', response);
+                },
+                error: function(xhr) {
+                    console.error('Error requesting sub-category for category=' + parent_pk);
+                    showApiError(xhr);
+                }
+            }
+        );
+    }
+
     table.inventreeTable({
         treeEnable: tree_view,
         rootParentId: tree_view ? options.params.parent : null,
@@ -1839,6 +1869,20 @@ function loadPartCategoryTable(table, options) {
 
                         }
                     });
+
+                    // Callback for 'load sub category' button
+                    $(table).find('.load-sub-category').click(function(event) {
+                        event.preventDefault();
+
+                        const pk = $(this).attr('pk');
+                        const row = $(table).bootstrapTable('getRowByUniqueId', pk);
+
+                        // Request sub-category for this category
+                        requestSubItems(row.pk);
+
+                        row.subRequested = true;
+                        $(table).bootstrapTable('updateByUniqueId', pk, row, true);
+                    });
                 } else {
                     $('#view-category-tree').removeClass('btn-secondary').addClass('btn-outline-secondary');
                     $('#view-category-list').removeClass('btn-outline-secondary').addClass('btn-secondary');
@@ -1859,8 +1903,20 @@ function loadPartCategoryTable(table, options) {
                 switchable: true,
                 sortable: true,
                 formatter: function(value, row) {
+                    let html = '';
 
-                    var html = renderLink(
+                    if (row._level >= global_settings.INVENTREE_TREE_DEPTH && !row.subReceived) {
+                        if (row.subRequested) {
+                            html += `<a href='#'><span class='fas fa-sync fa-spin'></span></a>`;
+                        } else {
+                            html += `
+                                <a href='#' pk='${row.pk}' class='load-sub-category'>
+                                    <span class='fas fa-sync-alt' title='{% trans "Load Subcategories" %}'></span>
+                                </a> `;
+                        }
+                    }
+
+                    html += renderLink(
                         value,
                         `/part/category/${row.pk}/`
                     );

InvenTree/templates/js/translated/stock.js

@@ -2226,6 +2226,7 @@ function loadStockLocationTable(table, options) {
 
     if (tree_view) {
         params.cascade = true;
+        params.depth = global_settings.INVENTREE_TREE_DEPTH;
     }
 
     var filters = {};
@@ -2248,6 +2249,35 @@ function loadStockLocationTable(table, options) {
         filters[key] = params[key];
     }
 
+    // Function to request sub-location items
+    function requestSubItems(parent_pk) {
+        inventreeGet(
+            options.url || '{% url "api-location-list" %}',
+            {
+                parent: parent_pk,
+            },
+            {
+                success: function(response) {
+                    // Add the returned sub-items to the table
+                    for (var idx = 0; idx < response.length; idx++) {
+                        response[idx].parent = parent_pk;
+                    }
+
+                    const row = $(table).bootstrapTable('getRowByUniqueId', parent_pk);
+                    row.subReceived = true;
+
+                    $(table).bootstrapTable('updateByUniqueId', parent_pk, row, true);
+
+                    table.bootstrapTable('append', response);
+                },
+                error: function(xhr) {
+                    console.error('Error requesting sub-locations for location=' + parent_pk);
+                    showApiError(xhr);
+                }
+            }
+        );
+    }
+
     table.inventreeTable({
         treeEnable: tree_view,
         rootParentId: tree_view ? options.params.parent : null,
@@ -2286,6 +2316,20 @@ function loadStockLocationTable(table, options) {
 
                         }
                     });
+
+                    // Callback for 'load sub location' button
+                    $(table).find('.load-sub-location').click(function(event) {
+                        event.preventDefault();
+
+                        const pk = $(this).attr('pk');
+                        const row = $(table).bootstrapTable('getRowByUniqueId', pk);
+
+                        // Request sub-location for this location
+                        requestSubItems(row.pk);
+
+                        row.subRequested = true;
+                        $(table).bootstrapTable('updateByUniqueId', pk, row, true);
+                    });
                 } else {
                     $('#view-location-tree').removeClass('btn-secondary').addClass('btn-outline-secondary');
                     $('#view-location-list').removeClass('btn-outline-secondary').addClass('btn-secondary');
@@ -2345,10 +2389,25 @@ function loadStockLocationTable(table, options) {
                 switchable: true,
                 sortable: true,
                 formatter: function(value, row) {
-                    return renderLink(
+                    let html = '';
+
+                    if (row._level >= global_settings.INVENTREE_TREE_DEPTH && !row.subReceived) {
+                        if (row.subRequested) {
+                            html += `<a href='#'><span class='fas fa-sync fa-spin'></span></a>`;
+                        } else {
+                            html += `
+                                <a href='#' pk='${row.pk}' class='load-sub-location'>
+                                    <span class='fas fa-sync-alt' title='{% trans "Load Subloactions" %}'></span>
+                                </a> `;
+                        }
+                    }
+
+                    html += renderLink(
                         value,
                         `/stock/location/${row.pk}/`
                     );
+
+                    return html;
                 },
             },
             {