Setting caching (#3178)

* Revert "Remove stat context variables"

This reverts commit 0989c308d0.

* Add a caching framework for inventree settings

- Actions that use "settings" require a DB hit every time
- For example the part.full_name() method looks at the PART_NAME_FORMAT setting
- This means 1 DB hit for every part which is serialized!!

* Fixes for DebugToolbar integration

- Requires different INTERNAL_IPS when running behind docker
- Some issues with TEMPLATES framework

* Revert "Revert "Remove stat context variables""

This reverts commit 52e6359265.

* Add unit tests for settings caching

* Update existing unit tests to handle cache framework

* Fix for unit test

* Re-enable cache for default part values

* Clear cache for further unit tests

InvenTree/common/apps.py

@@ -24,7 +24,7 @@ class CommonConfig(AppConfig):
         try:
             import common.models
 
-            if common.models.InvenTreeSetting.get_setting('SERVER_RESTART_REQUIRED', backup_value=False, create=False):
+            if common.models.InvenTreeSetting.get_setting('SERVER_RESTART_REQUIRED', backup_value=False, create=False, cache=False):
                 logger.info("Clearing SERVER_RESTART_REQUIRED flag")
                 common.models.InvenTreeSetting.set_setting('SERVER_RESTART_REQUIRED', False, None)
         except Exception:

InvenTree/common/models.py

@@ -21,7 +21,8 @@ from django.contrib.auth.models import Group, User
 from django.contrib.contenttypes.fields import GenericForeignKey
 from django.contrib.contenttypes.models import ContentType
 from django.contrib.humanize.templatetags.humanize import naturaltime
-from django.core.exceptions import ValidationError
+from django.core.cache import cache
+from django.core.exceptions import AppRegistryNotReady, ValidationError
 from django.core.validators import MinValueValidator, URLValidator
 from django.db import models, transaction
 from django.db.utils import IntegrityError, OperationalError
@@ -69,11 +70,56 @@ class BaseInvenTreeSetting(models.Model):
         """Enforce validation and clean before saving."""
         self.key = str(self.key).upper()
 
+        do_cache = kwargs.pop('cache', True)
+
         self.clean(**kwargs)
         self.validate_unique(**kwargs)
 
+        # Update this setting in the cache
+        if do_cache:
+            self.save_to_cache()
+
         super().save()
 
+    @property
+    def cache_key(self):
+        """Generate a unique cache key for this settings object"""
+        return self.__class__.create_cache_key(self.key, **self.get_kwargs())
+
+    def save_to_cache(self):
+        """Save this setting object to cache"""
+
+        ckey = self.cache_key
+
+        logger.debug(f"Saving setting '{ckey}' to cache")
+
+        try:
+            cache.set(
+                ckey,
+                self,
+                timeout=3600
+            )
+        except TypeError:
+            # Some characters cause issues with caching; ignore and move on
+            pass
+
+    @classmethod
+    def create_cache_key(cls, setting_key, **kwargs):
+        """Create a unique cache key for a particular setting object.
+
+        The cache key uses the following elements to ensure the key is 'unique':
+        - The name of the class
+        - The unique KEY string
+        - Any key:value kwargs associated with the particular setting type (e.g. user-id)
+        """
+
+        key = f"{str(cls.__name__)}:{setting_key}"
+
+        for k, v in kwargs.items():
+            key += f"_{k}:{v}"
+
+        return key
+
     @classmethod
     def allValues(cls, user=None, exclude_hidden=False):
         """Return a dict of "all" defined global settings.
@@ -220,11 +266,12 @@ class BaseInvenTreeSetting(models.Model):
 
         - Key is case-insensitive
         - Returns None if no match is made
+
+        First checks the cache to see if this object has recently been accessed,
+        and returns the cached version if so.
         """
         key = str(key).strip().upper()
 
-        settings = cls.objects.all()
-
         filters = {
             'key__iexact': key,
         }
@@ -253,7 +300,25 @@ class BaseInvenTreeSetting(models.Model):
         if method is not None:
             filters['method'] = method
 
+        # Perform cache lookup by default
+        do_cache = kwargs.pop('cache', True)
+
+        ckey = cls.create_cache_key(key, **kwargs)
+
+        if do_cache:
+            try:
+                # First attempt to find the setting object in the cache
+                cached_setting = cache.get(ckey)
+
+                if cached_setting is not None:
+                    return cached_setting
+
+            except AppRegistryNotReady:
+                # Cache is not ready yet
+                do_cache = False
+
         try:
+            settings = cls.objects.all()
             setting = settings.filter(**filters).first()
         except (ValueError, cls.DoesNotExist):
             setting = None
@@ -282,6 +347,10 @@ class BaseInvenTreeSetting(models.Model):
                     # It might be the case that the database isn't created yet
                     pass
 
+        if setting and do_cache:
+            # Cache this setting object
+            setting.save_to_cache()
+
         return setting
 
     @classmethod
@@ -1507,11 +1576,6 @@ class InvenTreeUserSetting(BaseInvenTreeSetting):
         help_text=_('User'),
     )
 
-    @classmethod
-    def get_setting_object(cls, key, user=None):
-        """Return setting object for provided user."""
-        return super().get_setting_object(key, user=user)
-
     def validate_unique(self, exclude=None, **kwargs):
         """Return if the setting (including key) is unique."""
         return super().validate_unique(exclude=exclude, user=self.user)

InvenTree/common/settings.py

@@ -12,7 +12,7 @@ def currency_code_default():
     from common.models import InvenTreeSetting
 
     try:
-        code = InvenTreeSetting.get_setting('INVENTREE_DEFAULT_CURRENCY', create=False)
+        code = InvenTreeSetting.get_setting('INVENTREE_DEFAULT_CURRENCY', create=False, cache=False)
     except ProgrammingError:  # pragma: no cover
         # database is not initialized yet
         code = ''

InvenTree/common/tests.py

@@ -4,6 +4,8 @@ import json
 from datetime import timedelta
 from http import HTTPStatus
 
+from django.contrib.auth import get_user_model
+from django.core.cache import cache
 from django.test import Client, TestCase
 from django.urls import reverse
 
@@ -45,10 +47,10 @@ class SettingsTest(InvenTreeTestCase):
         """Test settings functions and properties."""
         # define settings to check
         instance_ref = 'INVENTREE_INSTANCE'
-        instance_obj = InvenTreeSetting.get_setting_object(instance_ref)
+        instance_obj = InvenTreeSetting.get_setting_object(instance_ref, cache=False)
 
         stale_ref = 'STOCK_STALE_DAYS'
-        stale_days = InvenTreeSetting.get_setting_object(stale_ref)
+        stale_days = InvenTreeSetting.get_setting_object(stale_ref, cache=False)
 
         report_size_obj = InvenTreeSetting.get_setting_object('REPORT_DEFAULT_PAGE_SIZE')
         report_test_obj = InvenTreeSetting.get_setting_object('REPORT_ENABLE_TEST_REPORT')
@@ -189,6 +191,56 @@ class SettingsTest(InvenTreeTestCase):
                 if setting.default_value not in [True, False]:
                     raise ValueError(f'Non-boolean default value specified for {key}')  # pragma: no cover
 
+    def test_global_setting_caching(self):
+        """Test caching operations for the global settings class"""
+
+        key = 'PART_NAME_FORMAT'
+
+        cache_key = InvenTreeSetting.create_cache_key(key)
+        self.assertEqual(cache_key, 'InvenTreeSetting:PART_NAME_FORMAT')
+
+        cache.clear()
+
+        self.assertIsNone(cache.get(cache_key))
+
+        # First request should set cache
+        val = InvenTreeSetting.get_setting(key)
+        self.assertEqual(cache.get(cache_key).value, val)
+
+        for val in ['A', '{{ part.IPN }}', 'C']:
+            # Check that the cached value is updated whenever the setting is saved
+            InvenTreeSetting.set_setting(key, val, None)
+            self.assertEqual(cache.get(cache_key).value, val)
+            self.assertEqual(InvenTreeSetting.get_setting(key), val)
+
+    def test_user_setting_caching(self):
+        """Test caching operation for the user settings class"""
+
+        cache.clear()
+
+        # Generate a number of new usesr
+        for idx in range(5):
+            get_user_model().objects.create(
+                username=f"User_{idx}",
+                password="hunter42",
+                email="email@dot.com",
+            )
+
+        key = 'SEARCH_PREVIEW_RESULTS'
+
+        # Check that the settings are correctly cached for each separate user
+        for user in get_user_model().objects.all():
+            setting = InvenTreeUserSetting.get_setting_object(key, user=user)
+            cache_key = setting.cache_key
+            self.assertEqual(cache_key, f"InvenTreeUserSetting:SEARCH_PREVIEW_RESULTS_user:{user.username}")
+            InvenTreeUserSetting.set_setting(key, user.pk, None, user=user)
+            self.assertIsNotNone(cache.get(cache_key))
+
+        # Iterate through a second time, ensure the values have been cached correctly
+        for user in get_user_model().objects.all():
+            value = InvenTreeUserSetting.get_setting(key, user=user)
+            self.assertEqual(value, user.pk)
+
 
 class GlobalSettingsApiTest(InvenTreeAPITestCase):
     """Tests for the global settings API."""
@@ -199,7 +251,7 @@ class GlobalSettingsApiTest(InvenTreeAPITestCase):
 
         # Read out each of the global settings value, to ensure they are instantiated in the database
         for key in InvenTreeSetting.SETTINGS:
-            InvenTreeSetting.get_setting_object(key)
+            InvenTreeSetting.get_setting_object(key, cache=False)
 
         response = self.get(url, expected_code=200)
 
@@ -422,7 +474,8 @@ class UserSettingsApiTest(InvenTreeAPITestCase):
         """Test a integer user setting value."""
         setting = InvenTreeUserSetting.get_setting_object(
             'SEARCH_PREVIEW_RESULTS',
-            user=self.user
+            user=self.user,
+            cache=False,
         )
 
         url = reverse('api-user-setting-detail', kwargs={'key': setting.key})