Setting caching (#3178)

* Revert "Remove stat context variables"

This reverts commit 0989c308d0.

* Add a caching framework for inventree settings

- Actions that use "settings" require a DB hit every time
- For example the part.full_name() method looks at the PART_NAME_FORMAT setting
- This means 1 DB hit for every part which is serialized!!

* Fixes for DebugToolbar integration

- Requires different INTERNAL_IPS when running behind docker
- Some issues with TEMPLATES framework

* Revert "Revert "Remove stat context variables""

This reverts commit 52e6359265.

* Add unit tests for settings caching

* Update existing unit tests to handle cache framework

* Fix for unit test

* Re-enable cache for default part values

* Clear cache for further unit tests

InvenTree/InvenTree/settings.py

@@ -18,7 +18,6 @@ import sys
 from datetime import datetime
 
 import django.conf.locale
-from django.contrib.messages import constants as messages
 from django.core.files.storage import default_storage
 from django.utils.translation import gettext_lazy as _
 
@@ -302,12 +301,24 @@ AUTHENTICATION_BACKENDS = CONFIG.get('authentication_backends', [
     'allauth.account.auth_backends.AuthenticationBackend',      # SSO login via external providers
 ])
 
+DEBUG_TOOLBAR_ENABLED = DEBUG and CONFIG.get('debug_toolbar', False)
+
 # If the debug toolbar is enabled, add the modules
-if DEBUG and CONFIG.get('debug_toolbar', False):  # pragma: no cover
+if DEBUG_TOOLBAR_ENABLED:  # pragma: no cover
     logger.info("Running with DEBUG_TOOLBAR enabled")
     INSTALLED_APPS.append('debug_toolbar')
     MIDDLEWARE.append('debug_toolbar.middleware.DebugToolbarMiddleware')
 
+# Internal IP addresses allowed to see the debug toolbar
+INTERNAL_IPS = [
+    '127.0.0.1',
+]
+
+if DOCKER:
+    # Internal IP addresses are different when running under docker
+    hostname, ___, ips = socket.gethostbyname_ex(socket.gethostname())
+    INTERNAL_IPS = [ip[: ip.rfind(".")] + ".1" for ip in ips] + ["127.0.0.1", "10.0.2.2"]
+
 # Allow secure http developer server in debug mode
 if DEBUG:
     INSTALLED_APPS.append('sslserver')
@@ -354,6 +365,12 @@ TEMPLATES = [
     },
 ]
 
+if DEBUG_TOOLBAR_ENABLED:
+    # Note that the APP_DIRS value must be set when using debug_toolbar
+    # But this will kill template loading for plugins
+    TEMPLATES[0]['APP_DIRS'] = True
+    del TEMPLATES[0]['OPTIONS']['loaders']
+
 REST_FRAMEWORK = {
     'EXCEPTION_HANDLER': 'InvenTree.exceptions.exception_handler',
     'DATETIME_FORMAT': '%Y-%m-%d %H:%M',
@@ -810,17 +827,6 @@ CRISPY_TEMPLATE_PACK = 'bootstrap4'
 # Use database transactions when importing / exporting data
 IMPORT_EXPORT_USE_TRANSACTIONS = True
 
-# Internal IP addresses allowed to see the debug toolbar
-INTERNAL_IPS = [
-    '127.0.0.1',
-]
-
-MESSAGE_TAGS = {
-    messages.SUCCESS: 'alert alert-block alert-success',
-    messages.ERROR: 'alert alert-block alert-danger',
-    messages.INFO: 'alert alert-block alert-info',
-}
-
 SITE_ID = 1
 
 # Load the allauth social backends

InvenTree/InvenTree/urls.py

@@ -188,10 +188,10 @@ if settings.DEBUG:
     urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
 
     # Debug toolbar access (only allowed in DEBUG mode)
-    if 'debug_toolbar' in settings.INSTALLED_APPS:  # pragma: no cover
+    if settings.DEBUG_TOOLBAR_ENABLED:
         import debug_toolbar
         urlpatterns = [
-            path('__debug/', include(debug_toolbar.urls)),
+            path('__debug__/', include(debug_toolbar.urls)),
         ] + urlpatterns
 
 # Send any unknown URLs to the parts page

InvenTree/common/apps.py

@@ -24,7 +24,7 @@ class CommonConfig(AppConfig):
         try:
             import common.models
 
-            if common.models.InvenTreeSetting.get_setting('SERVER_RESTART_REQUIRED', backup_value=False, create=False):
+            if common.models.InvenTreeSetting.get_setting('SERVER_RESTART_REQUIRED', backup_value=False, create=False, cache=False):
                 logger.info("Clearing SERVER_RESTART_REQUIRED flag")
                 common.models.InvenTreeSetting.set_setting('SERVER_RESTART_REQUIRED', False, None)
         except Exception:

InvenTree/common/models.py

@@ -21,7 +21,8 @@ from django.contrib.auth.models import Group, User
 from django.contrib.contenttypes.fields import GenericForeignKey
 from django.contrib.contenttypes.models import ContentType
 from django.contrib.humanize.templatetags.humanize import naturaltime
-from django.core.exceptions import ValidationError
+from django.core.cache import cache
+from django.core.exceptions import AppRegistryNotReady, ValidationError
 from django.core.validators import MinValueValidator, URLValidator
 from django.db import models, transaction
 from django.db.utils import IntegrityError, OperationalError
@@ -69,11 +70,56 @@ class BaseInvenTreeSetting(models.Model):
         """Enforce validation and clean before saving."""
         self.key = str(self.key).upper()
 
+        do_cache = kwargs.pop('cache', True)
+
         self.clean(**kwargs)
         self.validate_unique(**kwargs)
 
+        # Update this setting in the cache
+        if do_cache:
+            self.save_to_cache()
+
         super().save()
 
+    @property
+    def cache_key(self):
+        """Generate a unique cache key for this settings object"""
+        return self.__class__.create_cache_key(self.key, **self.get_kwargs())
+
+    def save_to_cache(self):
+        """Save this setting object to cache"""
+
+        ckey = self.cache_key
+
+        logger.debug(f"Saving setting '{ckey}' to cache")
+
+        try:
+            cache.set(
+                ckey,
+                self,
+                timeout=3600
+            )
+        except TypeError:
+            # Some characters cause issues with caching; ignore and move on
+            pass
+
+    @classmethod
+    def create_cache_key(cls, setting_key, **kwargs):
+        """Create a unique cache key for a particular setting object.
+
+        The cache key uses the following elements to ensure the key is 'unique':
+        - The name of the class
+        - The unique KEY string
+        - Any key:value kwargs associated with the particular setting type (e.g. user-id)
+        """
+
+        key = f"{str(cls.__name__)}:{setting_key}"
+
+        for k, v in kwargs.items():
+            key += f"_{k}:{v}"
+
+        return key
+
     @classmethod
     def allValues(cls, user=None, exclude_hidden=False):
         """Return a dict of "all" defined global settings.
@@ -220,11 +266,12 @@ class BaseInvenTreeSetting(models.Model):
 
         - Key is case-insensitive
         - Returns None if no match is made
+
+        First checks the cache to see if this object has recently been accessed,
+        and returns the cached version if so.
         """
         key = str(key).strip().upper()
 
-        settings = cls.objects.all()
-
         filters = {
             'key__iexact': key,
         }
@@ -253,7 +300,25 @@ class BaseInvenTreeSetting(models.Model):
         if method is not None:
             filters['method'] = method
 
+        # Perform cache lookup by default
+        do_cache = kwargs.pop('cache', True)
+
+        ckey = cls.create_cache_key(key, **kwargs)
+
+        if do_cache:
             try:
+                # First attempt to find the setting object in the cache
+                cached_setting = cache.get(ckey)
+
+                if cached_setting is not None:
+                    return cached_setting
+
+            except AppRegistryNotReady:
+                # Cache is not ready yet
+                do_cache = False
+
+        try:
+            settings = cls.objects.all()
             setting = settings.filter(**filters).first()
         except (ValueError, cls.DoesNotExist):
             setting = None
@@ -282,6 +347,10 @@ class BaseInvenTreeSetting(models.Model):
                     # It might be the case that the database isn't created yet
                     pass
 
+        if setting and do_cache:
+            # Cache this setting object
+            setting.save_to_cache()
+
         return setting
 
     @classmethod
@@ -1507,11 +1576,6 @@ class InvenTreeUserSetting(BaseInvenTreeSetting):
         help_text=_('User'),
     )
 
-    @classmethod
-    def get_setting_object(cls, key, user=None):
-        """Return setting object for provided user."""
-        return super().get_setting_object(key, user=user)
-
     def validate_unique(self, exclude=None, **kwargs):
         """Return if the setting (including key) is unique."""
         return super().validate_unique(exclude=exclude, user=self.user)

InvenTree/common/settings.py

@@ -12,7 +12,7 @@ def currency_code_default():
     from common.models import InvenTreeSetting
 
     try:
-        code = InvenTreeSetting.get_setting('INVENTREE_DEFAULT_CURRENCY', create=False)
+        code = InvenTreeSetting.get_setting('INVENTREE_DEFAULT_CURRENCY', create=False, cache=False)
     except ProgrammingError:  # pragma: no cover
         # database is not initialized yet
         code = ''

InvenTree/common/tests.py

@@ -4,6 +4,8 @@ import json
 from datetime import timedelta
 from http import HTTPStatus
 
+from django.contrib.auth import get_user_model
+from django.core.cache import cache
 from django.test import Client, TestCase
 from django.urls import reverse
 
@@ -45,10 +47,10 @@ class SettingsTest(InvenTreeTestCase):
         """Test settings functions and properties."""
         # define settings to check
         instance_ref = 'INVENTREE_INSTANCE'
-        instance_obj = InvenTreeSetting.get_setting_object(instance_ref)
+        instance_obj = InvenTreeSetting.get_setting_object(instance_ref, cache=False)
 
         stale_ref = 'STOCK_STALE_DAYS'
-        stale_days = InvenTreeSetting.get_setting_object(stale_ref)
+        stale_days = InvenTreeSetting.get_setting_object(stale_ref, cache=False)
 
         report_size_obj = InvenTreeSetting.get_setting_object('REPORT_DEFAULT_PAGE_SIZE')
         report_test_obj = InvenTreeSetting.get_setting_object('REPORT_ENABLE_TEST_REPORT')
@@ -189,6 +191,56 @@ class SettingsTest(InvenTreeTestCase):
                 if setting.default_value not in [True, False]:
                     raise ValueError(f'Non-boolean default value specified for {key}')  # pragma: no cover
 
+    def test_global_setting_caching(self):
+        """Test caching operations for the global settings class"""
+
+        key = 'PART_NAME_FORMAT'
+
+        cache_key = InvenTreeSetting.create_cache_key(key)
+        self.assertEqual(cache_key, 'InvenTreeSetting:PART_NAME_FORMAT')
+
+        cache.clear()
+
+        self.assertIsNone(cache.get(cache_key))
+
+        # First request should set cache
+        val = InvenTreeSetting.get_setting(key)
+        self.assertEqual(cache.get(cache_key).value, val)
+
+        for val in ['A', '{{ part.IPN }}', 'C']:
+            # Check that the cached value is updated whenever the setting is saved
+            InvenTreeSetting.set_setting(key, val, None)
+            self.assertEqual(cache.get(cache_key).value, val)
+            self.assertEqual(InvenTreeSetting.get_setting(key), val)
+
+    def test_user_setting_caching(self):
+        """Test caching operation for the user settings class"""
+
+        cache.clear()
+
+        # Generate a number of new usesr
+        for idx in range(5):
+            get_user_model().objects.create(
+                username=f"User_{idx}",
+                password="hunter42",
+                email="email@dot.com",
+            )
+
+        key = 'SEARCH_PREVIEW_RESULTS'
+
+        # Check that the settings are correctly cached for each separate user
+        for user in get_user_model().objects.all():
+            setting = InvenTreeUserSetting.get_setting_object(key, user=user)
+            cache_key = setting.cache_key
+            self.assertEqual(cache_key, f"InvenTreeUserSetting:SEARCH_PREVIEW_RESULTS_user:{user.username}")
+            InvenTreeUserSetting.set_setting(key, user.pk, None, user=user)
+            self.assertIsNotNone(cache.get(cache_key))
+
+        # Iterate through a second time, ensure the values have been cached correctly
+        for user in get_user_model().objects.all():
+            value = InvenTreeUserSetting.get_setting(key, user=user)
+            self.assertEqual(value, user.pk)
+
 
 class GlobalSettingsApiTest(InvenTreeAPITestCase):
     """Tests for the global settings API."""
@@ -199,7 +251,7 @@ class GlobalSettingsApiTest(InvenTreeAPITestCase):
 
         # Read out each of the global settings value, to ensure they are instantiated in the database
         for key in InvenTreeSetting.SETTINGS:
-            InvenTreeSetting.get_setting_object(key)
+            InvenTreeSetting.get_setting_object(key, cache=False)
 
         response = self.get(url, expected_code=200)
 
@@ -422,7 +474,8 @@ class UserSettingsApiTest(InvenTreeAPITestCase):
         """Test a integer user setting value."""
         setting = InvenTreeUserSetting.get_setting_object(
             'SEARCH_PREVIEW_RESULTS',
-            user=self.user
+            user=self.user,
+            cache=False,
         )
 
         url = reverse('api-user-setting-detail', kwargs={'key': setting.key})

InvenTree/part/test_part.py

@@ -3,6 +3,7 @@
 import os
 
 from django.conf import settings
+from django.core.cache import cache
 from django.core.exceptions import ValidationError
 from django.test import TestCase
 
@@ -394,6 +395,9 @@ class PartSettingsTest(InvenTreeTestCase):
 
     def make_part(self):
         """Helper function to create a simple part."""
+
+        cache.clear()
+
         part = Part.objects.create(
             name='Test Part',
             description='I am but a humble test part',
@@ -404,6 +408,9 @@ class PartSettingsTest(InvenTreeTestCase):
 
     def test_defaults(self):
         """Test that the default values for the part settings are correct."""
+
+        cache.clear()
+
         self.assertTrue(part.settings.part_component_default())
         self.assertTrue(part.settings.part_purchaseable_default())
         self.assertFalse(part.settings.part_salable_default())
@@ -411,6 +418,9 @@ class PartSettingsTest(InvenTreeTestCase):
 
     def test_initial(self):
         """Test the 'initial' default values (no default values have been set)"""
+
+        cache.clear()
+
         part = self.make_part()
 
         self.assertTrue(part.component)

InvenTree/plugin/apps.py

@@ -36,7 +36,7 @@ class PluginAppConfig(AppConfig):
                     # this is the first startup
                     try:
                         from common.models import InvenTreeSetting
-                        if InvenTreeSetting.get_setting('PLUGIN_ON_STARTUP', create=False):
+                        if InvenTreeSetting.get_setting('PLUGIN_ON_STARTUP', create=False, cache=False):
                             # make sure all plugins are installed
                             registry.install_plugin_file()
                     except Exception:  # pragma: no cover

InvenTree/report/tests.py

@@ -204,7 +204,7 @@ class BuildReportTest(ReportTest):
         self.assertEqual(headers['Content-Disposition'], 'attachment; filename="report.pdf"')
 
         # Now, set the download type to be "inline"
-        inline = InvenTreeUserSetting.get_setting_object('REPORT_INLINE', self.user)
+        inline = InvenTreeUserSetting.get_setting_object('REPORT_INLINE', user=self.user)
         inline.value = True
         inline.save()