From 6eddcd3c23b1ee64dd49c48da97ba8d6a3117a50 Mon Sep 17 00:00:00 2001 From: Oliver Date: Sun, 12 Jun 2022 10:56:16 +1000 Subject: [PATCH] Setting caching (#3178) * Revert "Remove stat context variables" This reverts commit 0989c308d0cea9b9405a1338d257b542c6d33d73. * Add a caching framework for inventree settings - Actions that use "settings" require a DB hit every time - For example the part.full_name() method looks at the PART_NAME_FORMAT setting - This means 1 DB hit for every part which is serialized!! * Fixes for DebugToolbar integration - Requires different INTERNAL_IPS when running behind docker - Some issues with TEMPLATES framework * Revert "Revert "Remove stat context variables"" This reverts commit 52e6359265226126da7ed6ed2aed2b83aa33de17. * Add unit tests for settings caching * Update existing unit tests to handle cache framework * Fix for unit test * Re-enable cache for default part values * Clear cache for further unit tests --- InvenTree/InvenTree/settings.py | 32 +++++++------ InvenTree/InvenTree/urls.py | 4 +- InvenTree/common/apps.py | 2 +- InvenTree/common/models.py | 80 +++++++++++++++++++++++++++++---- InvenTree/common/settings.py | 2 +- InvenTree/common/tests.py | 61 +++++++++++++++++++++++-- InvenTree/part/test_part.py | 10 +++++ InvenTree/plugin/apps.py | 2 +- InvenTree/report/tests.py | 2 +- 9 files changed, 164 insertions(+), 31 deletions(-) diff --git a/InvenTree/InvenTree/settings.py b/InvenTree/InvenTree/settings.py index 059ae72909..0a01937e16 100644 --- a/InvenTree/InvenTree/settings.py +++ b/InvenTree/InvenTree/settings.py @@ -18,7 +18,6 @@ import sys from datetime import datetime import django.conf.locale -from django.contrib.messages import constants as messages from django.core.files.storage import default_storage from django.utils.translation import gettext_lazy as _ @@ -302,12 +301,24 @@ AUTHENTICATION_BACKENDS = CONFIG.get('authentication_backends', [ 'allauth.account.auth_backends.AuthenticationBackend', # SSO login via external providers ]) +DEBUG_TOOLBAR_ENABLED = DEBUG and CONFIG.get('debug_toolbar', False) + # If the debug toolbar is enabled, add the modules -if DEBUG and CONFIG.get('debug_toolbar', False): # pragma: no cover +if DEBUG_TOOLBAR_ENABLED: # pragma: no cover logger.info("Running with DEBUG_TOOLBAR enabled") INSTALLED_APPS.append('debug_toolbar') MIDDLEWARE.append('debug_toolbar.middleware.DebugToolbarMiddleware') +# Internal IP addresses allowed to see the debug toolbar +INTERNAL_IPS = [ + '127.0.0.1', +] + +if DOCKER: + # Internal IP addresses are different when running under docker + hostname, ___, ips = socket.gethostbyname_ex(socket.gethostname()) + INTERNAL_IPS = [ip[: ip.rfind(".")] + ".1" for ip in ips] + ["127.0.0.1", "10.0.2.2"] + # Allow secure http developer server in debug mode if DEBUG: INSTALLED_APPS.append('sslserver') @@ -354,6 +365,12 @@ TEMPLATES = [ }, ] +if DEBUG_TOOLBAR_ENABLED: + # Note that the APP_DIRS value must be set when using debug_toolbar + # But this will kill template loading for plugins + TEMPLATES[0]['APP_DIRS'] = True + del TEMPLATES[0]['OPTIONS']['loaders'] + REST_FRAMEWORK = { 'EXCEPTION_HANDLER': 'InvenTree.exceptions.exception_handler', 'DATETIME_FORMAT': '%Y-%m-%d %H:%M', @@ -810,17 +827,6 @@ CRISPY_TEMPLATE_PACK = 'bootstrap4' # Use database transactions when importing / exporting data IMPORT_EXPORT_USE_TRANSACTIONS = True -# Internal IP addresses allowed to see the debug toolbar -INTERNAL_IPS = [ - '127.0.0.1', -] - -MESSAGE_TAGS = { - messages.SUCCESS: 'alert alert-block alert-success', - messages.ERROR: 'alert alert-block alert-danger', - messages.INFO: 'alert alert-block alert-info', -} - SITE_ID = 1 # Load the allauth social backends diff --git a/InvenTree/InvenTree/urls.py b/InvenTree/InvenTree/urls.py index 897350e611..c42ab7699c 100644 --- a/InvenTree/InvenTree/urls.py +++ b/InvenTree/InvenTree/urls.py @@ -188,10 +188,10 @@ if settings.DEBUG: urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) # Debug toolbar access (only allowed in DEBUG mode) - if 'debug_toolbar' in settings.INSTALLED_APPS: # pragma: no cover + if settings.DEBUG_TOOLBAR_ENABLED: import debug_toolbar urlpatterns = [ - path('__debug/', include(debug_toolbar.urls)), + path('__debug__/', include(debug_toolbar.urls)), ] + urlpatterns # Send any unknown URLs to the parts page diff --git a/InvenTree/common/apps.py b/InvenTree/common/apps.py index 16897db534..331ea5bf7d 100644 --- a/InvenTree/common/apps.py +++ b/InvenTree/common/apps.py @@ -24,7 +24,7 @@ class CommonConfig(AppConfig): try: import common.models - if common.models.InvenTreeSetting.get_setting('SERVER_RESTART_REQUIRED', backup_value=False, create=False): + if common.models.InvenTreeSetting.get_setting('SERVER_RESTART_REQUIRED', backup_value=False, create=False, cache=False): logger.info("Clearing SERVER_RESTART_REQUIRED flag") common.models.InvenTreeSetting.set_setting('SERVER_RESTART_REQUIRED', False, None) except Exception: diff --git a/InvenTree/common/models.py b/InvenTree/common/models.py index 35693ee56e..e47ad56783 100644 --- a/InvenTree/common/models.py +++ b/InvenTree/common/models.py @@ -21,7 +21,8 @@ from django.contrib.auth.models import Group, User from django.contrib.contenttypes.fields import GenericForeignKey from django.contrib.contenttypes.models import ContentType from django.contrib.humanize.templatetags.humanize import naturaltime -from django.core.exceptions import ValidationError +from django.core.cache import cache +from django.core.exceptions import AppRegistryNotReady, ValidationError from django.core.validators import MinValueValidator, URLValidator from django.db import models, transaction from django.db.utils import IntegrityError, OperationalError @@ -69,11 +70,56 @@ class BaseInvenTreeSetting(models.Model): """Enforce validation and clean before saving.""" self.key = str(self.key).upper() + do_cache = kwargs.pop('cache', True) + self.clean(**kwargs) self.validate_unique(**kwargs) + # Update this setting in the cache + if do_cache: + self.save_to_cache() + super().save() + @property + def cache_key(self): + """Generate a unique cache key for this settings object""" + return self.__class__.create_cache_key(self.key, **self.get_kwargs()) + + def save_to_cache(self): + """Save this setting object to cache""" + + ckey = self.cache_key + + logger.debug(f"Saving setting '{ckey}' to cache") + + try: + cache.set( + ckey, + self, + timeout=3600 + ) + except TypeError: + # Some characters cause issues with caching; ignore and move on + pass + + @classmethod + def create_cache_key(cls, setting_key, **kwargs): + """Create a unique cache key for a particular setting object. + + The cache key uses the following elements to ensure the key is 'unique': + - The name of the class + - The unique KEY string + - Any key:value kwargs associated with the particular setting type (e.g. user-id) + """ + + key = f"{str(cls.__name__)}:{setting_key}" + + for k, v in kwargs.items(): + key += f"_{k}:{v}" + + return key + @classmethod def allValues(cls, user=None, exclude_hidden=False): """Return a dict of "all" defined global settings. @@ -220,11 +266,12 @@ class BaseInvenTreeSetting(models.Model): - Key is case-insensitive - Returns None if no match is made + + First checks the cache to see if this object has recently been accessed, + and returns the cached version if so. """ key = str(key).strip().upper() - settings = cls.objects.all() - filters = { 'key__iexact': key, } @@ -253,7 +300,25 @@ class BaseInvenTreeSetting(models.Model): if method is not None: filters['method'] = method + # Perform cache lookup by default + do_cache = kwargs.pop('cache', True) + + ckey = cls.create_cache_key(key, **kwargs) + + if do_cache: + try: + # First attempt to find the setting object in the cache + cached_setting = cache.get(ckey) + + if cached_setting is not None: + return cached_setting + + except AppRegistryNotReady: + # Cache is not ready yet + do_cache = False + try: + settings = cls.objects.all() setting = settings.filter(**filters).first() except (ValueError, cls.DoesNotExist): setting = None @@ -282,6 +347,10 @@ class BaseInvenTreeSetting(models.Model): # It might be the case that the database isn't created yet pass + if setting and do_cache: + # Cache this setting object + setting.save_to_cache() + return setting @classmethod @@ -1507,11 +1576,6 @@ class InvenTreeUserSetting(BaseInvenTreeSetting): help_text=_('User'), ) - @classmethod - def get_setting_object(cls, key, user=None): - """Return setting object for provided user.""" - return super().get_setting_object(key, user=user) - def validate_unique(self, exclude=None, **kwargs): """Return if the setting (including key) is unique.""" return super().validate_unique(exclude=exclude, user=self.user) diff --git a/InvenTree/common/settings.py b/InvenTree/common/settings.py index 3bf544e13e..3e9ce15566 100644 --- a/InvenTree/common/settings.py +++ b/InvenTree/common/settings.py @@ -12,7 +12,7 @@ def currency_code_default(): from common.models import InvenTreeSetting try: - code = InvenTreeSetting.get_setting('INVENTREE_DEFAULT_CURRENCY', create=False) + code = InvenTreeSetting.get_setting('INVENTREE_DEFAULT_CURRENCY', create=False, cache=False) except ProgrammingError: # pragma: no cover # database is not initialized yet code = '' diff --git a/InvenTree/common/tests.py b/InvenTree/common/tests.py index 8a087930c0..aba3f99377 100644 --- a/InvenTree/common/tests.py +++ b/InvenTree/common/tests.py @@ -4,6 +4,8 @@ import json from datetime import timedelta from http import HTTPStatus +from django.contrib.auth import get_user_model +from django.core.cache import cache from django.test import Client, TestCase from django.urls import reverse @@ -45,10 +47,10 @@ class SettingsTest(InvenTreeTestCase): """Test settings functions and properties.""" # define settings to check instance_ref = 'INVENTREE_INSTANCE' - instance_obj = InvenTreeSetting.get_setting_object(instance_ref) + instance_obj = InvenTreeSetting.get_setting_object(instance_ref, cache=False) stale_ref = 'STOCK_STALE_DAYS' - stale_days = InvenTreeSetting.get_setting_object(stale_ref) + stale_days = InvenTreeSetting.get_setting_object(stale_ref, cache=False) report_size_obj = InvenTreeSetting.get_setting_object('REPORT_DEFAULT_PAGE_SIZE') report_test_obj = InvenTreeSetting.get_setting_object('REPORT_ENABLE_TEST_REPORT') @@ -189,6 +191,56 @@ class SettingsTest(InvenTreeTestCase): if setting.default_value not in [True, False]: raise ValueError(f'Non-boolean default value specified for {key}') # pragma: no cover + def test_global_setting_caching(self): + """Test caching operations for the global settings class""" + + key = 'PART_NAME_FORMAT' + + cache_key = InvenTreeSetting.create_cache_key(key) + self.assertEqual(cache_key, 'InvenTreeSetting:PART_NAME_FORMAT') + + cache.clear() + + self.assertIsNone(cache.get(cache_key)) + + # First request should set cache + val = InvenTreeSetting.get_setting(key) + self.assertEqual(cache.get(cache_key).value, val) + + for val in ['A', '{{ part.IPN }}', 'C']: + # Check that the cached value is updated whenever the setting is saved + InvenTreeSetting.set_setting(key, val, None) + self.assertEqual(cache.get(cache_key).value, val) + self.assertEqual(InvenTreeSetting.get_setting(key), val) + + def test_user_setting_caching(self): + """Test caching operation for the user settings class""" + + cache.clear() + + # Generate a number of new usesr + for idx in range(5): + get_user_model().objects.create( + username=f"User_{idx}", + password="hunter42", + email="email@dot.com", + ) + + key = 'SEARCH_PREVIEW_RESULTS' + + # Check that the settings are correctly cached for each separate user + for user in get_user_model().objects.all(): + setting = InvenTreeUserSetting.get_setting_object(key, user=user) + cache_key = setting.cache_key + self.assertEqual(cache_key, f"InvenTreeUserSetting:SEARCH_PREVIEW_RESULTS_user:{user.username}") + InvenTreeUserSetting.set_setting(key, user.pk, None, user=user) + self.assertIsNotNone(cache.get(cache_key)) + + # Iterate through a second time, ensure the values have been cached correctly + for user in get_user_model().objects.all(): + value = InvenTreeUserSetting.get_setting(key, user=user) + self.assertEqual(value, user.pk) + class GlobalSettingsApiTest(InvenTreeAPITestCase): """Tests for the global settings API.""" @@ -199,7 +251,7 @@ class GlobalSettingsApiTest(InvenTreeAPITestCase): # Read out each of the global settings value, to ensure they are instantiated in the database for key in InvenTreeSetting.SETTINGS: - InvenTreeSetting.get_setting_object(key) + InvenTreeSetting.get_setting_object(key, cache=False) response = self.get(url, expected_code=200) @@ -422,7 +474,8 @@ class UserSettingsApiTest(InvenTreeAPITestCase): """Test a integer user setting value.""" setting = InvenTreeUserSetting.get_setting_object( 'SEARCH_PREVIEW_RESULTS', - user=self.user + user=self.user, + cache=False, ) url = reverse('api-user-setting-detail', kwargs={'key': setting.key}) diff --git a/InvenTree/part/test_part.py b/InvenTree/part/test_part.py index ac03baf0a5..30cd890744 100644 --- a/InvenTree/part/test_part.py +++ b/InvenTree/part/test_part.py @@ -3,6 +3,7 @@ import os from django.conf import settings +from django.core.cache import cache from django.core.exceptions import ValidationError from django.test import TestCase @@ -394,6 +395,9 @@ class PartSettingsTest(InvenTreeTestCase): def make_part(self): """Helper function to create a simple part.""" + + cache.clear() + part = Part.objects.create( name='Test Part', description='I am but a humble test part', @@ -404,6 +408,9 @@ class PartSettingsTest(InvenTreeTestCase): def test_defaults(self): """Test that the default values for the part settings are correct.""" + + cache.clear() + self.assertTrue(part.settings.part_component_default()) self.assertTrue(part.settings.part_purchaseable_default()) self.assertFalse(part.settings.part_salable_default()) @@ -411,6 +418,9 @@ class PartSettingsTest(InvenTreeTestCase): def test_initial(self): """Test the 'initial' default values (no default values have been set)""" + + cache.clear() + part = self.make_part() self.assertTrue(part.component) diff --git a/InvenTree/plugin/apps.py b/InvenTree/plugin/apps.py index 405b8cfcd2..9fdd43e6c6 100644 --- a/InvenTree/plugin/apps.py +++ b/InvenTree/plugin/apps.py @@ -36,7 +36,7 @@ class PluginAppConfig(AppConfig): # this is the first startup try: from common.models import InvenTreeSetting - if InvenTreeSetting.get_setting('PLUGIN_ON_STARTUP', create=False): + if InvenTreeSetting.get_setting('PLUGIN_ON_STARTUP', create=False, cache=False): # make sure all plugins are installed registry.install_plugin_file() except Exception: # pragma: no cover diff --git a/InvenTree/report/tests.py b/InvenTree/report/tests.py index 336f48d6fc..512a163d0d 100644 --- a/InvenTree/report/tests.py +++ b/InvenTree/report/tests.py @@ -204,7 +204,7 @@ class BuildReportTest(ReportTest): self.assertEqual(headers['Content-Disposition'], 'attachment; filename="report.pdf"') # Now, set the download type to be "inline" - inline = InvenTreeUserSetting.get_setting_object('REPORT_INLINE', self.user) + inline = InvenTreeUserSetting.get_setting_object('REPORT_INLINE', user=self.user) inline.value = True inline.save()