2
0
mirror of https://github.com/inventree/InvenTree.git synced 2025-07-01 03:00:54 +00:00

refactor (backend): move config files out of the source directories (#9769)

* moving config files out of the source directories
Fixes #9756

* add folder for config

* fix lookup paths

* reorder ignores

* reduce diff

* better error message

* fix paths

* Update tests.py

* save global warning to db

* fix import

* more import fixes / docs

* fix default

* fix default

* ensure secret_key/get_config_file is tested fully

* try fixing path on docker

* try to make it work on GitHub CI

* refactor testfolder path into config var

* fix test path

* fix test

* do not test on docker

* more tests

* add testing for global warning dict

* fix error handling
This commit is contained in:
Matthias Mair
2025-06-25 00:12:24 +02:00
committed by GitHub
parent 03bb1eb709
commit bbe94ee9c2
14 changed files with 318 additions and 43 deletions

View File

@ -8,6 +8,7 @@ import random
import shutil
import string
from pathlib import Path
from typing import Optional, Union
logger = logging.getLogger('inventree')
CONFIG_DATA = None
@ -68,6 +69,21 @@ def get_base_dir() -> Path:
return Path(__file__).parent.parent.resolve()
def get_root_dir() -> Path:
"""Returns the InvenTree root directory."""
return get_base_dir().parent.parent.parent
def get_config_dir() -> Path:
"""Returns the InvenTree configuration directory."""
return get_root_dir().joinpath('config').resolve()
def get_testfolder_dir() -> Path:
"""Returns the InvenTree test folder directory."""
return get_base_dir().joinpath('_testfolder').resolve()
def ensure_dir(path: Path, storage=None) -> None:
"""Ensure that a directory exists.
@ -90,15 +106,19 @@ def get_config_file(create=True) -> Path:
Note: It will be created it if does not already exist!
"""
conf_dir = get_config_dir()
base_dir = get_base_dir()
cfg_filename = os.getenv('INVENTREE_CONFIG_FILE')
if cfg_filename:
cfg_filename = Path(cfg_filename.strip()).resolve()
elif get_base_dir().joinpath('config.yaml').exists():
# If the config file is in the old directory, use that
cfg_filename = base_dir.joinpath('config.yaml').resolve()
else:
# Config file is *not* specified - use the default
cfg_filename = base_dir.joinpath('config.yaml').resolve()
cfg_filename = conf_dir.joinpath('config.yaml').resolve()
if not cfg_filename.exists() and create:
print(
@ -110,6 +130,7 @@ def get_config_file(create=True) -> Path:
shutil.copyfile(cfg_template, cfg_filename)
print(f'Created config file {cfg_filename}')
check_config_dir('INVENTREE_CONFIG_FILE', cfg_filename, conf_dir)
return cfg_filename
@ -291,7 +312,7 @@ def get_backup_dir(create=True, error=True):
return bd
def get_plugin_file():
def get_plugin_file() -> Path:
"""Returns the path of the InvenTree plugins specification file.
Note: It will be created if it does not already exist!
@ -319,6 +340,7 @@ def get_plugin_file():
'# InvenTree Plugins (uses PIP framework to install)\n\n'
)
check_config_dir('INVENTREE_PLUGIN_FILE', plugin_file)
return plugin_file
@ -327,7 +349,7 @@ def get_plugin_dir():
return get_setting('INVENTREE_PLUGIN_DIR', 'plugin_dir')
def get_secret_key():
def get_secret_key(return_path: bool = False) -> Union[str, Path]:
"""Return the secret key value which will be used by django.
Following options are tested, in descending order of preference:
@ -336,18 +358,24 @@ def get_secret_key():
B) Check for environment variable INVENTREE_SECRET_KEY_FILE => Load key data from file
C) Look for default key file "secret_key.txt"
D) Create "secret_key.txt" if it does not exist
Args:
return_path (bool): If True, return the path to the secret key file instead of the key data.
"""
# Look for environment variable
if secret_key := get_setting('INVENTREE_SECRET_KEY', 'secret_key'):
logger.info('SECRET_KEY loaded by INVENTREE_SECRET_KEY') # pragma: no cover
return secret_key
return str(secret_key)
# Look for secret key file
if secret_key_file := get_setting('INVENTREE_SECRET_KEY_FILE', 'secret_key_file'):
secret_key_file = Path(secret_key_file).resolve()
elif get_base_dir().joinpath('secret_key.txt').exists():
secret_key_file = get_base_dir().joinpath('secret_key.txt')
else:
# Default location for secret key file
secret_key_file = get_base_dir().joinpath('secret_key.txt').resolve()
secret_key_file = get_config_dir().joinpath('secret_key.txt').resolve()
check_config_dir('INVENTREE_SECRET_KEY_FILE', secret_key_file)
if not secret_key_file.exists():
logger.info("Generating random key file at '%s'", secret_key_file)
@ -358,14 +386,14 @@ def get_secret_key():
key = ''.join([random.choice(options) for _idx in range(100)])
secret_key_file.write_text(key)
if return_path:
return secret_key_file
logger.debug("Loading SECRET_KEY from '%s'", secret_key_file)
key_data = secret_key_file.read_text().strip()
return key_data
return secret_key_file.read_text().strip()
def get_oidc_private_key():
def get_oidc_private_key(return_path: bool = False) -> Union[str, Path]:
"""Return the private key for OIDC authentication.
Following options are tested, in descending order of preference:
@ -383,11 +411,19 @@ def get_oidc_private_key():
get_setting(
'INVENTREE_OIDC_PRIVATE_KEY_FILE',
'oidc_private_key_file',
get_base_dir().joinpath('oidc.pem'),
get_config_dir().joinpath('oidc.pem'),
)
)
# Trying old default location
if not key_loc.exists():
old_def_path = get_base_dir().joinpath('oidc.pem')
if old_def_path.exists():
key_loc = old_def_path.resolve()
check_config_dir('INVENTREE_OIDC_PRIVATE_KEY_FILE', key_loc)
if key_loc.exists():
return key_loc.read_text()
return key_loc.read_text() if not return_path else key_loc
else:
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric import rsa
@ -407,8 +443,7 @@ def get_oidc_private_key():
encryption_algorithm=serialization.NoEncryption(),
)
)
RSA_KEY = key_loc.read_text()
return RSA_KEY
return key_loc.read_text() if not return_path else key_loc
def get_custom_file(
@ -494,3 +529,28 @@ def get_frontend_settings(debug=True):
frontend_settings['url_compatibility'] = True
return frontend_settings
def check_config_dir(
setting_name: str, current_path: Path, config_dir: Optional[Path] = None
) -> None:
"""Warn if the config directory is not used."""
if not config_dir:
config_dir = get_config_dir()
if not current_path.is_relative_to(config_dir):
logger.warning(
"INVE-W10 - Config for '%s' not in recommended directory '%s'.",
setting_name,
config_dir,
)
try:
from common.settings import GlobalWarningCode, set_global_warning
set_global_warning(
GlobalWarningCode.UNCOMMON_CONFIG, {'path': str(config_dir)}
)
except ModuleNotFoundError: # pragma: no cover
pass
return

View File

@ -73,7 +73,7 @@ BASE_DIR = config.get_base_dir()
CONFIG = config.load_config_data(set_cache=True)
# Load VERSION data if it exists
version_file = BASE_DIR.parent.parent.parent.joinpath('VERSION')
version_file = config.get_root_dir().joinpath('VERSION')
if version_file.exists():
load_dotenv(version_file)

View File

@ -4,6 +4,7 @@ import os
import time
from datetime import datetime, timedelta
from decimal import Decimal
from pathlib import Path
from unittest import mock
from zoneinfo import ZoneInfo
@ -1199,39 +1200,128 @@ class TestSettings(InvenTreeTestCase):
"""Test get_config_file."""
# normal run - not configured
valid = ['inventree/config.yaml', 'inventree/data/config.yaml']
valid = ['config/config.yaml', 'inventree/data/config.yaml']
trgt_path = str(config.get_config_file()).lower()
self.assertTrue(
any(opt in str(config.get_config_file()).lower() for opt in valid)
any(opt in trgt_path for opt in valid), f'Path {trgt_path} not in {valid}'
)
# with env set
with in_env_context({
'INVENTREE_CONFIG_FILE': '_testfolder/my_special_conf.yaml'
}):
self.assertIn(
'inventree/_testfolder/my_special_conf.yaml',
str(config.get_config_file()).lower(),
test_file = config.get_testfolder_dir() / 'my_special_conf.yaml'
with in_env_context({'INVENTREE_CONFIG_FILE': str(test_file)}):
self.assertEqual(
str(test_file).lower(), str(config.get_config_file()).lower()
)
# LEGACY - old path
if settings.DOCKER: # pragma: no cover
# In Docker, the legacy path is not used
return
legacy_path = config.get_base_dir().joinpath('config.yaml')
assert not legacy_path.exists(), (
'Legacy config file does exist, stopping as a percaution!'
)
self.assertTrue(test_file.exists(), f'Test file {test_file} does not exist!')
test_file.rename(legacy_path)
self.assertIn(
'src/backend/inventree/config.yaml', str(config.get_config_file()).lower()
)
# Clean up again
legacy_path.unlink(missing_ok=True)
def test_helpers_plugin_file(self):
"""Test get_plugin_file."""
# normal run - not configured
valid = ['inventree/plugins.txt', 'inventree/data/plugins.txt']
valid = ['config/plugins.txt', 'inventree/data/plugins.txt']
trgt_path = str(config.get_plugin_file()).lower()
self.assertTrue(
any(opt in str(config.get_plugin_file()).lower() for opt in valid)
any(opt in trgt_path for opt in valid), f'Path {trgt_path} not in {valid}'
)
# with env set
with in_env_context({
'INVENTREE_PLUGIN_FILE': '_testfolder/my_special_plugins.txt'
}):
test_file = config.get_testfolder_dir() / 'my_special_plugins.txt'
with in_env_context({'INVENTREE_PLUGIN_FILE': str(test_file)}):
self.assertIn(str(test_file), str(config.get_plugin_file()))
def test_helpers_secret_key(self):
"""Test get_secret_key."""
# Normal file behavior - not configured
valid = ['config/secret_key.txt', 'inventree/data/secret_key.txt']
trgt_path = str(config.get_secret_key(return_path=True)).lower()
self.assertTrue(
any(opt in trgt_path for opt in valid), f'Path {trgt_path} not in {valid}'
)
# with env set
test_file = config.get_testfolder_dir() / 'my_secret_test.txt'
with in_env_context({'INVENTREE_SECRET_KEY_FILE': str(test_file)}):
self.assertIn(str(test_file), str(config.get_secret_key(return_path=True)))
# LEGACY - old path
if settings.DOCKER: # pragma: no cover
# In Docker, the legacy path is not used
return
legacy_path = config.get_base_dir().joinpath('secret_key.txt')
assert not legacy_path.exists(), (
'Legacy secret key file does exist, stopping as a percaution!'
)
test_file.rename(legacy_path)
self.assertIn(
'src/backend/inventree/secret_key.txt',
str(config.get_secret_key(return_path=True)).lower(),
)
# Clean up again
legacy_path.unlink(missing_ok=True)
# Test with content set per environment
with in_env_context({'INVENTREE_SECRET_KEY': '123abc123'}):
self.assertEqual(config.get_secret_key(), '123abc123')
def test_helpers_get_oidc_private_key(self):
"""Test get_oidc_private_key."""
# Normal file behavior - not configured
valid = ['config/oidc.pem', 'inventree/data/oidc.pem']
trgt_path = config.get_oidc_private_key(return_path=True)
self.assertTrue(
any(opt in str(trgt_path) for opt in valid),
f'Path {trgt_path} not in {valid}',
)
# with env set
test_file = config.get_testfolder_dir() / 'my_oidc_private_key.pem'
with in_env_context({'INVENTREE_OIDC_PRIVATE_KEY_FILE': str(test_file)}):
self.assertIn(
'_testfolder/my_special_plugins.txt', str(config.get_plugin_file())
str(test_file), str(config.get_oidc_private_key(return_path=True))
)
# Override with environment variable
with in_env_context({'INVENTREE_OIDC_PRIVATE_KEY': '123abc123'}):
self.assertEqual(config.get_oidc_private_key(), '123abc123')
# LEGACY - old path
if settings.DOCKER: # pragma: no cover
# In Docker, the legacy path is not used
return
legacy_path = config.get_base_dir().joinpath('oidc.pem')
assert not legacy_path.exists(), (
'Legacy OIDC private key file does exist, stopping as a precaution!'
)
test_file.rename(legacy_path)
assert isinstance(trgt_path, Path)
new_path = trgt_path.rename(
trgt_path.parent / '_oidc.pem'
) # move out current config
self.assertIn(
'src/backend/inventree/oidc.pem',
str(config.get_oidc_private_key(return_path=True)).lower(),
)
# Clean up again
legacy_path.unlink(missing_ok=True)
new_path.rename(trgt_path) # restore original path for current config
def test_helpers_setting(self):
"""Test get_setting."""
TEST_ENV_NAME = '123TEST'

View File

@ -162,6 +162,12 @@ class BaseURLValidator(URLValidator):
super().__call__(value)
class SystemSetId:
"""Shared system settings identifiers."""
GLOBAL_WARNING = '_GLOBAL_WARNING'
SYSTEM_SETTINGS: dict[str, InvenTreeSettingsKeyType] = {
'SERVER_RESTART_REQUIRED': {
'name': _('Restart required'),
@ -176,6 +182,13 @@ SYSTEM_SETTINGS: dict[str, InvenTreeSettingsKeyType] = {
'default': 0,
'validator': int,
},
SystemSetId.GLOBAL_WARNING: {
'name': _('Active warning codes'),
'description': _('A dict of active warning codes'),
'validator': json.loads,
'default': '{}',
'hidden': True,
},
'INVENTREE_INSTANCE_ID': {
'name': _('Instance ID'),
'description': _('Unique identifier for this InvenTree instance'),

View File

@ -1,6 +1,16 @@
"""User-configurable settings for the common app."""
import json
from os import environ
from typing import Optional
from django.core.exceptions import AppRegistryNotReady
import structlog
import InvenTree.ready
logger = structlog.get_logger('inventree')
def global_setting_overrides() -> dict:
@ -41,6 +51,63 @@ def set_global_setting(key, value, change_user=None, create=True, **kwargs):
return InvenTreeSetting.set_setting(key, value, **kwargs)
class GlobalWarningCode:
"""Warning codes that reflect to the status of the instance."""
UNCOMMON_CONFIG = 'INVE-W10'
TEST_KEY = '_TEST'
def set_global_warning(key: str, options: Optional[dict] = None) -> bool:
"""Set a global warning for a code.
Args:
key (str): The key for the warning.
options (dict or bool): Options for the warning, or True to set a default warning.
Raises:
ValueError: If the key is not provided.
Returns:
bool: True if the warning was checked / set successfully, False if no check was performed.
"""
if not key:
raise ValueError('Key must be provided for global warning setting.')
# Ensure DB is ready
if not InvenTree.ready.canAppAccessDatabase(allow_test=True):
return False
try:
from common.models import InvenTreeSetting
from common.setting.system import SystemSetId
except AppRegistryNotReady:
# App registry not ready, cannot set global warning
return False
# Get and write (if necessary) the current global settings warning
global_dict = get_global_setting(SystemSetId.GLOBAL_WARNING, '{}', create=False)
try:
global_dict = json.loads(global_dict)
except json.JSONDecodeError:
global_dict = {}
if global_dict is None or not isinstance(global_dict, dict):
global_dict = {}
if key not in global_dict or global_dict[key] != options:
global_dict[key] = options if options is not None else True
try:
global_dict_val = json.dumps(global_dict)
except TypeError:
# If the options cannot be serialized, we will set an empty the warning
global_dict_val = 'true'
logger.warning(
f'Failed to serialize global warning options for key "{key}". Setting to True.'
)
InvenTreeSetting.set_setting(
SystemSetId.GLOBAL_WARNING, global_dict_val, change_user=None, create=True
)
return True
def stock_expiry_enabled():
"""Returns True if the stock expiry feature is enabled."""
from common.models import InvenTreeSetting

View File

@ -509,6 +509,44 @@ class SettingsTest(InvenTreeTestCase):
value = InvenTreeUserSetting.get_setting(key, user=user)
self.assertEqual(value, user.pk)
def test_set_global_warning(self):
"""Test set_global_warning function."""
from common.setting.system import SystemSetId
from common.settings import GlobalWarningCode, set_global_warning
# Set a warning
self.assertTrue(
set_global_warning(GlobalWarningCode.TEST_KEY, {'test': 'value'})
)
# Check that the warning has been set
self.assertIn(
GlobalWarningCode.TEST_KEY,
InvenTreeSetting.get_setting(SystemSetId.GLOBAL_WARNING),
)
# Check that the warning can be retrieved
warning = InvenTreeSetting.get_setting_object(SystemSetId.GLOBAL_WARNING)
warning_dict = json.loads(warning.value)
self.assertEqual(warning_dict[GlobalWarningCode.TEST_KEY], {'test': 'value'})
# Clear the warning
self.assertTrue(set_global_warning(GlobalWarningCode.TEST_KEY, False))
# Check that the warning has been cleared
self.assertFalse(
json.loads(InvenTreeSetting.get_setting(SystemSetId.GLOBAL_WARNING)).get(
GlobalWarningCode.TEST_KEY
)
)
# No key - warning
with self.assertRaises(ValueError):
set_global_warning(None)
# Wrong structure
self.assertTrue(set_global_warning(GlobalWarningCode.TEST_KEY, {'test': json}))
class GlobalSettingsApiTest(InvenTreeAPITestCase):
"""Tests for the global settings API."""

View File

@ -20,7 +20,7 @@ import order.models
from build.status_codes import BuildStatus
from common.models import InvenTreeSetting
from company.models import Company, SupplierPart
from InvenTree.settings import BASE_DIR
from InvenTree.config import get_testfolder_dir
from InvenTree.unit_test import InvenTreeAPITestCase
from order.status_codes import PurchaseOrderStatusGroups
from part.models import (
@ -64,7 +64,7 @@ class PartImageTestMixin:
"""Create a test image file."""
p = Part.objects.first()
fn = BASE_DIR / '_testfolder' / 'part_image_123abc.png'
fn = get_testfolder_dir() / 'part_image_123abc.png'
img = PIL.Image.new('RGB', (128, 128), color='blue')
img.save(fn)
@ -1694,7 +1694,7 @@ class PartDetailTests(PartImageTestMixin, PartAPITestBase):
print(p.image.file)
# Try to upload a non-image file
test_path = BASE_DIR / '_testfolder' / 'dummy_image'
test_path = get_testfolder_dir() / 'dummy_image'
with open(f'{test_path}.txt', 'w', encoding='utf-8') as dummy_image:
dummy_image.write('hello world')
@ -1757,7 +1757,7 @@ class PartDetailTests(PartImageTestMixin, PartAPITestBase):
# First, upload an image for an existing part
p = Part.objects.first()
fn = BASE_DIR / '_testfolder' / 'part_image_123abc.png'
fn = get_testfolder_dir() / 'part_image_123abc.png'
img = PIL.Image.new('RGB', (128, 128), color='blue')
img.save(fn)

View File

@ -10,7 +10,7 @@ from django.urls import reverse
from pdfminer.high_level import extract_text
from PIL import Image
from InvenTree.settings import BASE_DIR
from InvenTree.config import get_testfolder_dir
from InvenTree.unit_test import InvenTreeAPITestCase
from part.models import Part
from plugin import InvenTreePlugin, PluginMixinEnum, registry
@ -90,7 +90,7 @@ class LabelMixinTests(PrintTestMixins, InvenTreeAPITestCase):
apps.get_app_config('report').create_default_labels()
apps.get_app_config('report').create_default_reports()
test_path = BASE_DIR / '_testfolder' / 'label'
test_path = get_testfolder_dir() / 'label'
parts = Part.objects.all()[:2]

View File

@ -5,7 +5,7 @@ This does not function in real usage and is more to show the required components
from rest_framework import serializers
from InvenTree.settings import BASE_DIR
from InvenTree.config import get_testfolder_dir
from plugin import InvenTreePlugin
from plugin.mixins import LabelPrintingMixin
@ -38,7 +38,7 @@ class SampleLabelPrinter(LabelPrintingMixin, InvenTreePlugin):
kwargs['label_instance'], kwargs['item_instance'], **kwargs
)
filename = str(BASE_DIR / '_testfolder' / 'label.pdf')
filename = str(get_testfolder_dir() / 'label.pdf')
# Dump the PDF to a local file
with open(filename, 'wb') as pdf_out:

View File

@ -12,6 +12,7 @@ from djmoney.money import Money
from PIL import Image
from common.models import InvenTreeSetting
from InvenTree.config import get_testfolder_dir
from InvenTree.unit_test import InvenTreeTestCase
from part.models import Part, PartParameter, PartParameterTemplate
from part.test_api import PartImageTestMixin
@ -309,7 +310,7 @@ class ReportTagTest(PartImageTestMixin, InvenTreeTestCase):
def test_encode_svg_image(self):
"""Test the encode_svg_image template tag."""
# Generate smallest possible SVG for testing
svg_path = settings.BASE_DIR / '_testfolder' / 'part_image_123abc.png'
svg_path = get_testfolder_dir() / 'part_image_123abc.png'
with open(svg_path, 'w', encoding='utf8') as f:
f.write('<svg xmlns="http://www.w3.org/2000/svg>')