inventree/InvenTree
2
0
Fork 0
mirror of https://github.com/inventree/InvenTree.git synced 2025-07-01 03:00:54 +00:00
Code Activity

refactor (backend): move config files out of the source directories (#9769)

Browse Source 
* moving config files out of the source directories
Fixes #9756

* add folder for config

* fix lookup paths

* reorder ignores

* reduce diff

* better error message

* fix paths

* Update tests.py

* save global warning to db

* fix import

* more import fixes / docs

* fix default

* fix default

* ensure secret_key/get_config_file is tested fully

* try fixing path on docker

* try to make it work on GitHub CI

* refactor testfolder path into config var

* fix test path

* fix test

* do not test on docker

* more tests

* add testing for global warning dict

* fix error handling
This commit is contained in:
Matthias Mair
2025-06-25 00:12:24 +02:00
committed by GitHub
parent 03bb1eb709
commit bbe94ee9c2
14 changed files with 318 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
.gitignore vendored
View File

@ -46,19 +46,17 @@ inventree_media
inventree_static
static_i18n
# Local config file
# Local config files
config.yaml
plugins.txt
secret_key.txt
oidc.pem
# Default data file
data.json
*.json.tmp
*.tmp.json
# Key file
secret_key.txt
oidc.pem
# IDE / development files
.idea/
*.code-workspace

1
config/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
*

1
config/README Normal file
View File

@ -0,0 +1 @@
This folder is recommended for configuration values and excluded from change tracking in git

6
docs/docs/settings/error_codes.md
View File

@ -111,6 +111,12 @@ Steps very between deployment methods.
The command that was used to run invoke is not the one that is recommended. This might be caused by a wrong PATH variable or by thinking you are using a different deployment method.
The warning text will show the recommended command for intended use.
#### INVE-W10
**Config not in recommended directory - Backend**
A configuration file is not in the recommended directory. This might lead to issues with the deployment method you are using. It might also lead to confusinon.
The warning text will show the recommended directory for your deployment method.
#### INVE-W10
**Exception during mail delivery - Backend**

88
src/backend/InvenTree/InvenTree/config.py
View File

@ -8,6 +8,7 @@ import random
import shutil
import string
from pathlib import Path
from typing import Optional, Union
logger = logging.getLogger('inventree')
CONFIG_DATA = None
@ -68,6 +69,21 @@ def get_base_dir() -> Path:
return Path(__file__).parent.parent.resolve()
def get_root_dir() -> Path:
"""Returns the InvenTree root directory."""
return get_base_dir().parent.parent.parent
def get_config_dir() -> Path:
"""Returns the InvenTree configuration directory."""
return get_root_dir().joinpath('config').resolve()
def get_testfolder_dir() -> Path:
"""Returns the InvenTree test folder directory."""
return get_base_dir().joinpath('_testfolder').resolve()
def ensure_dir(path: Path, storage=None) -> None:
"""Ensure that a directory exists.
@ -90,15 +106,19 @@ def get_config_file(create=True) -> Path:
Note: It will be created it if does not already exist!
"""
conf_dir = get_config_dir()
base_dir = get_base_dir()
cfg_filename = os.getenv('INVENTREE_CONFIG_FILE')
if cfg_filename:
cfg_filename = Path(cfg_filename.strip()).resolve()
elif get_base_dir().joinpath('config.yaml').exists():
# If the config file is in the old directory, use that
cfg_filename = base_dir.joinpath('config.yaml').resolve()
else:
# Config file is *not* specified - use the default
cfg_filename = base_dir.joinpath('config.yaml').resolve()
cfg_filename = conf_dir.joinpath('config.yaml').resolve()
if not cfg_filename.exists() and create:
print(
@ -110,6 +130,7 @@ def get_config_file(create=True) -> Path:
shutil.copyfile(cfg_template, cfg_filename)
print(f'Created config file {cfg_filename}')
check_config_dir('INVENTREE_CONFIG_FILE', cfg_filename, conf_dir)
return cfg_filename
@ -291,7 +312,7 @@ def get_backup_dir(create=True, error=True):
return bd
def get_plugin_file():
def get_plugin_file() -> Path:
"""Returns the path of the InvenTree plugins specification file.
Note: It will be created if it does not already exist!
@ -319,6 +340,7 @@ def get_plugin_file():
'# InvenTree Plugins (uses PIP framework to install)\n\n'
)
check_config_dir('INVENTREE_PLUGIN_FILE', plugin_file)
return plugin_file
@ -327,7 +349,7 @@ def get_plugin_dir():
return get_setting('INVENTREE_PLUGIN_DIR', 'plugin_dir')
def get_secret_key():
def get_secret_key(return_path: bool = False) -> Union[str, Path]:
"""Return the secret key value which will be used by django.
Following options are tested, in descending order of preference:
@ -336,18 +358,24 @@ def get_secret_key():
B) Check for environment variable INVENTREE_SECRET_KEY_FILE => Load key data from file
C) Look for default key file "secret_key.txt"
D) Create "secret_key.txt" if it does not exist
Args:
return_path (bool): If True, return the path to the secret key file instead of the key data.
"""
# Look for environment variable
if secret_key := get_setting('INVENTREE_SECRET_KEY', 'secret_key'):
logger.info('SECRET_KEY loaded by INVENTREE_SECRET_KEY') # pragma: no cover
return secret_key
return str(secret_key)
# Look for secret key file
if secret_key_file := get_setting('INVENTREE_SECRET_KEY_FILE', 'secret_key_file'):
secret_key_file = Path(secret_key_file).resolve()
elif get_base_dir().joinpath('secret_key.txt').exists():
secret_key_file = get_base_dir().joinpath('secret_key.txt')
else:
# Default location for secret key file
secret_key_file = get_base_dir().joinpath('secret_key.txt').resolve()
secret_key_file = get_config_dir().joinpath('secret_key.txt').resolve()
check_config_dir('INVENTREE_SECRET_KEY_FILE', secret_key_file)
if not secret_key_file.exists():
logger.info("Generating random key file at '%s'", secret_key_file)
@ -358,14 +386,14 @@ def get_secret_key():
key = ''.join([random.choice(options) for _idx in range(100)])
secret_key_file.write_text(key)
if return_path:
return secret_key_file
logger.debug("Loading SECRET_KEY from '%s'", secret_key_file)
key_data = secret_key_file.read_text().strip()
return key_data
return secret_key_file.read_text().strip()
def get_oidc_private_key():
def get_oidc_private_key(return_path: bool = False) -> Union[str, Path]:
"""Return the private key for OIDC authentication.
Following options are tested, in descending order of preference:
@ -383,11 +411,19 @@ def get_oidc_private_key():
get_setting(
'INVENTREE_OIDC_PRIVATE_KEY_FILE',
'oidc_private_key_file',
get_base_dir().joinpath('oidc.pem'),
get_config_dir().joinpath('oidc.pem'),
)
)
# Trying old default location
if not key_loc.exists():
old_def_path = get_base_dir().joinpath('oidc.pem')
if old_def_path.exists():
key_loc = old_def_path.resolve()
check_config_dir('INVENTREE_OIDC_PRIVATE_KEY_FILE', key_loc)
if key_loc.exists():
return key_loc.read_text()
return key_loc.read_text() if not return_path else key_loc
else:
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric import rsa
@ -407,8 +443,7 @@ def get_oidc_private_key():
encryption_algorithm=serialization.NoEncryption(),
)
)
RSA_KEY = key_loc.read_text()
return RSA_KEY
return key_loc.read_text() if not return_path else key_loc
def get_custom_file(
@ -494,3 +529,28 @@ def get_frontend_settings(debug=True):
frontend_settings['url_compatibility'] = True
return frontend_settings
def check_config_dir(
setting_name: str, current_path: Path, config_dir: Optional[Path] = None
) -> None:
"""Warn if the config directory is not used."""
if not config_dir:
config_dir = get_config_dir()
if not current_path.is_relative_to(config_dir):
logger.warning(
"INVE-W10 - Config for '%s' not in recommended directory '%s'.",
setting_name,
config_dir,
)
try:
from common.settings import GlobalWarningCode, set_global_warning
set_global_warning(
GlobalWarningCode.UNCOMMON_CONFIG, {'path': str(config_dir)}
)
except ModuleNotFoundError: # pragma: no cover
pass
return

2
src/backend/InvenTree/InvenTree/settings.py
View File

@ -73,7 +73,7 @@ BASE_DIR = config.get_base_dir()
CONFIG = config.load_config_data(set_cache=True)
# Load VERSION data if it exists
version_file = BASE_DIR.parent.parent.parent.joinpath('VERSION')
version_file = config.get_root_dir().joinpath('VERSION')
if version_file.exists():
load_dotenv(version_file)

118
src/backend/InvenTree/InvenTree/tests.py
View File

@ -4,6 +4,7 @@ import os
import time
from datetime import datetime, timedelta
from decimal import Decimal
from pathlib import Path
from unittest import mock
from zoneinfo import ZoneInfo
@ -1199,39 +1200,128 @@ class TestSettings(InvenTreeTestCase):
"""Test get_config_file."""
# normal run - not configured
valid = ['inventree/config.yaml', 'inventree/data/config.yaml']
valid = ['config/config.yaml', 'inventree/data/config.yaml']
trgt_path = str(config.get_config_file()).lower()
self.assertTrue(
any(opt in str(config.get_config_file()).lower() for opt in valid)
any(opt in trgt_path for opt in valid), f'Path {trgt_path} not in {valid}'
)
# with env set
with in_env_context({
'INVENTREE_CONFIG_FILE': '_testfolder/my_special_conf.yaml'
}):
self.assertIn(
'inventree/_testfolder/my_special_conf.yaml',
str(config.get_config_file()).lower(),
test_file = config.get_testfolder_dir() / 'my_special_conf.yaml'
with in_env_context({'INVENTREE_CONFIG_FILE': str(test_file)}):
self.assertEqual(
str(test_file).lower(), str(config.get_config_file()).lower()
)
# LEGACY - old path
if settings.DOCKER: # pragma: no cover
# In Docker, the legacy path is not used
return
legacy_path = config.get_base_dir().joinpath('config.yaml')
assert not legacy_path.exists(), (
'Legacy config file does exist, stopping as a percaution!'
)
self.assertTrue(test_file.exists(), f'Test file {test_file} does not exist!')
test_file.rename(legacy_path)
self.assertIn(
'src/backend/inventree/config.yaml', str(config.get_config_file()).lower()
)
# Clean up again
legacy_path.unlink(missing_ok=True)
def test_helpers_plugin_file(self):
"""Test get_plugin_file."""
# normal run - not configured
valid = ['inventree/plugins.txt', 'inventree/data/plugins.txt']
valid = ['config/plugins.txt', 'inventree/data/plugins.txt']
trgt_path = str(config.get_plugin_file()).lower()
self.assertTrue(
any(opt in str(config.get_plugin_file()).lower() for opt in valid)
any(opt in trgt_path for opt in valid), f'Path {trgt_path} not in {valid}'
)
# with env set
with in_env_context({
'INVENTREE_PLUGIN_FILE': '_testfolder/my_special_plugins.txt'
}):
test_file = config.get_testfolder_dir() / 'my_special_plugins.txt'
with in_env_context({'INVENTREE_PLUGIN_FILE': str(test_file)}):
self.assertIn(str(test_file), str(config.get_plugin_file()))
def test_helpers_secret_key(self):
"""Test get_secret_key."""
# Normal file behavior - not configured
valid = ['config/secret_key.txt', 'inventree/data/secret_key.txt']
trgt_path = str(config.get_secret_key(return_path=True)).lower()
self.assertTrue(
any(opt in trgt_path for opt in valid), f'Path {trgt_path} not in {valid}'
)
# with env set
test_file = config.get_testfolder_dir() / 'my_secret_test.txt'
with in_env_context({'INVENTREE_SECRET_KEY_FILE': str(test_file)}):
self.assertIn(str(test_file), str(config.get_secret_key(return_path=True)))
# LEGACY - old path
if settings.DOCKER: # pragma: no cover
# In Docker, the legacy path is not used
return
legacy_path = config.get_base_dir().joinpath('secret_key.txt')
assert not legacy_path.exists(), (
'Legacy secret key file does exist, stopping as a percaution!'
)
test_file.rename(legacy_path)
self.assertIn(
'src/backend/inventree/secret_key.txt',
str(config.get_secret_key(return_path=True)).lower(),
)
# Clean up again
legacy_path.unlink(missing_ok=True)
# Test with content set per environment
with in_env_context({'INVENTREE_SECRET_KEY': '123abc123'}):
self.assertEqual(config.get_secret_key(), '123abc123')
def test_helpers_get_oidc_private_key(self):
"""Test get_oidc_private_key."""
# Normal file behavior - not configured
valid = ['config/oidc.pem', 'inventree/data/oidc.pem']
trgt_path = config.get_oidc_private_key(return_path=True)
self.assertTrue(
any(opt in str(trgt_path) for opt in valid),
f'Path {trgt_path} not in {valid}',
)
# with env set
test_file = config.get_testfolder_dir() / 'my_oidc_private_key.pem'
with in_env_context({'INVENTREE_OIDC_PRIVATE_KEY_FILE': str(test_file)}):
self.assertIn(
'_testfolder/my_special_plugins.txt', str(config.get_plugin_file())
str(test_file), str(config.get_oidc_private_key(return_path=True))
)
# Override with environment variable
with in_env_context({'INVENTREE_OIDC_PRIVATE_KEY': '123abc123'}):
self.assertEqual(config.get_oidc_private_key(), '123abc123')
# LEGACY - old path
if settings.DOCKER: # pragma: no cover
# In Docker, the legacy path is not used
return
legacy_path = config.get_base_dir().joinpath('oidc.pem')
assert not legacy_path.exists(), (
'Legacy OIDC private key file does exist, stopping as a precaution!'
)
test_file.rename(legacy_path)
assert isinstance(trgt_path, Path)
new_path = trgt_path.rename(
trgt_path.parent / '_oidc.pem'
) # move out current config
self.assertIn(
'src/backend/inventree/oidc.pem',
str(config.get_oidc_private_key(return_path=True)).lower(),
)
# Clean up again
legacy_path.unlink(missing_ok=True)
new_path.rename(trgt_path) # restore original path for current config
def test_helpers_setting(self):
"""Test get_setting."""
TEST_ENV_NAME = '123TEST'

13
src/backend/InvenTree/common/setting/system.py
View File

@ -162,6 +162,12 @@ class BaseURLValidator(URLValidator):
super().__call__(value)
class SystemSetId:
"""Shared system settings identifiers."""
GLOBAL_WARNING = '_GLOBAL_WARNING'
SYSTEM_SETTINGS: dict[str, InvenTreeSettingsKeyType] = {
'SERVER_RESTART_REQUIRED': {
'name': _('Restart required'),
@ -176,6 +182,13 @@ SYSTEM_SETTINGS: dict[str, InvenTreeSettingsKeyType] = {
'default': 0,
'validator': int,
},
SystemSetId.GLOBAL_WARNING: {
'name': _('Active warning codes'),
'description': _('A dict of active warning codes'),
'validator': json.loads,
'default': '{}',
'hidden': True,
},
'INVENTREE_INSTANCE_ID': {
'name': _('Instance ID'),
'description': _('Unique identifier for this InvenTree instance'),

67
src/backend/InvenTree/common/settings.py
View File

@ -1,6 +1,16 @@
"""User-configurable settings for the common app."""
import json
from os import environ
from typing import Optional
from django.core.exceptions import AppRegistryNotReady
import structlog
import InvenTree.ready
logger = structlog.get_logger('inventree')
def global_setting_overrides() -> dict:
@ -41,6 +51,63 @@ def set_global_setting(key, value, change_user=None, create=True, **kwargs):
return InvenTreeSetting.set_setting(key, value, **kwargs)
class GlobalWarningCode:
"""Warning codes that reflect to the status of the instance."""
UNCOMMON_CONFIG = 'INVE-W10'
TEST_KEY = '_TEST'
def set_global_warning(key: str, options: Optional[dict] = None) -> bool:
"""Set a global warning for a code.
Args:
key (str): The key for the warning.
options (dict or bool): Options for the warning, or True to set a default warning.
Raises:
ValueError: If the key is not provided.
Returns:
bool: True if the warning was checked / set successfully, False if no check was performed.
"""
if not key:
raise ValueError('Key must be provided for global warning setting.')
# Ensure DB is ready
if not InvenTree.ready.canAppAccessDatabase(allow_test=True):
return False
try:
from common.models import InvenTreeSetting
from common.setting.system import SystemSetId
except AppRegistryNotReady:
# App registry not ready, cannot set global warning
return False
# Get and write (if necessary) the current global settings warning
global_dict = get_global_setting(SystemSetId.GLOBAL_WARNING, '{}', create=False)
try:
global_dict = json.loads(global_dict)
except json.JSONDecodeError:
global_dict = {}
if global_dict is None or not isinstance(global_dict, dict):
global_dict = {}
if key not in global_dict or global_dict[key] != options:
global_dict[key] = options if options is not None else True
try:
global_dict_val = json.dumps(global_dict)
except TypeError:
# If the options cannot be serialized, we will set an empty the warning
global_dict_val = 'true'
logger.warning(
f'Failed to serialize global warning options for key "{key}". Setting to True.'
)
InvenTreeSetting.set_setting(
SystemSetId.GLOBAL_WARNING, global_dict_val, change_user=None, create=True
)
return True
def stock_expiry_enabled():
"""Returns True if the stock expiry feature is enabled."""
from common.models import InvenTreeSetting

38
src/backend/InvenTree/common/tests.py
View File

@ -509,6 +509,44 @@ class SettingsTest(InvenTreeTestCase):
value = InvenTreeUserSetting.get_setting(key, user=user)
self.assertEqual(value, user.pk)
def test_set_global_warning(self):
"""Test set_global_warning function."""
from common.setting.system import SystemSetId
from common.settings import GlobalWarningCode, set_global_warning
# Set a warning
self.assertTrue(
set_global_warning(GlobalWarningCode.TEST_KEY, {'test': 'value'})
)
# Check that the warning has been set
self.assertIn(
GlobalWarningCode.TEST_KEY,
InvenTreeSetting.get_setting(SystemSetId.GLOBAL_WARNING),
)
# Check that the warning can be retrieved
warning = InvenTreeSetting.get_setting_object(SystemSetId.GLOBAL_WARNING)
warning_dict = json.loads(warning.value)
self.assertEqual(warning_dict[GlobalWarningCode.TEST_KEY], {'test': 'value'})
# Clear the warning
self.assertTrue(set_global_warning(GlobalWarningCode.TEST_KEY, False))
# Check that the warning has been cleared
self.assertFalse(
json.loads(InvenTreeSetting.get_setting(SystemSetId.GLOBAL_WARNING)).get(
GlobalWarningCode.TEST_KEY
)
)
# No key - warning
with self.assertRaises(ValueError):
set_global_warning(None)
# Wrong structure
self.assertTrue(set_global_warning(GlobalWarningCode.TEST_KEY, {'test': json}))
class GlobalSettingsApiTest(InvenTreeAPITestCase):
"""Tests for the global settings API."""

8
src/backend/InvenTree/part/test_api.py
View File

@ -20,7 +20,7 @@ import order.models
from build.status_codes import BuildStatus
from common.models import InvenTreeSetting
from company.models import Company, SupplierPart
from InvenTree.settings import BASE_DIR
from InvenTree.config import get_testfolder_dir
from InvenTree.unit_test import InvenTreeAPITestCase
from order.status_codes import PurchaseOrderStatusGroups
from part.models import (
@ -64,7 +64,7 @@ class PartImageTestMixin:
"""Create a test image file."""
p = Part.objects.first()
fn = BASE_DIR / '_testfolder' / 'part_image_123abc.png'
fn = get_testfolder_dir() / 'part_image_123abc.png'
img = PIL.Image.new('RGB', (128, 128), color='blue')
img.save(fn)
@ -1694,7 +1694,7 @@ class PartDetailTests(PartImageTestMixin, PartAPITestBase):
print(p.image.file)
# Try to upload a non-image file
test_path = BASE_DIR / '_testfolder' / 'dummy_image'
test_path = get_testfolder_dir() / 'dummy_image'
with open(f'{test_path}.txt', 'w', encoding='utf-8') as dummy_image:
dummy_image.write('hello world')
@ -1757,7 +1757,7 @@ class PartDetailTests(PartImageTestMixin, PartAPITestBase):
# First, upload an image for an existing part
p = Part.objects.first()
fn = BASE_DIR / '_testfolder' / 'part_image_123abc.png'
fn = get_testfolder_dir() / 'part_image_123abc.png'
img = PIL.Image.new('RGB', (128, 128), color='blue')
img.save(fn)

4
src/backend/InvenTree/plugin/base/label/test_label_mixin.py
View File

@ -10,7 +10,7 @@ from django.urls import reverse
from pdfminer.high_level import extract_text
from PIL import Image
from InvenTree.settings import BASE_DIR
from InvenTree.config import get_testfolder_dir
from InvenTree.unit_test import InvenTreeAPITestCase
from part.models import Part
from plugin import InvenTreePlugin, PluginMixinEnum, registry
@ -90,7 +90,7 @@ class LabelMixinTests(PrintTestMixins, InvenTreeAPITestCase):
apps.get_app_config('report').create_default_labels()
apps.get_app_config('report').create_default_reports()
test_path = BASE_DIR / '_testfolder' / 'label'
test_path = get_testfolder_dir() / 'label'
parts = Part.objects.all()[:2]

4
src/backend/InvenTree/plugin/samples/integration/label_sample.py
View File

@ -5,7 +5,7 @@ This does not function in real usage and is more to show the required components
from rest_framework import serializers
from InvenTree.settings import BASE_DIR
from InvenTree.config import get_testfolder_dir
from plugin import InvenTreePlugin
from plugin.mixins import LabelPrintingMixin
@ -38,7 +38,7 @@ class SampleLabelPrinter(LabelPrintingMixin, InvenTreePlugin):
kwargs['label_instance'], kwargs['item_instance'], **kwargs
)
filename = str(BASE_DIR / '_testfolder' / 'label.pdf')
filename = str(get_testfolder_dir() / 'label.pdf')
# Dump the PDF to a local file
with open(filename, 'wb') as pdf_out:

3
src/backend/InvenTree/report/test_tags.py
View File

@ -12,6 +12,7 @@ from djmoney.money import Money
from PIL import Image
from common.models import InvenTreeSetting
from InvenTree.config import get_testfolder_dir
from InvenTree.unit_test import InvenTreeTestCase
from part.models import Part, PartParameter, PartParameterTemplate
from part.test_api import PartImageTestMixin
@ -309,7 +310,7 @@ class ReportTagTest(PartImageTestMixin, InvenTreeTestCase):
def test_encode_svg_image(self):
"""Test the encode_svg_image template tag."""
# Generate smallest possible SVG for testing
svg_path = settings.BASE_DIR / '_testfolder' / 'part_image_123abc.png'
svg_path = get_testfolder_dir() / 'part_image_123abc.png'
with open(svg_path, 'w', encoding='utf8') as f:
f.write('<svg xmlns="http://www.w3.org/2000/svg>')