Update demo data hook to copy media files (#3441)

* Simplify settings.py / config.py

- get_setting function has been streamlined
- move some functions into config.py

* Spelling fix: IGNORRED is IGNORED

* Ensure yaml is installed as part of docker image

- invoke path is still mucking us around

* Fix broken migration

* Copy media files from demo dataset when installing test data

* Cleanup settings.py

* Fix for configuration file traversal

* Line fix

* Update quickstart guide for docker

* Allow plugin file and plugin dir to be specified in configuration file

* Cleanup config template file

* Allow secret_key information to be provided in configuration file

* Adjust root paths for CI tests

* resolve paths

* Revert paths for CI step

* remove dead code

* Revert configuration variables to old names

- Prevent breaking changes

* Simplify secret key generation

* Fix default timeout for background worker process

* Revert change for customization options

.gitignore

@@ -50,7 +50,6 @@ docs/_build
 inventree_media
 inventree_static
 static_i18n
-inventree-data
 
 # Local config file
 config.yaml
@@ -79,6 +78,8 @@ js_tmp/
 
 # Development files
 dev/
+data/
+env/
 
 # Locale stats file
 locale_stats.json

CONTRIBUTING.md

@@ -18,6 +18,7 @@ pip install invoke && invoke setup-dev --tests
 
 ```bash
 git clone https://github.com/inventree/InvenTree.git && cd InvenTree
+docker compose run inventree-dev-server invoke install
 docker compose run inventree-dev-server invoke setup-test
 docker compose up -d
 ```

InvenTree/InvenTree/apps.py

@@ -172,21 +172,13 @@ class InvenTreeConfig(AppConfig):
             return
 
         # get values
-        add_user = get_setting(
+        add_user = get_setting('INVENTREE_ADMIN_USER', 'admin_user')
-            'INVENTREE_ADMIN_USER',
+        add_email = get_setting('INVENTREE_ADMIN_EMAIL', 'admin_email')
-            settings.CONFIG.get('admin_user', False)
+        add_password = get_setting('INVENTREE_ADMIN_PASSWORD', 'admin_password')
-        )
-        add_email = get_setting(
-            'INVENTREE_ADMIN_EMAIL',
-            settings.CONFIG.get('admin_email', False)
-        )
-        add_password = get_setting(
-            'INVENTREE_ADMIN_PASSWORD',
-            settings.CONFIG.get('admin_password', False)
-        )
 
         # check if all values are present
         set_variables = 0
+
         for tested_var in [add_user, add_email, add_password]:
             if tested_var:
                 set_variables += 1

InvenTree/InvenTree/config.py

@@ -2,18 +2,27 @@
 
 import logging
 import os
+import random
 import shutil
+import string
 from pathlib import Path
 
+import yaml
+
 logger = logging.getLogger('inventree')
 
 
+def is_true(x):
+    """Shortcut function to determine if a value "looks" like a boolean"""
+    return str(x).strip().lower() in ['1', 'y', 'yes', 't', 'true', 'on']
+
+
 def get_base_dir() -> Path:
     """Returns the base (top-level) InvenTree directory."""
     return Path(__file__).parent.parent.resolve()
 
 
-def get_config_file() -> Path:
+def get_config_file(create=True) -> Path:
     """Returns the path of the InvenTree configuration file.
 
     Note: It will be created it if does not already exist!
@@ -28,7 +37,7 @@ def get_config_file() -> Path:
         # Config file is *not* specified - use the default
         cfg_filename = base_dir.joinpath('config.yaml').resolve()
 
-    if not cfg_filename.exists():
+    if not cfg_filename.exists() and create:
         print("InvenTree configuration file 'config.yaml' not found - creating default file")
 
         cfg_template = base_dir.joinpath("config_template.yaml")
@@ -38,45 +47,159 @@ def get_config_file() -> Path:
     return cfg_filename
 
 
-def get_plugin_file():
+def load_config_data() -> map:
-    """Returns the path of the InvenTree plugins specification file.
+    """Load configuration data from the config file."""
 
-    Note: It will be created if it does not already exist!
+    cfg_file = get_config_file()
-    """
-    # Check if the plugin.txt file (specifying required plugins) is specified
-    PLUGIN_FILE = os.getenv('INVENTREE_PLUGIN_FILE')
 
-    if not PLUGIN_FILE:
+    with open(cfg_file, 'r') as cfg:
-        # If not specified, look in the same directory as the configuration file
+        data = yaml.safe_load(cfg)
-        config_dir = get_config_file().parent
-        PLUGIN_FILE = config_dir.joinpath('plugins.txt')
-    else:
-        # Make sure we are using a modern Path object
-        PLUGIN_FILE = Path(PLUGIN_FILE)
 
-    if not PLUGIN_FILE.exists():
+    return data
-        logger.warning("Plugin configuration file does not exist")
-        logger.info(f"Creating plugin file at '{PLUGIN_FILE}'")
-
-        # If opening the file fails (no write permission, for example), then this will throw an error
-        PLUGIN_FILE.write_text("# InvenTree Plugins (uses PIP framework to install)\n\n")
-
-    return PLUGIN_FILE
 
 
-def get_setting(environment_var, backup_val, default_value=None):
+def get_setting(env_var=None, config_key=None, default_value=None):
     """Helper function for retrieving a configuration setting value.
 
     - First preference is to look for the environment variable
     - Second preference is to look for the value of the settings file
     - Third preference is the default value
+
+    Arguments:
+        env_var: Name of the environment variable e.g. 'INVENTREE_STATIC_ROOT'
+        config_key: Key to lookup in the configuration file
+        default_value: Value to return if first two options are not provided
+
     """
-    val = os.getenv(environment_var)
+
+    # First, try to load from the environment variables
+    if env_var is not None:
+        val = os.getenv(env_var, None)
 
         if val is not None:
             return val
 
-    if backup_val is not None:
+    # Next, try to load from configuration file
-        return backup_val
+    if config_key is not None:
+        cfg_data = load_config_data()
 
+        result = None
+
+        # Hack to allow 'path traversal' in configuration file
+        for key in config_key.strip().split('.'):
+
+            if type(cfg_data) is not dict or key not in cfg_data:
+                result = None
+                break
+
+            result = cfg_data[key]
+            cfg_data = cfg_data[key]
+
+        if result is not None:
+            return result
+
+    # Finally, return the default value
     return default_value
+
+
+def get_boolean_setting(env_var=None, config_key=None, default_value=False):
+    """Helper function for retreiving a boolean configuration setting"""
+
+    return is_true(get_setting(env_var, config_key, default_value))
+
+
+def get_media_dir(create=True):
+    """Return the absolute path for the 'media' directory (where uploaded files are stored)"""
+
+    md = get_setting('INVENTREE_MEDIA_ROOT', 'media_root')
+
+    if not md:
+        raise FileNotFoundError('INVENTREE_MEDIA_ROOT not specified')
+
+    md = Path(md).resolve()
+
+    if create:
+        md.mkdir(parents=True, exist_ok=True)
+
+    return md
+
+
+def get_static_dir(create=True):
+    """Return the absolute path for the 'static' directory (where static files are stored)"""
+
+    sd = get_setting('INVENTREE_STATIC_ROOT', 'static_root')
+
+    if not sd:
+        raise FileNotFoundError('INVENTREE_STATIC_ROOT not specified')
+
+    sd = Path(sd).resolve()
+
+    if create:
+        sd.mkdir(parents=True, exist_ok=True)
+
+    return sd
+
+
+def get_plugin_file():
+    """Returns the path of the InvenTree plugins specification file.
+
+    Note: It will be created if it does not already exist!
+    """
+
+    # Check if the plugin.txt file (specifying required plugins) is specified
+    plugin_file = get_setting('INVENTREE_PLUGIN_FILE', 'plugin_file')
+
+    if not plugin_file:
+        # If not specified, look in the same directory as the configuration file
+        config_dir = get_config_file().parent
+        plugin_file = config_dir.joinpath('plugins.txt')
+    else:
+        # Make sure we are using a modern Path object
+        plugin_file = Path(plugin_file)
+
+    if not plugin_file.exists():
+        logger.warning("Plugin configuration file does not exist - creating default file")
+        logger.info(f"Creating plugin file at '{plugin_file}'")
+
+        # If opening the file fails (no write permission, for example), then this will throw an error
+        plugin_file.write_text("# InvenTree Plugins (uses PIP framework to install)\n\n")
+
+    return plugin_file
+
+
+def get_secret_key():
+    """Return the secret key value which will be used by django.
+
+    Following options are tested, in descending order of preference:
+
+    A) Check for environment variable INVENTREE_SECRET_KEY => Use raw key data
+    B) Check for environment variable INVENTREE_SECRET_KEY_FILE => Load key data from file
+    C) Look for default key file "secret_key.txt"
+    D) Create "secret_key.txt" if it does not exist
+    """
+
+    # Look for environment variable
+    if secret_key := get_setting('INVENTREE_SECRET_KEY', 'secret_key'):
+        logger.info("SECRET_KEY loaded by INVENTREE_SECRET_KEY")  # pragma: no cover
+        return secret_key
+
+    # Look for secret key file
+    if secret_key_file := get_setting('INVENTREE_SECRET_KEY_FILE', 'secret_key_file'):
+        secret_key_file = Path(secret_key_file).resolve()
+    else:
+        # Default location for secret key file
+        secret_key_file = get_base_dir().joinpath("secret_key.txt").resolve()
+
+    if not secret_key_file.exists():
+        logger.info(f"Generating random key file at '{secret_key_file}'")
+
+        # Create a random key file
+        options = string.digits + string.ascii_letters + string.punctuation
+        key = ''.join([random.choice(options) for i in range(100)])
+        secret_key_file.write_text(key)
+
+    logger.info(f"Loading SECRET_KEY from '{secret_key_file}'")
+
+    key_data = secret_key_file.read_text().strip()
+
+    return key_data

InvenTree/InvenTree/exceptions.py

@@ -29,7 +29,7 @@ def log_error(path):
     kind, info, data = sys.exc_info()
 
     # Check if the eror is on the ignore list
-    if kind in settings.IGNORRED_ERRORS:
+    if kind in settings.IGNORED_ERRORS:
         return
 
     Error.objects.create(

InvenTree/InvenTree/middleware.py

@@ -157,7 +157,7 @@ class InvenTreeExceptionProcessor(ExceptionProcessor):
         kind, info, data = sys.exc_info()
 
         # Check if the eror is on the ignore list
-        if kind in settings.IGNORRED_ERRORS:
+        if kind in settings.IGNORED_ERRORS:
             return
 
         return super().process_exception(request, exception)

InvenTree/InvenTree/settings.py

@@ -11,9 +11,7 @@ database setup in this file.
 
 import logging
 import os
-import random
 import socket
-import string
 import sys
 from pathlib import Path
 
@@ -24,22 +22,14 @@ from django.utils.translation import gettext_lazy as _
 
 import moneyed
 import sentry_sdk
-import yaml
 from sentry_sdk.integrations.django import DjangoIntegration
 
-from .config import get_base_dir, get_config_file, get_plugin_file, get_setting
+from . import config
-
+from .config import get_boolean_setting, get_setting
-
-def _is_true(x):
-    # Shortcut function to determine if a value "looks" like a boolean
-    return str(x).strip().lower() in ['1', 'y', 'yes', 't', 'true']
-
-
-# Default Sentry DSN (can be overriden if user wants custom sentry integration)
-INVENTREE_DSN = 'https://3928ccdba1d34895abde28031fd00100@o378676.ingest.sentry.io/6494600'
 
 # Determine if we are running in "test" mode e.g. "manage.py test"
 TESTING = 'test' in sys.argv
+
 # Are enviroment variables manipulated by tests? Needs to be set by testing code
 TESTING_ENV = False
 
@@ -47,33 +37,17 @@ TESTING_ENV = False
 DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
 
 # Build paths inside the project like this: BASE_DIR.joinpath(...)
-BASE_DIR = get_base_dir()
+BASE_DIR = config.get_base_dir()
 
-cfg_filename = get_config_file()
+# Load configuration data
-
+CONFIG = config.load_config_data()
-with open(cfg_filename, 'r') as cfg:
-    CONFIG = yaml.safe_load(cfg)
-
-# We will place any config files in the same directory as the config file
-config_dir = cfg_filename.parent
 
 # Default action is to run the system in Debug mode
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = _is_true(get_setting(
+DEBUG = get_boolean_setting('INVENTREE_DEBUG', 'debug', True)
-    'INVENTREE_DEBUG',
-    CONFIG.get('debug', True)
-))
-
-DOCKER = _is_true(get_setting(
-    'INVENTREE_DOCKER',
-    False
-))
 
 # Configure logging settings
-log_level = get_setting(
+log_level = get_setting('INVENTREE_LOG_LEVEL', 'log_level', 'WARNING')
-    'INVENTREE_LOG_LEVEL',
-    CONFIG.get('log_level', 'WARNING')
-)
 
 logging.basicConfig(
     level=log_level,
@@ -105,78 +79,20 @@ LOGGING = {
 # Get a logger instance for this setup file
 logger = logging.getLogger("inventree")
 
-"""
+# Load SECRET_KEY
-Specify a secret key to be used by django.
+SECRET_KEY = config.get_secret_key()
-
-Following options are tested, in descending order of preference:
-
-A) Check for environment variable INVENTREE_SECRET_KEY => Use raw key data
-B) Check for environment variable INVENTREE_SECRET_KEY_FILE => Load key data from file
-C) Look for default key file "secret_key.txt"
-d) Create "secret_key.txt" if it does not exist
-"""
-
-if secret_key := os.getenv("INVENTREE_SECRET_KEY"):
-    # Secret key passed in directly
-    SECRET_KEY = secret_key.strip()  # pragma: no cover
-    logger.info("SECRET_KEY loaded by INVENTREE_SECRET_KEY")  # pragma: no cover
-else:
-    # Secret key passed in by file location
-    key_file = os.getenv("INVENTREE_SECRET_KEY_FILE")
-
-    if key_file:
-        key_file = Path(key_file).resolve()  # pragma: no cover
-    else:
-        # default secret key location
-        key_file = BASE_DIR.joinpath("secret_key.txt").resolve()
-
-    if not key_file.exists():  # pragma: no cover
-        logger.info(f"Generating random key file at '{key_file}'")
-        # Create a random key file
-        options = string.digits + string.ascii_letters + string.punctuation
-        key = ''.join([random.choice(options) for i in range(100)])
-        key_file.write_text(key)
-
-    logger.info(f"Loading SECRET_KEY from '{key_file}'")
-
-    try:
-        SECRET_KEY = open(key_file, "r").read().strip()
-    except Exception:  # pragma: no cover
-        logger.exception(f"Couldn't load keyfile {key_file}")
-        sys.exit(-1)
 
 # The filesystem location for served static files
-STATIC_ROOT = Path(
+STATIC_ROOT = config.get_static_dir()
-    get_setting(
-        'INVENTREE_STATIC_ROOT',
-        CONFIG.get('static_root', None)
-    )
-).resolve()
 
-if STATIC_ROOT is None:  # pragma: no cover
+# The filesystem location for uploaded meadia files
-    print("ERROR: INVENTREE_STATIC_ROOT directory not defined")
+MEDIA_ROOT = config.get_media_dir()
-    sys.exit(1)
-else:
-    # Ensure the root really is availalble
-    STATIC_ROOT.mkdir(parents=True, exist_ok=True)
-
-# The filesystem location for served static files
-MEDIA_ROOT = Path(
-    get_setting(
-        'INVENTREE_MEDIA_ROOT',
-        CONFIG.get('media_root', None)
-    )
-).resolve()
-
-if MEDIA_ROOT is None:  # pragma: no cover
-    print("ERROR: INVENTREE_MEDIA_ROOT directory is not defined")
-    sys.exit(1)
-else:
-    # Ensure the root really is availalble
-    MEDIA_ROOT.mkdir(parents=True, exist_ok=True)
 
 # List of allowed hosts (default = allow all)
-ALLOWED_HOSTS = CONFIG.get('allowed_hosts', ['*'])
+ALLOWED_HOSTS = get_setting(
+    config_key='allowed_hosts',
+    default_value=['*']
+)
 
 # Cross Origin Resource Sharing (CORS) options
 
@@ -184,13 +100,15 @@ ALLOWED_HOSTS = CONFIG.get('allowed_hosts', ['*'])
 CORS_URLS_REGEX = r'^/api/.*$'
 
 # Extract CORS options from configuration file
-cors_opt = CONFIG.get('cors', None)
+CORS_ORIGIN_ALLOW_ALL = get_boolean_setting(
+    config_key='cors.allow_all',
+    default_value=False,
+)
 
-if cors_opt:
+CORS_ORIGIN_WHITELIST = get_setting(
-    CORS_ORIGIN_ALLOW_ALL = cors_opt.get('allow_all', False)
+    config_key='cors.whitelist',
-
+    default_value=[]
-    if not CORS_ORIGIN_ALLOW_ALL:
+)
-        CORS_ORIGIN_WHITELIST = cors_opt.get('whitelist', [])  # pragma: no cover
 
 # Web URL endpoint for served static files
 STATIC_URL = '/static/'
@@ -214,12 +132,6 @@ STATIC_COLOR_THEMES_DIR = STATIC_ROOT.joinpath('css', 'color-themes')
 # Web URL endpoint for served media files
 MEDIA_URL = '/media/'
 
-if DEBUG:
-    logger.info("InvenTree running with DEBUG enabled")
-
-logger.debug(f"MEDIA_ROOT: '{MEDIA_ROOT}'")
-logger.debug(f"STATIC_ROOT: '{STATIC_ROOT}'")
-
 # Application definition
 
 INSTALLED_APPS = [
@@ -320,6 +232,9 @@ INTERNAL_IPS = [
     '127.0.0.1',
 ]
 
+# Internal flag to determine if we are running in docker mode
+DOCKER = get_boolean_setting('INVENTREE_DOCKER', default_value=False)
+
 if DOCKER:  # pragma: no cover
     # Internal IP addresses are different when running under docker
     hostname, ___, ips = socket.gethostbyname_ex(socket.gethostname())
@@ -334,7 +249,8 @@ if DEBUG:
 # Base URL for admin pages (default="admin")
 INVENTREE_ADMIN_URL = get_setting(
     'INVENTREE_ADMIN_URL',
-    CONFIG.get('admin_url', 'admin'),
+    config_key='admin_url',
+    default_value='admin'
 )
 
 ROOT_URLCONF = 'InvenTree.urls'
@@ -498,7 +414,7 @@ if "postgres" in db_engine:  # pragma: no cover
         # long to connect to the database server
         # # seconds, 2 is minium allowed by libpq
         db_options["connect_timeout"] = int(
-            os.getenv("INVENTREE_DB_TIMEOUT", 2)
+            get_setting('INVENTREE_DB_TIMEOUT', 'database.timeout', 2)
         )
 
     # Setup TCP keepalive
@@ -509,23 +425,27 @@ if "postgres" in db_engine:  # pragma: no cover
     # # 0 - TCP Keepalives disabled; 1 - enabled
     if "keepalives" not in db_options:
         db_options["keepalives"] = int(
-            os.getenv("INVENTREE_DB_TCP_KEEPALIVES", "1")
+            get_setting('INVENTREE_DB_TCP_KEEPALIVES', 'database.tcp_keepalives', 1)
         )
-    # # Seconds after connection is idle to send keep alive
+
+    # Seconds after connection is idle to send keep alive
     if "keepalives_idle" not in db_options:
         db_options["keepalives_idle"] = int(
-            os.getenv("INVENTREE_DB_TCP_KEEPALIVES_IDLE", "1")
+            get_setting('INVENTREE_DB_TCP_KEEPALIVES_IDLE', 'database.tcp_keepalives_idle', 1)
         )
-    # # Seconds after missing ACK to send another keep alive
+
+    # Seconds after missing ACK to send another keep alive
     if "keepalives_interval" not in db_options:
         db_options["keepalives_interval"] = int(
-            os.getenv("INVENTREE_DB_TCP_KEEPALIVES_INTERVAL", "1")
+            get_setting("INVENTREE_DB_TCP_KEEPALIVES_INTERVAL", "database.tcp_keepalives_internal", "1")
         )
-    # # Number of missing ACKs before we close the connection
+
+    # Number of missing ACKs before we close the connection
     if "keepalives_count" not in db_options:
         db_options["keepalives_count"] = int(
-            os.getenv("INVENTREE_DB_TCP_KEEPALIVES_COUNT", "5")
+            get_setting("INVENTREE_DB_TCP_KEEPALIVES_COUNT", "database.tcp_keepalives_count", "5")
         )
+
     # # Milliseconds for how long pending data should remain unacked
     # by the remote server
     # TODO: Supported starting in PSQL 11
@@ -538,17 +458,11 @@ if "postgres" in db_engine:  # pragma: no cover
     # https://www.postgresql.org/docs/devel/transaction-iso.html
     # https://docs.djangoproject.com/en/3.2/ref/databases/#isolation-level
     if "isolation_level" not in db_options:
-        serializable = _is_true(
+        serializable = get_boolean_setting('INVENTREE_DB_ISOLATION_SERIALIZABLE', 'database.serializable', False)
-            os.getenv("INVENTREE_DB_ISOLATION_SERIALIZABLE", "false")
+        db_options["isolation_level"] = ISOLATION_LEVEL_SERIALIZABLE if serializable else ISOLATION_LEVEL_READ_COMMITTED
-        )
-        db_options["isolation_level"] = (
-            ISOLATION_LEVEL_SERIALIZABLE
-            if serializable
-            else ISOLATION_LEVEL_READ_COMMITTED
-        )
 
 # Specific options for MySql / MariaDB backend
-if "mysql" in db_engine:  # pragma: no cover
+elif "mysql" in db_engine:  # pragma: no cover
     # TODO TCP time outs and keepalives
 
     # MariaDB's default isolation level is Repeatable Read which is
@@ -558,15 +472,11 @@ if "mysql" in db_engine:  # pragma: no cover
     # https://mariadb.com/kb/en/mariadb-transactions-and-isolation-levels-for-sql-server-users/#changing-the-isolation-level
     # https://docs.djangoproject.com/en/3.2/ref/databases/#mysql-isolation-level
     if "isolation_level" not in db_options:
-        serializable = _is_true(
+        serializable = get_boolean_setting('INVENTREE_DB_ISOLATION_SERIALIZABLE', 'database.serializable', False)
-            os.getenv("INVENTREE_DB_ISOLATION_SERIALIZABLE", "false")
+        db_options["isolation_level"] = "serializable" if serializable else "read committed"
-        )
-        db_options["isolation_level"] = (
-            "serializable" if serializable else "read committed"
-        )
 
 # Specific options for sqlite backend
-if "sqlite" in db_engine:
+elif "sqlite" in db_engine:
     # TODO: Verify timeouts are not an issue because no network is involved for SQLite
 
     # SQLite's default isolation level is Serializable due to SQLite's
@@ -591,13 +501,11 @@ DATABASES = {
     'default': db_config
 }
 
-_cache_config = CONFIG.get("cache", {})
+# Cache configuration
-_cache_host = _cache_config.get("host", os.getenv("INVENTREE_CACHE_HOST"))
+cache_host = get_setting('INVENTREE_CACHE_HOST', 'cache.host', None)
-_cache_port = _cache_config.get(
+cache_port = get_setting('INVENTREE_CACHE_PORT', 'cache.port', '6379')
-    "port", os.getenv("INVENTREE_CACHE_PORT", "6379")
-)
 
-if _cache_host:  # pragma: no cover
+if cache_host:  # pragma: no cover
     # We are going to rely upon a possibly non-localhost for our cache,
     # so don't wait too long for the cache as nothing in the cache should be
     # irreplacable.
@@ -606,7 +514,7 @@ if _cache_host:  # pragma: no cover
         "SOCKET_CONNECT_TIMEOUT": int(os.getenv("CACHE_CONNECT_TIMEOUT", "2")),
         "SOCKET_TIMEOUT": int(os.getenv("CACHE_SOCKET_TIMEOUT", "2")),
         "CONNECTION_POOL_KWARGS": {
-            "socket_keepalive": _is_true(
+            "socket_keepalive": config.is_true(
                 os.getenv("CACHE_TCP_KEEPALIVE", "1")
             ),
             "socket_keepalive_options": {
@@ -628,7 +536,7 @@ if _cache_host:  # pragma: no cover
     CACHES = {
         "default": {
             "BACKEND": "django_redis.cache.RedisCache",
-            "LOCATION": f"redis://{_cache_host}:{_cache_port}/0",
+            "LOCATION": f"redis://{cache_host}:{cache_port}/0",
             "OPTIONS": _cache_options,
         },
     }
@@ -639,17 +547,11 @@ else:
         },
     }
 
-try:
+# django-q background worker configuration
-    # 4 background workers seems like a sensible default
-    background_workers = int(os.environ.get('INVENTREE_BACKGROUND_WORKERS', 4))
-except ValueError:  # pragma: no cover
-    background_workers = 4
-
-# django-q configuration
 Q_CLUSTER = {
     'name': 'InvenTree',
-    'workers': background_workers,
+    'workers': int(get_setting('INVENTREE_BACKGROUND_WORKERS', 'background.workers', 4)),
-    'timeout': 90,
+    'timeout': int(get_setting('INVENTREE_BACKGROUND_TIMEOUT', 'background.timeout', 90)),
     'retry': 120,
     'queue_limit': 50,
     'bulk': 10,
@@ -657,7 +559,7 @@ Q_CLUSTER = {
     'sync': False,
 }
 
-if _cache_host:  # pragma: no cover
+if cache_host:  # pragma: no cover
     # If using external redis cache, make the cache the broker for Django Q
     # as well
     Q_CLUSTER["django_redis"] = "worker"
@@ -698,8 +600,7 @@ if type(EXTRA_URL_SCHEMES) not in [list]:  # pragma: no cover
 
 # Internationalization
 # https://docs.djangoproject.com/en/dev/topics/i18n/
-
+LANGUAGE_CODE = get_setting('INVENTREE_LANGUAGE', 'language', 'en-us')
-LANGUAGE_CODE = CONFIG.get('language', 'en-us')
 
 # If a new language translation is supported, it must be added here
 LANGUAGES = [
@@ -730,7 +631,7 @@ LANGUAGES = [
 ]
 
 # Testing interface translations
-if get_setting('TEST_TRANSLATIONS', False):  # pragma: no cover
+if get_boolean_setting('TEST_TRANSLATIONS', default_value=False):  # pragma: no cover
     # Set default language
     LANGUAGE_CODE = 'xx'
 
@@ -762,68 +663,29 @@ for currency in CURRENCIES:
         print(f"Currency code '{currency}' is not supported")
         sys.exit(1)
 
-
 # Custom currency exchange backend
 EXCHANGE_BACKEND = 'InvenTree.exchange.InvenTreeExchange'
 
-# Extract email settings from the config file
+# Email configuration options
-email_config = CONFIG.get('email', {})
+EMAIL_BACKEND = get_setting('INVENTREE_EMAIL_BACKEND', 'email.backend', 'django.core.mail.backends.smtp.EmailBackend')
+EMAIL_HOST = get_setting('INVENTREE_EMAIL_HOST', 'email.host', '')
+EMAIL_PORT = int(get_setting('INVENTREE_EMAIL_PORT', 'email.port', 25))
+EMAIL_HOST_USER = get_setting('INVENTREE_EMAIL_USERNAME', 'email.username', '')
+EMAIL_HOST_PASSWORD = get_setting('INVENTREE_EMAIL_PASSWORD', 'email.password', '')
+EMAIL_SUBJECT_PREFIX = get_setting('INVENTREE_EMAIL_PREFIX', 'email.prefix', '[InvenTree] ')
+EMAIL_USE_TLS = get_boolean_setting('INVENTREE_EMAIL_TLS', 'email.tls', False)
+EMAIL_USE_SSL = get_boolean_setting('INVENTREE_EMAIL_SSL', 'email.ssl', False)
 
-EMAIL_BACKEND = get_setting(
+DEFUALT_FROM_EMAIL = get_setting('INVENTREE_EMAIL_SENDER', 'email.sender', '')
-    'INVENTREE_EMAIL_BACKEND',
-    email_config.get('backend', 'django.core.mail.backends.smtp.EmailBackend')
-)
-
-# Email backend settings
-EMAIL_HOST = get_setting(
-    'INVENTREE_EMAIL_HOST',
-    email_config.get('host', '')
-)
-
-EMAIL_PORT = get_setting(
-    'INVENTREE_EMAIL_PORT',
-    email_config.get('port', 25)
-)
-
-EMAIL_HOST_USER = get_setting(
-    'INVENTREE_EMAIL_USERNAME',
-    email_config.get('username', ''),
-)
-
-EMAIL_HOST_PASSWORD = get_setting(
-    'INVENTREE_EMAIL_PASSWORD',
-    email_config.get('password', ''),
-)
-
-DEFAULT_FROM_EMAIL = get_setting(
-    'INVENTREE_EMAIL_SENDER',
-    email_config.get('sender', ''),
-)
-
-EMAIL_SUBJECT_PREFIX = '[InvenTree] '
 
 EMAIL_USE_LOCALTIME = False
-
-EMAIL_USE_TLS = get_setting(
-    'INVENTREE_EMAIL_TLS',
-    email_config.get('tls', False),
-)
-
-EMAIL_USE_SSL = get_setting(
-    'INVENTREE_EMAIL_SSL',
-    email_config.get('ssl', False),
-)
-
 EMAIL_TIMEOUT = 60
 
 LOCALE_PATHS = (
     BASE_DIR.joinpath('locale/'),
 )
 
-TIME_ZONE = get_setting(
+TIME_ZONE = get_setting('INVENTREE_TIMEZONE', 'timezone', 'UTC')
-    'INVENTREE_TIMEZONE',
-    CONFIG.get('timezone', 'UTC')
-)
 
 USE_I18N = True
 
@@ -856,8 +718,8 @@ SOCIALACCOUNT_PROVIDERS = CONFIG.get('social_providers', [])
 SOCIALACCOUNT_STORE_TOKENS = True
 
 # settings for allauth
-ACCOUNT_EMAIL_CONFIRMATION_EXPIRE_DAYS = get_setting('INVENTREE_LOGIN_CONFIRM_DAYS', CONFIG.get('login_confirm_days', 3))
+ACCOUNT_EMAIL_CONFIRMATION_EXPIRE_DAYS = get_setting('INVENTREE_LOGIN_CONFIRM_DAYS', 'login_confirm_days', 3)
-ACCOUNT_LOGIN_ATTEMPTS_LIMIT = get_setting('INVENTREE_LOGIN_ATTEMPTS', CONFIG.get('login_attempts', 5))
+ACCOUNT_LOGIN_ATTEMPTS_LIMIT = get_setting('INVENTREE_LOGIN_ATTEMPTS', 'login_attempts', 5)
 ACCOUNT_LOGOUT_ON_PASSWORD_CHANGE = True
 ACCOUNT_PREVENT_ENUMERATION = True
 
@@ -877,8 +739,8 @@ SOCIALACCOUNT_ADAPTER = 'InvenTree.forms.CustomSocialAccountAdapter'
 ACCOUNT_ADAPTER = 'InvenTree.forms.CustomAccountAdapter'
 
 # login settings
-REMOTE_LOGIN = get_setting('INVENTREE_REMOTE_LOGIN', CONFIG.get('remote_login', False))
+REMOTE_LOGIN = get_boolean_setting('INVENTREE_REMOTE_LOGIN', 'remote_login_enabled', False)
-REMOTE_LOGIN_HEADER = get_setting('INVENTREE_REMOTE_LOGIN_HEADER', CONFIG.get('remote_login_header', 'REMOTE_USER'))
+REMOTE_LOGIN_HEADER = get_setting('INVENTREE_REMOTE_LOGIN_HEADER', 'remote_login_header', 'REMOTE_USER')
 
 # Markdownify configuration
 # Ref: https://django-markdownify.readthedocs.io/en/latest/settings.html
@@ -909,11 +771,12 @@ MARKDOWNIFY = {
     }
 }
 
-# Error reporting
+# sentry.io integration for error reporting
-SENTRY_ENABLED = get_setting('INVENTREE_SENTRY_ENABLED', CONFIG.get('sentry_enabled', False))
+SENTRY_ENABLED = get_boolean_setting('INVENTREE_SENTRY_ENABLED', 'sentry_enabled', False)
-SENTRY_DSN = get_setting('INVENTREE_SENTRY_DSN', CONFIG.get('sentry_dsn', INVENTREE_DSN))
+# Default Sentry DSN (can be overriden if user wants custom sentry integration)
-
+INVENTREE_DSN = 'https://3928ccdba1d34895abde28031fd00100@o378676.ingest.sentry.io/6494600'
-SENTRY_SAMPLE_RATE = float(get_setting('INVENTREE_SENTRY_SAMPLE_RATE', CONFIG.get('sentry_sample_rate', 0.1)))
+SENTRY_DSN = get_setting('INVENTREE_SENTRY_DSN', 'sentry_dsn', INVENTREE_DSN)
+SENTRY_SAMPLE_RATE = float(get_setting('INVENTREE_SENTRY_SAMPLE_RATE', 'sentry_sample_rate', 0.1))
 
 if SENTRY_ENABLED and SENTRY_DSN:  # pragma: no cover
     sentry_sdk.init(
@@ -932,7 +795,7 @@ if SENTRY_ENABLED and SENTRY_DSN:  # pragma: no cover
         sentry_sdk.set_tag(f'inventree_{key}', val)
 
 # In-database error logging
-IGNORRED_ERRORS = [
+IGNORED_ERRORS = [
     Http404
 ]
 
@@ -941,33 +804,29 @@ MAINTENANCE_MODE_RETRY_AFTER = 60
 MAINTENANCE_MODE_STATE_BACKEND = 'maintenance_mode.backends.DefaultStorageBackend'
 
 # Are plugins enabled?
-PLUGINS_ENABLED = _is_true(get_setting(
+PLUGINS_ENABLED = get_boolean_setting('INVENTREE_PLUGINS_ENABLED', 'plugins_enabled', False)
-    'INVENTREE_PLUGINS_ENABLED',
-    CONFIG.get('plugins_enabled', False),
-))
 
-PLUGIN_FILE = get_plugin_file()
+PLUGIN_FILE = config.get_plugin_file()
 
 # Plugin test settings
-PLUGIN_TESTING = get_setting('PLUGIN_TESTING', TESTING)  # are plugins beeing tested?
+PLUGIN_TESTING = CONFIG.get('PLUGIN_TESTING', TESTING)  # are plugins beeing tested?
-PLUGIN_TESTING_SETUP = get_setting('PLUGIN_TESTING_SETUP', False)  # load plugins from setup hooks in testing?
+PLUGIN_TESTING_SETUP = CONFIG.get('PLUGIN_TESTING_SETUP', False)  # load plugins from setup hooks in testing?
 PLUGIN_TESTING_EVENTS = False                  # Flag if events are tested right now
-PLUGIN_RETRY = get_setting('PLUGIN_RETRY', 5)  # how often should plugin loading be tried?
+PLUGIN_RETRY = CONFIG.get('PLUGIN_RETRY', 5)  # how often should plugin loading be tried?
 PLUGIN_FILE_CHECKED = False                    # Was the plugin file checked?
 
 # User interface customization values
-CUSTOMIZE = get_setting(
+CUSTOMIZE = get_setting('INVENTREE_CUSTOMIZE', 'customize', {})
-    'INVENTREE_CUSTOMIZE',
-    CONFIG.get('customize', {}),
-    {}
-)
 
-CUSTOM_LOGO = get_setting(
+CUSTOM_LOGO = get_setting('INVENTREE_CUSTOM_LOGO', 'customize.logo', None)
-    'INVENTREE_CUSTOM_LOGO',
-    CUSTOMIZE.get('logo', False)
-)
 
 # check that the logo-file exsists in media
 if CUSTOM_LOGO and not default_storage.exists(CUSTOM_LOGO):  # pragma: no cover
     logger.warning(f"The custom logo file '{CUSTOM_LOGO}' could not be found in the default media storage")
     CUSTOM_LOGO = False
+
+if DEBUG:
+    logger.info("InvenTree running with DEBUG enabled")
+
+logger.debug(f"MEDIA_ROOT: '{MEDIA_ROOT}'")
+logger.debug(f"STATIC_ROOT: '{STATIC_ROOT}'")

InvenTree/InvenTree/test_middleware.py

@@ -84,7 +84,7 @@ class MiddlewareTests(InvenTreeTestCase):
             log_error('testpath')
 
         # Test setup without ignored errors
-        settings.IGNORRED_ERRORS = []
+        settings.IGNORED_ERRORS = []
         response = self.client.get(reverse('part-detail', kwargs={'pk': 9999}))
         self.assertEqual(response.status_code, 404)
         check(1)

InvenTree/common/migrations/0010_migrate_currency_setting.py

@@ -2,12 +2,12 @@
 
 from django.db import migrations
 from common.models import InvenTreeSetting
-from InvenTree.settings import get_setting, CONFIG
+from InvenTree.config import get_setting
 
 def set_default_currency(apps, schema_editor):
     """ migrate the currency setting from config.yml to db """
     # get value from settings-file
-    base_currency = get_setting('INVENTREE_BASE_CURRENCY', CONFIG.get('base_currency', 'USD'))
+    base_currency = get_setting('INVENTREE_BASE_CURRENCY', 'base_currency', 'USD')
     # write to database
     InvenTreeSetting.set_setting('INVENTREE_DEFAULT_CURRENCY', base_currency, None, create=True)
 

InvenTree/config_template.yaml

@@ -1,7 +1,6 @@
 
 # Database backend selection - Configure backend database settings
-# Ref: https://docs.djangoproject.com/en/dev/ref/settings/#std:setting-DATABASES
+# Documentation: https://inventree.readthedocs.io/en/latest/start/config/
-# Specify database parameters below as they appear in the Django docs
 
 # Note: Database configuration options can also be specified from environmental variables,
 #       with the prefix INVENTREE_DB_
@@ -44,20 +43,32 @@ database:
   # ENGINE: sqlite3
   # NAME: '/home/inventree/database.sqlite3'
 
+# Set debug to False to run in production mode
+# Use the environment variable INVENTREE_DEBUG
+debug: True
+
+# Configure the system logging level
+# Use environment variable INVENTREE_LOG_LEVEL
+# Options: DEBUG / INFO / WARNING / ERROR / CRITICAL
+log_level: WARNING
+
 # Select default system language (default is 'en-us')
+# Use the environment variable INVENTREE_LANGUAGE
 language: en-us
 
 # System time-zone (default is UTC)
 # Reference: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
-# Select an option from the "TZ database name" column
-# Use the environment variable INVENTREE_TIMEZONE
 timezone: UTC
 
-# Base currency code
+# Base currency code (or use env var INVENTREE_BASE_CURRENCY)
 base_currency: USD
 
-# List of currencies supported by default.
+# Add new user on first startup
-# Add other currencies here to allow use in InvenTree
+#admin_user: admin
+#admin_email: info@example.com
+#admin_password: inventree
+
+# List of currencies supported by default. Add other currencies here to allow use in InvenTree
 currencies:
   - AUD
   - CAD
@@ -70,15 +81,6 @@ currencies:
 
 # Email backend configuration
 # Ref: https://docs.djangoproject.com/en/dev/topics/email/
-# Available options:
-# host: Email server host address
-# port: Email port
-# username: Account username
-# password: Account password
-# prefix: Email subject prefix
-# tls: Enable TLS support
-# ssl: Enable SSL support
-
 # Alternatively, these options can all be set using environment variables,
 # with the INVENTREE_EMAIL_ prefix:
 # e.g. INVENTREE_EMAIL_HOST / INVENTREE_EMAIL_PORT / INVENTREE_EMAIL_USERNAME
@@ -94,31 +96,17 @@ email:
   tls: False
   ssl: False
 
-# Set debug to False to run in production mode
-# Use the environment variable INVENTREE_DEBUG
-debug: True
-
-# Set debug_toolbar to True to enable a debugging toolbar for InvenTree
-# Note: This will only be displayed if DEBUG mode is enabled,
-#       and only if InvenTree is accessed from a local IP (127.0.0.1)
-debug_toolbar: False
-
 # Set sentry_enabled to True to report errors back to the maintainers
-# Use the environment variable INVENTREE_SENTRY_ENABLED
+# Set sentry,dsn to your custom DSN if you want to use your own instance for error reporting
-# sentry_enabled: True
+sentry_enabled: False
-
+#sentry_sample_rate: 0.1
-# Set sentry_dsn to your custom DSN if you want to use your own instance for error reporting
-# Use the environment variable INVENTREE_SENTRY_DSN
 #sentry_dsn: https://custom@custom.ingest.sentry.io/custom
 
 # Set this variable to True to enable InvenTree Plugins
 # Alternatively, use the environment variable INVENTREE_PLUGINS_ENABLED
 plugins_enabled: False
-
+#plugin_file: /path/to/plugins.txt
-# Configure the system logging level
+#plugin_dir: /path/to/plugins/
-# Use environment variable INVENTREE_LOG_LEVEL
-# Options: DEBUG / INFO / WARNING / ERROR / CRITICAL
-log_level: WARNING
 
 # Allowed hosts (see ALLOWED_HOSTS in Django settings documentation)
 # A list of strings representing the host/domain names that this Django site can serve.
@@ -138,14 +126,15 @@ cors:
   # - https://sub.example.com
 
 # MEDIA_ROOT is the local filesystem location for storing uploaded files
-# By default, it is stored under /home/inventree/data/media
+# media_root: '/home/inventree/data/media'
-# Use environment variable INVENTREE_MEDIA_ROOT
-media_root: '/home/inventree/data/media'
 
 # STATIC_ROOT is the local filesystem location for storing static files
-# By default, it is stored under /home/inventree/data/static
+# static_root: '/home/inventree/data/static'
-# Use environment variable INVENTREE_STATIC_ROOT
+
-static_root: '/home/inventree/data/static'
+# Background worker options
+background:
+  workers: 4
+  timeout: 90
 
 # Optional URL schemes to allow in URL fields
 # By default, only the following schemes are allowed: ['http', 'https', 'ftp', 'ftps']
@@ -156,25 +145,14 @@ static_root: '/home/inventree/data/static'
 #  - ssh
 
 # Login configuration
-# How long do confirmation mail last?
+login_confirm_days: 3
-# Use environment variable INVENTREE_LOGIN_CONFIRM_DAYS
+login_attempts: 5
-#login_confirm_days: 3
-# How many wrong login attempts are permitted?
-# Use environment variable INVENTREE_LOGIN_ATTEMPTS
-#login_attempts: 5
 
 # Remote / proxy login
 # These settings can introduce security problems if configured incorrectly. Please read
 # https://docs.djangoproject.com/en/4.0/howto/auth-remote-user/ for more details
-# Use environment variable INVENTREE_REMOTE_LOGIN
+remote_login_enabled: False
-# remote_login: True
+remote_login_header: REMOTE_USER
-# Use environment variable INVENTREE_REMOTE_LOGIN_HEADER
-# remote_login_header: REMOTE_USER
-
-# Add new user on first startup
-#admin_user: admin
-#admin_email: info@example.com
-#admin_password: inventree
 
 # Permit custom authentication backends
 #authentication_backends:

InvenTree/plugin/registry.py

@@ -22,6 +22,8 @@ from django.utils.text import slugify
 from maintenance_mode.core import (get_maintenance_mode, maintenance_mode_on,
                                    set_maintenance_mode)
 
+from InvenTree.config import get_setting
+
 from .helpers import (IntegrationPluginError, get_plugins, handle_error,
                       log_error)
 from .plugin import InvenTreePlugin
@@ -199,7 +201,7 @@ class PluginsRegistry:
         if settings.TESTING:
             custom_dirs = os.getenv('INVENTREE_PLUGIN_TEST_DIR', None)
         else:
-            custom_dirs = os.getenv('INVENTREE_PLUGIN_DIR', None)
+            custom_dirs = get_setting('INVENTREE_PLUGIN_DIR', 'plugin_dir')
 
             # Load from user specified directories (unless in testing mode)
             dirs.append('plugins')

docker/requirements.txt

@@ -1,9 +1,10 @@
 # Base python requirements for docker containers
 
 # Basic package requirements
+invoke>=1.4.0                   # Invoke build tool
+pyyaml>=6.0
 setuptools==60.0.5
 wheel>=0.37.0
-invoke>=1.4.0                   # Invoke build tool
 
 # Database links
 psycopg2>=2.9.1

tasks.py

@@ -4,6 +4,7 @@ import json
 import os
 import pathlib
 import re
+import shutil
 import sys
 from pathlib import Path
 
@@ -522,6 +523,8 @@ def test(c, database=None):
 def setup_test(c, ignore_update=False, dev=False, path="inventree-demo-dataset"):
     """Setup a testing enviroment."""
 
+    from InvenTree.InvenTree.config import get_media_dir
+
     if not ignore_update:
         update(c)
 
@@ -540,8 +543,16 @@ def setup_test(c, ignore_update=False, dev=False, path="inventree-demo-dataset")
         migrate(c)
 
     # Load data
-    print("Loading data ...")
+    print("Loading database records ...")
     import_records(c, filename=f'{path}/inventree_data.json', clear=True)
+
+    # Copy media files
+    print("Copying media files ...")
+    src = Path(path).joinpath('media').resolve()
+    dst = get_media_dir()
+
+    shutil.copytree(src, dst, dirs_exist_ok=True)
+
     print("Done setting up test enviroment...")
     print("========================================")