inventree/InvenTree
2
0
Fork 0
mirror of https://github.com/inventree/InvenTree.git synced 2026-03-21 11:44:42 +00:00
Code Activity

[db] Backend setting improvements (#11500)

Browse Source 
* Refactor database engine options

- Move to setting/db_backend.py
- Cleanup settings.py

* Fix documentation for postgres settings

* docs updates

* Add transaction_mode options for sqlite

* Update CHANGELOG with breaking changes

* Remove hard-coded database config

* Raise error on invalid backend

* Fix typos

* Fix broken redis link

* Limit to single worker thread for sqlite

* Update docs

* Add verbosity switch to dev.test task

* Add test timeout - kill hanging tests after 120s

* Set WAL mode for sqlite

* Use IMMEDIATE mode for background worker thread

* Use config to set WAL rather than custom hook

* Tweak pyproject settings

* Tweak code

* Increase timeouts

* Reset requirements to master
This commit is contained in:
Oliver
2026-03-18 00:01:17 +11:00
committed by GitHub
parent 34b7a559d7
commit 756c0be0b5
13 changed files with 220 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/backend/InvenTree/.gitignore vendored
View File

@@ -1,5 +1,6 @@
# Files generated during unit testing
_testfolder/
_tests_report*.txt
# Playwright files for CI
InvenTree/static/img/playwright*.png

2
src/backend/InvenTree/InvenTree/api.py
View File

@@ -507,7 +507,7 @@ class BulkCreateMixin:
if unique_create_fields := getattr(self, 'unique_create_fields', None):
existing = collections.defaultdict(list)
for idx, item in enumerate(data):
key = tuple(item[v] for v in unique_create_fields)
key = tuple(item[v] for v in list(unique_create_fields))
existing[key].append(idx)
unique_errors = [[] for _ in range(len(data))]

2
src/backend/InvenTree/InvenTree/config.py
View File

@@ -191,7 +191,7 @@ def load_config_data(set_cache: bool = False) -> map | None:
if CONFIG_DATA is not None and not set_cache:
return CONFIG_DATA
import yaml
import yaml.parser
cfg_file = get_config_file()

2
src/backend/InvenTree/InvenTree/exceptions.py
View File

@@ -66,7 +66,7 @@ def log_error(
data = error_data
else:
try:
formatted_exception = traceback.format_exception(kind, info, data) # type: ignore[no-matching-overload]
formatted_exception = traceback.format_exception(kind, info, data)
data = '\n'.join(formatted_exception)
except AttributeError:
data = 'No traceback information available'

152
src/backend/InvenTree/InvenTree/setting/db_backend.py Normal file
View File

@@ -0,0 +1,152 @@
"""Configuration settings specific to a particular database backend."""
import structlog
from InvenTree.config import get_boolean_setting, get_setting
logger = structlog.get_logger('inventree')
def set_db_options(engine: str, db_options: dict):
"""Update database options based on the specified database backend.
Arguments:
engine: The database engine (e.g. 'sqlite3', 'postgresql', etc.)
db_options: The database options dictionary to update
"""
logger.debug('Setting database options: %s', engine)
if 'postgres' in engine:
set_postgres_options(db_options)
elif 'mysql' in engine:
set_mysql_options(db_options)
elif 'sqlite' in engine:
set_sqlite_options(db_options)
else:
raise ValueError(f'Unknown database engine: {engine}')
def set_postgres_options(db_options: dict):
"""Set database options specific to postgres backend."""
from django.db.backends.postgresql.psycopg_any import ( # type: ignore[unresolved-import]
IsolationLevel,
)
# Connection timeout
if 'connect_timeout' not in db_options:
# The DB server is in the same data center, it should not take very
# long to connect to the database server
# # seconds, 2 is minimum allowed by libpq
db_options['connect_timeout'] = int(
get_setting('INVENTREE_DB_TIMEOUT', 'database.timeout', 2)
)
# Setup TCP keepalive
# DB server is in the same DC, it should not become unresponsive for
# very long. With the defaults below we wait 5 seconds for the network
# issue to resolve itself. If that doesn't happen, whatever happened
# is probably fatal and no amount of waiting is going to fix it.
# # 0 - TCP Keepalives disabled; 1 - enabled
if 'keepalives' not in db_options:
db_options['keepalives'] = int(
get_setting('INVENTREE_DB_TCP_KEEPALIVES', 'database.tcp_keepalives', 1)
)
# Seconds after connection is idle to send keep alive
if 'keepalives_idle' not in db_options:
db_options['keepalives_idle'] = int(
get_setting(
'INVENTREE_DB_TCP_KEEPALIVES_IDLE', 'database.tcp_keepalives_idle', 1
)
)
# Seconds after missing ACK to send another keep alive
if 'keepalives_interval' not in db_options:
db_options['keepalives_interval'] = int(
get_setting(
'INVENTREE_DB_TCP_KEEPALIVES_INTERVAL',
'database.tcp_keepalives_interval',
'1',
)
)
# Number of missing ACKs before we close the connection
if 'keepalives_count' not in db_options:
db_options['keepalives_count'] = int(
get_setting(
'INVENTREE_DB_TCP_KEEPALIVES_COUNT',
'database.tcp_keepalives_count',
'5',
)
)
# # Milliseconds for how long pending data should remain unacked
# by the remote server
# TODO: Supported starting in PSQL 11
# "tcp_user_timeout": int(os.getenv("PGTCP_USER_TIMEOUT", "1000"),
# Postgres's default isolation level is Read Committed which is
# normally fine, but most developers think the database server is
# actually going to do Serializable type checks on the queries to
# protect against simultaneous changes.
# https://www.postgresql.org/docs/devel/transaction-iso.html
# https://docs.djangoproject.com/en/3.2/ref/databases/#isolation-level
if 'isolation_level' not in db_options:
serializable = get_boolean_setting(
'INVENTREE_DB_ISOLATION_SERIALIZABLE', 'database.serializable', False
)
db_options['isolation_level'] = (
IsolationLevel.SERIALIZABLE
if serializable
else IsolationLevel.READ_COMMITTED
)
def set_mysql_options(db_options: dict):
"""Set database options specific to mysql backend."""
# TODO TCP time outs and keepalives
# MariaDB's default isolation level is Repeatable Read which is
# normally fine, but most developers think the database server is
# actually going to Serializable type checks on the queries to
# protect against simultaneous changes.
# https://mariadb.com/kb/en/mariadb-transactions-and-isolation-levels-for-sql-server-users/#changing-the-isolation-level
# https://docs.djangoproject.com/en/3.2/ref/databases/#mysql-isolation-level
if 'isolation_level' not in db_options:
serializable = get_boolean_setting(
'INVENTREE_DB_ISOLATION_SERIALIZABLE', 'database.serializable', False
)
db_options['isolation_level'] = (
'serializable' if serializable else 'read committed'
)
def set_sqlite_options(db_options: dict):
"""Set database options specific to sqlite backend.
References:
- https://docs.djangoproject.com/en/5.0/ref/databases/#sqlite-notes
- https://docs.djangoproject.com/en/6.0/ref/databases/#database-is-locked-errors
"""
import InvenTree.ready
# Specify minimum timeout behavior for SQLite connections
if 'timeout' not in db_options:
db_options['timeout'] = int(
get_setting('INVENTREE_DB_TIMEOUT', 'database.timeout', 10)
)
# Specify the transaction mode for the database
# For the backend worker thread, IMMEDIATE mode is used,
# it has been determined to provide better protection against database locks in the worker thread
db_options['transaction_mode'] = (
'IMMEDIATE' if InvenTree.ready.isInWorkerThread() else 'DEFERRED'
)
# SQLite's default isolation level is Serializable due to SQLite's
# single writer implementation. Presumably as a result of this, it is
# not possible to implement any lower isolation levels in SQLite.
# https://www.sqlite.org/isolation.html
# Specify that we want to use Write-Ahead Logging (WAL) mode for SQLite databases, as this allows for better concurrency and performance
db_options['init_command'] = 'PRAGMA journal_mode=WAL;'

110
src/backend/InvenTree/InvenTree/settings.py
View File

@@ -33,7 +33,7 @@ from InvenTree.version import checkMinPythonVersion, inventreeCommitHash
from users.oauth2_scopes import oauth2_scopes
from . import config
from .setting import locales, markdown, spectacular, storages
from .setting import db_backend, locales, markdown, spectacular, storages
try:
import django_stubs_ext
@@ -720,108 +720,8 @@ db_options = db_config.get('OPTIONS', db_config.get('options'))
if db_options is None:
db_options = {}
# Specific options for postgres backend
if 'postgres' in DB_ENGINE: # pragma: no cover
from django.db.backends.postgresql.psycopg_any import ( # type: ignore[unresolved-import]
IsolationLevel,
)
# Connection timeout
if 'connect_timeout' not in db_options:
# The DB server is in the same data center, it should not take very
# long to connect to the database server
# # seconds, 2 is minimum allowed by libpq
db_options['connect_timeout'] = int(
get_setting('INVENTREE_DB_TIMEOUT', 'database.timeout', 2)
)
# Setup TCP keepalive
# DB server is in the same DC, it should not become unresponsive for
# very long. With the defaults below we wait 5 seconds for the network
# issue to resolve itself. It it that doesn't happen whatever happened
# is probably fatal and no amount of waiting is going to fix it.
# # 0 - TCP Keepalives disabled; 1 - enabled
if 'keepalives' not in db_options:
db_options['keepalives'] = int(
get_setting('INVENTREE_DB_TCP_KEEPALIVES', 'database.tcp_keepalives', 1)
)
# Seconds after connection is idle to send keep alive
if 'keepalives_idle' not in db_options:
db_options['keepalives_idle'] = int(
get_setting(
'INVENTREE_DB_TCP_KEEPALIVES_IDLE', 'database.tcp_keepalives_idle', 1
)
)
# Seconds after missing ACK to send another keep alive
if 'keepalives_interval' not in db_options:
db_options['keepalives_interval'] = int(
get_setting(
'INVENTREE_DB_TCP_KEEPALIVES_INTERVAL',
'database.tcp_keepalives_internal',
'1',
)
)
# Number of missing ACKs before we close the connection
if 'keepalives_count' not in db_options:
db_options['keepalives_count'] = int(
get_setting(
'INVENTREE_DB_TCP_KEEPALIVES_COUNT',
'database.tcp_keepalives_count',
'5',
)
)
# # Milliseconds for how long pending data should remain unacked
# by the remote server
# TODO: Supported starting in PSQL 11
# "tcp_user_timeout": int(os.getenv("PGTCP_USER_TIMEOUT", "1000"),
# Postgres's default isolation level is Read Committed which is
# normally fine, but most developers think the database server is
# actually going to do Serializable type checks on the queries to
# protect against simultaneous changes.
# https://www.postgresql.org/docs/devel/transaction-iso.html
# https://docs.djangoproject.com/en/3.2/ref/databases/#isolation-level
if 'isolation_level' not in db_options:
serializable = get_boolean_setting(
'INVENTREE_DB_ISOLATION_SERIALIZABLE', 'database.serializable', False
)
db_options['isolation_level'] = (
IsolationLevel.SERIALIZABLE
if serializable
else IsolationLevel.READ_COMMITTED
)
# Specific options for MySql / MariaDB backend
elif 'mysql' in DB_ENGINE: # pragma: no cover
# TODO TCP time outs and keepalives
# MariaDB's default isolation level is Repeatable Read which is
# normally fine, but most developers think the database server is
# actually going to Serializable type checks on the queries to
# protect against siumltaneous changes.
# https://mariadb.com/kb/en/mariadb-transactions-and-isolation-levels-for-sql-server-users/#changing-the-isolation-level
# https://docs.djangoproject.com/en/3.2/ref/databases/#mysql-isolation-level
if 'isolation_level' not in db_options:
serializable = get_boolean_setting(
'INVENTREE_DB_ISOLATION_SERIALIZABLE', 'database.serializable', False
)
db_options['isolation_level'] = (
'serializable' if serializable else 'read committed'
)
# Specific options for sqlite backend
elif 'sqlite' in DB_ENGINE:
# TODO: Verify timeouts are not an issue because no network is involved for SQLite
# SQLite's default isolation level is Serializable due to SQLite's
# single writer implementation. Presumably as a result of this, it is
# not possible to implement any lower isolation levels in SQLite.
# https://www.sqlite.org/isolation.html
pass
# Set database-specific options
db_backend.set_db_options(DB_ENGINE, db_options)
# Provide OPTIONS dict back to the database configuration dict
db_config['OPTIONS'] = db_options
@@ -943,6 +843,10 @@ BACKGROUND_WORKER_COUNT = (
else 1
)
# If running with SQLite, limit background worker threads to 1 to prevent database locking issues
if 'sqlite' in DB_ENGINE:
BACKGROUND_WORKER_COUNT = 1
# django-q background worker configuration
Q_CLUSTER = {
'name': 'InvenTree',

5
src/backend/InvenTree/build/test_api.py
View File

@@ -1564,12 +1564,13 @@ class BuildLineTests(BuildAPITest):
# Filter by 'available' status
# Note: The max_query_time is bumped up here, as postgresql backend has some strange issues (only during testing)
response = self.get(url, data={'available': True}, max_query_time=15)
# TODO: This needs to be addressed in the future, as 25 seconds is an unacceptably long time for a query to take in testing
response = self.get(url, data={'available': True}, max_query_time=25)
n_t = len(response.data)
self.assertGreater(n_t, 0)
# Note: The max_query_time is bumped up here, as postgresql backend has some strange issues (only during testing)
response = self.get(url, data={'available': False}, max_query_time=15)
response = self.get(url, data={'available': False}, max_query_time=25)
n_f = len(response.data)
self.assertGreater(n_f, 0)