From 633fbd37bd6f2c2365040de1a56fffb3fa650628 Mon Sep 17 00:00:00 2001 From: Oliver Date: Thu, 8 Feb 2024 12:47:49 +1100 Subject: [PATCH] Maintenance Mode Improvements (#6451) * Custom migration step in tasks.py - Add custom management command - Wraps migration step in maintenance mode * Rename custom management command to "runmigrations" - Add command to isRunningMigrations * Add new data checks * Update database readiness checks - Set maintenance mode while performing certain management commands * Remove unused import * Re-add syncdb command * Log warning msg * Catch another potential error vector --- InvenTree/InvenTree/backends.py | 2 +- .../management/commands/rebuild_models.py | 33 ++++++--- .../management/commands/runmigrations.py | 19 +++++ InvenTree/InvenTree/ready.py | 73 +++++++++++++------ InvenTree/InvenTree/tasks.py | 16 ++-- InvenTree/common/models.py | 14 ++++ InvenTree/label/apps.py | 13 ++-- InvenTree/report/apps.py | 50 +++++++------ tasks.py | 5 +- 9 files changed, 153 insertions(+), 72 deletions(-) create mode 100644 InvenTree/InvenTree/management/commands/runmigrations.py diff --git a/InvenTree/InvenTree/backends.py b/InvenTree/InvenTree/backends.py index 82c8376579..51fd6e7c18 100644 --- a/InvenTree/InvenTree/backends.py +++ b/InvenTree/InvenTree/backends.py @@ -8,7 +8,7 @@ from django.db.utils import IntegrityError, OperationalError, ProgrammingError from maintenance_mode.backends import AbstractStateBackend import common.models -import InvenTree.helpers +import InvenTree.ready logger = logging.getLogger('inventree') diff --git a/InvenTree/InvenTree/management/commands/rebuild_models.py b/InvenTree/InvenTree/management/commands/rebuild_models.py index 02af71f3a5..f90664fb5b 100644 --- a/InvenTree/InvenTree/management/commands/rebuild_models.py +++ b/InvenTree/InvenTree/management/commands/rebuild_models.py @@ -3,60 +3,73 @@ - This is crucial after importing any fixtures, etc """ +import logging + from django.core.management.base import BaseCommand +from maintenance_mode.core import maintenance_mode_on, set_maintenance_mode + +logger = logging.getLogger('inventree') + class Command(BaseCommand): """Rebuild all database models which leverage the MPTT structure.""" def handle(self, *args, **kwargs): """Rebuild all database models which leverage the MPTT structure.""" + with maintenance_mode_on(): + self.rebuild_models() + + set_maintenance_mode(False) + + def rebuild_models(self): + """Rebuild all MPTT models in the database.""" # Part model try: - print('Rebuilding Part objects') + logger.info('Rebuilding Part objects') from part.models import Part Part.objects.rebuild() except Exception: - print('Error rebuilding Part objects') + logger.info('Error rebuilding Part objects') # Part category try: - print('Rebuilding PartCategory objects') + logger.info('Rebuilding PartCategory objects') from part.models import PartCategory PartCategory.objects.rebuild() except Exception: - print('Error rebuilding PartCategory objects') + logger.info('Error rebuilding PartCategory objects') # StockItem model try: - print('Rebuilding StockItem objects') + logger.info('Rebuilding StockItem objects') from stock.models import StockItem StockItem.objects.rebuild() except Exception: - print('Error rebuilding StockItem objects') + logger.info('Error rebuilding StockItem objects') # StockLocation model try: - print('Rebuilding StockLocation objects') + logger.info('Rebuilding StockLocation objects') from stock.models import StockLocation StockLocation.objects.rebuild() except Exception: - print('Error rebuilding StockLocation objects') + logger.info('Error rebuilding StockLocation objects') # Build model try: - print('Rebuilding Build objects') + logger.info('Rebuilding Build objects') from build.models import Build Build.objects.rebuild() except Exception: - print('Error rebuilding Build objects') + logger.info('Error rebuilding Build objects') diff --git a/InvenTree/InvenTree/management/commands/runmigrations.py b/InvenTree/InvenTree/management/commands/runmigrations.py new file mode 100644 index 0000000000..b06971724c --- /dev/null +++ b/InvenTree/InvenTree/management/commands/runmigrations.py @@ -0,0 +1,19 @@ +"""Check if there are any pending database migrations, and run them.""" + +import logging + +from django.core.management.base import BaseCommand + +from InvenTree.tasks import check_for_migrations + +logger = logging.getLogger('inventree') + + +class Command(BaseCommand): + """Check if there are any pending database migrations, and run them.""" + + def handle(self, *args, **kwargs): + """Check for any pending database migrations.""" + logger.info('Checking for pending database migrations') + check_for_migrations(force=True, reload_registry=False) + logger.info('Database migrations complete') diff --git a/InvenTree/InvenTree/ready.py b/InvenTree/InvenTree/ready.py index 1f942f3637..b8ebfb6e0a 100644 --- a/InvenTree/InvenTree/ready.py +++ b/InvenTree/InvenTree/ready.py @@ -10,13 +10,45 @@ def isInTestMode(): def isImportingData(): - """Returns True if the database is currently importing data, e.g. 'loaddata' command is performed.""" - return 'loaddata' in sys.argv + """Returns True if the database is currently importing (or exporting) data, e.g. 'loaddata' command is performed.""" + return any((x in sys.argv for x in ['flush', 'loaddata', 'dumpdata'])) def isRunningMigrations(): """Return True if the database is currently running migrations.""" - return any((x in sys.argv for x in ['migrate', 'makemigrations', 'showmigrations'])) + return any( + ( + x in sys.argv + for x in ['migrate', 'makemigrations', 'showmigrations', 'runmigrations'] + ) + ) + + +def isRebuildingData(): + """Return true if any of the rebuilding commands are being executed.""" + return any( + ( + x in sys.argv + for x in ['prerender', 'rebuild_models', 'rebuild_thumbnails', 'rebuild'] + ) + ) + + +def isRunningBackup(): + """Return true if any of the backup commands are being executed.""" + return any( + ( + x in sys.argv + for x in [ + 'backup', + 'restore', + 'dbbackup', + 'dbresotore', + 'mediabackup', + 'mediarestore', + ] + ) + ) def isInWorkerThread(): @@ -58,26 +90,30 @@ def canAppAccessDatabase( There are some circumstances where we don't want the ready function in apps.py to touch the database """ + # Prevent database access if we are running backups + if isRunningBackup(): + return False + + # Prevent database access if we are importing data + if isImportingData(): + return False + + # Prevent database access if we are rebuilding data + if isRebuildingData(): + return False + + # Prevent database access if we are running migrations + if not allow_plugins and isRunningMigrations(): + return False + # If any of the following management commands are being executed, # prevent custom "on load" code from running! excluded_commands = [ - 'flush', - 'loaddata', - 'dumpdata', 'check', 'createsuperuser', 'wait_for_db', - 'prerender', - 'rebuild_models', - 'rebuild_thumbnails', 'makemessages', 'compilemessages', - 'backup', - 'dbbackup', - 'mediabackup', - 'restore', - 'dbrestore', - 'mediarestore', ] if not allow_shell: @@ -88,12 +124,7 @@ def canAppAccessDatabase( excluded_commands.append('test') if not allow_plugins: - excluded_commands.extend([ - 'makemigrations', - 'showmigrations', - 'migrate', - 'collectstatic', - ]) + excluded_commands.extend(['collectstatic']) for cmd in excluded_commands: if cmd in sys.argv: diff --git a/InvenTree/InvenTree/tasks.py b/InvenTree/InvenTree/tasks.py index 05731dc60e..e3581b7f3c 100644 --- a/InvenTree/InvenTree/tasks.py +++ b/InvenTree/InvenTree/tasks.py @@ -644,7 +644,7 @@ def get_migration_plan(): @scheduled_task(ScheduledTask.DAILY) -def check_for_migrations(): +def check_for_migrations(force: bool = False, reload_registry: bool = True): """Checks if migrations are needed. If the setting auto_update is enabled we will start updating. @@ -659,8 +659,9 @@ def check_for_migrations(): logger.info('Checking for pending database migrations') - # Force plugin registry reload - registry.check_reload() + if reload_registry: + # Force plugin registry reload + registry.check_reload() plan = get_migration_plan() @@ -674,7 +675,7 @@ def check_for_migrations(): set_pending_migrations(n) # Test if auto-updates are enabled - if not get_setting('INVENTREE_AUTO_UPDATE', 'auto_update'): + if not force and not get_setting('INVENTREE_AUTO_UPDATE', 'auto_update'): logger.info('Auto-update is disabled - skipping migrations') return @@ -706,6 +707,7 @@ def check_for_migrations(): set_maintenance_mode(False) logger.info('Manually released maintenance mode') - # We should be current now - triggering full reload to make sure all models - # are loaded fully in their new state. - registry.reload_plugins(full_reload=True, force_reload=True, collect=True) + if reload_registry: + # We should be current now - triggering full reload to make sure all models + # are loaded fully in their new state. + registry.reload_plugins(full_reload=True, force_reload=True, collect=True) diff --git a/InvenTree/common/models.py b/InvenTree/common/models.py index 7bde02d2f1..bbe4b8af21 100644 --- a/InvenTree/common/models.py +++ b/InvenTree/common/models.py @@ -677,6 +677,16 @@ class BaseInvenTreeSetting(models.Model): setting = cls(key=key, **kwargs) else: return + except (OperationalError, ProgrammingError): + if not key.startswith('_'): + logger.warning("Database is locked, cannot set setting '%s'", key) + # Likely the DB is locked - not much we can do here + return + except Exception as exc: + logger.exception( + "Error setting setting '%s' for %s: %s", key, str(cls), str(type(exc)) + ) + return # Enforce standard boolean representation if setting.is_bool(): @@ -703,6 +713,10 @@ class BaseInvenTreeSetting(models.Model): attempts=attempts - 1, **kwargs, ) + except (OperationalError, ProgrammingError): + logger.warning("Database is locked, cannot set setting '%s'", key) + # Likely the DB is locked - not much we can do here + pass except Exception as exc: # Some other error logger.exception( diff --git a/InvenTree/label/apps.py b/InvenTree/label/apps.py index d40752d7fa..114b2ae55a 100644 --- a/InvenTree/label/apps.py +++ b/InvenTree/label/apps.py @@ -12,6 +12,8 @@ from django.conf import settings from django.core.exceptions import AppRegistryNotReady from django.db.utils import IntegrityError, OperationalError, ProgrammingError +from maintenance_mode.core import maintenance_mode_on, set_maintenance_mode + import InvenTree.helpers import InvenTree.ready @@ -32,13 +34,10 @@ class LabelConfig(AppConfig): ): return - if InvenTree.ready.isRunningMigrations(): - return + if not InvenTree.ready.canAppAccessDatabase(allow_test=False): + return # pragma: no cover - if ( - InvenTree.ready.canAppAccessDatabase(allow_test=False) - and not InvenTree.ready.isImportingData() - ): + with maintenance_mode_on(): try: self.create_labels() # pragma: no cover except ( @@ -52,6 +51,8 @@ class LabelConfig(AppConfig): 'Database was not ready for creating labels', stacklevel=2 ) + set_maintenance_mode(False) + def create_labels(self): """Create all default templates.""" # Test if models are ready diff --git a/InvenTree/report/apps.py b/InvenTree/report/apps.py index f5b71ace0a..634f9c10aa 100644 --- a/InvenTree/report/apps.py +++ b/InvenTree/report/apps.py @@ -11,6 +11,8 @@ from django.conf import settings from django.core.exceptions import AppRegistryNotReady from django.db.utils import IntegrityError, OperationalError, ProgrammingError +from maintenance_mode.core import maintenance_mode_on, set_maintenance_mode + import InvenTree.helpers logger = logging.getLogger('inventree') @@ -32,36 +34,36 @@ class ReportConfig(AppConfig): ): return - if InvenTree.ready.isRunningMigrations(): - return + if not InvenTree.ready.canAppAccessDatabase(allow_test=False): + return # pragma: no cover # Configure logging for PDF generation (disable "info" messages) logging.getLogger('fontTools').setLevel(logging.WARNING) logging.getLogger('weasyprint').setLevel(logging.WARNING) - # Create entries for default report templates - if ( - InvenTree.ready.canAppAccessDatabase(allow_test=False) - and not InvenTree.ready.isImportingData() + with maintenance_mode_on(): + self.create_reports() + + set_maintenance_mode(False) + + def create_reports(self): + """Create default report templates.""" + try: + self.create_default_test_reports() + self.create_default_build_reports() + self.create_default_bill_of_materials_reports() + self.create_default_purchase_order_reports() + self.create_default_sales_order_reports() + self.create_default_return_order_reports() + self.create_default_stock_location_reports() + except ( + AppRegistryNotReady, + IntegrityError, + OperationalError, + ProgrammingError, ): - try: - self.create_default_test_reports() - self.create_default_build_reports() - self.create_default_bill_of_materials_reports() - self.create_default_purchase_order_reports() - self.create_default_sales_order_reports() - self.create_default_return_order_reports() - self.create_default_stock_location_reports() - except ( - AppRegistryNotReady, - IntegrityError, - OperationalError, - ProgrammingError, - ): - # Database might not yet be ready - warnings.warn( - 'Database was not ready for creating reports', stacklevel=2 - ) + # Database might not yet be ready + warnings.warn('Database was not ready for creating reports', stacklevel=2) def create_default_reports(self, model, reports): """Copy default report files across to the media directory.""" diff --git a/tasks.py b/tasks.py index 2ca62f074d..0bd1254937 100644 --- a/tasks.py +++ b/tasks.py @@ -369,10 +369,9 @@ def migrate(c): print('Running InvenTree database migrations...') print('========================================') - manage(c, 'makemigrations') - manage(c, 'migrate --noinput') + # Run custom management command which wraps migrations in "maintenance mode" + manage(c, 'runmigrations', pty=True) manage(c, 'migrate --run-syncdb') - manage(c, 'check') print('========================================') print('InvenTree database migrations completed!')