Maintenance Mode Improvements (#6451)

* Custom migration step in tasks.py

- Add custom management command
- Wraps migration step in maintenance mode

* Rename custom management command to "runmigrations"

- Add command to isRunningMigrations

* Add new data checks

* Update database readiness checks

- Set maintenance mode while performing certain management commands

* Remove unused import

* Re-add syncdb command

* Log warning msg

* Catch another potential error vector

InvenTree/InvenTree/backends.py

@@ -8,7 +8,7 @@ from django.db.utils import IntegrityError, OperationalError, ProgrammingError
 from maintenance_mode.backends import AbstractStateBackend
 
 import common.models
-import InvenTree.helpers
+import InvenTree.ready
 
 logger = logging.getLogger('inventree')
 

InvenTree/InvenTree/management/commands/rebuild_models.py

@@ -3,60 +3,73 @@
 - This is crucial after importing any fixtures, etc
 """
 
+import logging
+
 from django.core.management.base import BaseCommand
 
+from maintenance_mode.core import maintenance_mode_on, set_maintenance_mode
+
+logger = logging.getLogger('inventree')
+
 
 class Command(BaseCommand):
     """Rebuild all database models which leverage the MPTT structure."""
 
     def handle(self, *args, **kwargs):
         """Rebuild all database models which leverage the MPTT structure."""
+        with maintenance_mode_on():
+            self.rebuild_models()
+
+        set_maintenance_mode(False)
+
+    def rebuild_models(self):
+        """Rebuild all MPTT models in the database."""
         # Part model
         try:
-            print('Rebuilding Part objects')
+            logger.info('Rebuilding Part objects')
 
             from part.models import Part
 
             Part.objects.rebuild()
         except Exception:
-            print('Error rebuilding Part objects')
+            logger.info('Error rebuilding Part objects')
 
         # Part category
         try:
-            print('Rebuilding PartCategory objects')
+            logger.info('Rebuilding PartCategory objects')
 
             from part.models import PartCategory
 
             PartCategory.objects.rebuild()
         except Exception:
-            print('Error rebuilding PartCategory objects')
+            logger.info('Error rebuilding PartCategory objects')
 
         # StockItem model
         try:
-            print('Rebuilding StockItem objects')
+            logger.info('Rebuilding StockItem objects')
 
             from stock.models import StockItem
 
             StockItem.objects.rebuild()
         except Exception:
-            print('Error rebuilding StockItem objects')
+            logger.info('Error rebuilding StockItem objects')
 
         # StockLocation model
         try:
-            print('Rebuilding StockLocation objects')
+            logger.info('Rebuilding StockLocation objects')
 
             from stock.models import StockLocation
 
             StockLocation.objects.rebuild()
         except Exception:
-            print('Error rebuilding StockLocation objects')
+            logger.info('Error rebuilding StockLocation objects')
 
         # Build model
         try:
-            print('Rebuilding Build objects')
+            logger.info('Rebuilding Build objects')
 
             from build.models import Build
 
             Build.objects.rebuild()
         except Exception:
-            print('Error rebuilding Build objects')
+            logger.info('Error rebuilding Build objects')

InvenTree/InvenTree/management/commands/runmigrations.py

@@ -0,0 +1,19 @@
+"""Check if there are any pending database migrations, and run them."""
+
+import logging
+
+from django.core.management.base import BaseCommand
+
+from InvenTree.tasks import check_for_migrations
+
+logger = logging.getLogger('inventree')
+
+
+class Command(BaseCommand):
+    """Check if there are any pending database migrations, and run them."""
+
+    def handle(self, *args, **kwargs):
+        """Check for any pending database migrations."""
+        logger.info('Checking for pending database migrations')
+        check_for_migrations(force=True, reload_registry=False)
+        logger.info('Database migrations complete')

InvenTree/InvenTree/ready.py

@@ -10,13 +10,45 @@ def isInTestMode():
 
 
 def isImportingData():
-    """Returns True if the database is currently importing data, e.g. 'loaddata' command is performed."""
+    """Returns True if the database is currently importing (or exporting) data, e.g. 'loaddata' command is performed."""
-    return 'loaddata' in sys.argv
+    return any((x in sys.argv for x in ['flush', 'loaddata', 'dumpdata']))
 
 
 def isRunningMigrations():
     """Return True if the database is currently running migrations."""
-    return any((x in sys.argv for x in ['migrate', 'makemigrations', 'showmigrations']))
+    return any(
+        (
+            x in sys.argv
+            for x in ['migrate', 'makemigrations', 'showmigrations', 'runmigrations']
+        )
+    )
+
+
+def isRebuildingData():
+    """Return true if any of the rebuilding commands are being executed."""
+    return any(
+        (
+            x in sys.argv
+            for x in ['prerender', 'rebuild_models', 'rebuild_thumbnails', 'rebuild']
+        )
+    )
+
+
+def isRunningBackup():
+    """Return true if any of the backup commands are being executed."""
+    return any(
+        (
+            x in sys.argv
+            for x in [
+                'backup',
+                'restore',
+                'dbbackup',
+                'dbresotore',
+                'mediabackup',
+                'mediarestore',
+            ]
+        )
+    )
 
 
 def isInWorkerThread():
@@ -58,26 +90,30 @@ def canAppAccessDatabase(
     There are some circumstances where we don't want the ready function in apps.py
     to touch the database
     """
+    # Prevent database access if we are running backups
+    if isRunningBackup():
+        return False
+
+    # Prevent database access if we are importing data
+    if isImportingData():
+        return False
+
+    # Prevent database access if we are rebuilding data
+    if isRebuildingData():
+        return False
+
+    # Prevent database access if we are running migrations
+    if not allow_plugins and isRunningMigrations():
+        return False
+
     # If any of the following management commands are being executed,
     # prevent custom "on load" code from running!
     excluded_commands = [
-        'flush',
-        'loaddata',
-        'dumpdata',
         'check',
         'createsuperuser',
         'wait_for_db',
-        'prerender',
-        'rebuild_models',
-        'rebuild_thumbnails',
         'makemessages',
         'compilemessages',
-        'backup',
-        'dbbackup',
-        'mediabackup',
-        'restore',
-        'dbrestore',
-        'mediarestore',
     ]
 
     if not allow_shell:
@@ -88,12 +124,7 @@ def canAppAccessDatabase(
         excluded_commands.append('test')
 
     if not allow_plugins:
-        excluded_commands.extend([
+        excluded_commands.extend(['collectstatic'])
-            'makemigrations',
-            'showmigrations',
-            'migrate',
-            'collectstatic',
-        ])
 
     for cmd in excluded_commands:
         if cmd in sys.argv:

InvenTree/InvenTree/tasks.py

@@ -644,7 +644,7 @@ def get_migration_plan():
 
 
 @scheduled_task(ScheduledTask.DAILY)
-def check_for_migrations():
+def check_for_migrations(force: bool = False, reload_registry: bool = True):
     """Checks if migrations are needed.
 
     If the setting auto_update is enabled we will start updating.
@@ -659,6 +659,7 @@ def check_for_migrations():
 
     logger.info('Checking for pending database migrations')
 
+    if reload_registry:
         # Force plugin registry reload
         registry.check_reload()
 
@@ -674,7 +675,7 @@ def check_for_migrations():
     set_pending_migrations(n)
 
     # Test if auto-updates are enabled
-    if not get_setting('INVENTREE_AUTO_UPDATE', 'auto_update'):
+    if not force and not get_setting('INVENTREE_AUTO_UPDATE', 'auto_update'):
         logger.info('Auto-update is disabled - skipping migrations')
         return
 
@@ -706,6 +707,7 @@ def check_for_migrations():
         set_maintenance_mode(False)
         logger.info('Manually released maintenance mode')
 
+    if reload_registry:
         # We should be current now - triggering full reload to make sure all models
         # are loaded fully in their new state.
         registry.reload_plugins(full_reload=True, force_reload=True, collect=True)

InvenTree/common/models.py

@@ -677,6 +677,16 @@ class BaseInvenTreeSetting(models.Model):
                 setting = cls(key=key, **kwargs)
             else:
                 return
+        except (OperationalError, ProgrammingError):
+            if not key.startswith('_'):
+                logger.warning("Database is locked, cannot set setting '%s'", key)
+            # Likely the DB is locked - not much we can do here
+            return
+        except Exception as exc:
+            logger.exception(
+                "Error setting setting '%s' for %s: %s", key, str(cls), str(type(exc))
+            )
+            return
 
         # Enforce standard boolean representation
         if setting.is_bool():
@@ -703,6 +713,10 @@ class BaseInvenTreeSetting(models.Model):
                     attempts=attempts - 1,
                     **kwargs,
                 )
+        except (OperationalError, ProgrammingError):
+            logger.warning("Database is locked, cannot set setting '%s'", key)
+            # Likely the DB is locked - not much we can do here
+            pass
         except Exception as exc:
             # Some other error
             logger.exception(

InvenTree/label/apps.py

@@ -12,6 +12,8 @@ from django.conf import settings
 from django.core.exceptions import AppRegistryNotReady
 from django.db.utils import IntegrityError, OperationalError, ProgrammingError
 
+from maintenance_mode.core import maintenance_mode_on, set_maintenance_mode
+
 import InvenTree.helpers
 import InvenTree.ready
 
@@ -32,13 +34,10 @@ class LabelConfig(AppConfig):
         ):
             return
 
-        if InvenTree.ready.isRunningMigrations():
+        if not InvenTree.ready.canAppAccessDatabase(allow_test=False):
-            return
+            return  # pragma: no cover
 
-        if (
+        with maintenance_mode_on():
-            InvenTree.ready.canAppAccessDatabase(allow_test=False)
-            and not InvenTree.ready.isImportingData()
-        ):
             try:
                 self.create_labels()  # pragma: no cover
             except (
@@ -52,6 +51,8 @@ class LabelConfig(AppConfig):
                     'Database was not ready for creating labels', stacklevel=2
                 )
 
+        set_maintenance_mode(False)
+
     def create_labels(self):
         """Create all default templates."""
         # Test if models are ready

InvenTree/report/apps.py

@@ -11,6 +11,8 @@ from django.conf import settings
 from django.core.exceptions import AppRegistryNotReady
 from django.db.utils import IntegrityError, OperationalError, ProgrammingError
 
+from maintenance_mode.core import maintenance_mode_on, set_maintenance_mode
+
 import InvenTree.helpers
 
 logger = logging.getLogger('inventree')
@@ -32,18 +34,20 @@ class ReportConfig(AppConfig):
         ):
             return
 
-        if InvenTree.ready.isRunningMigrations():
+        if not InvenTree.ready.canAppAccessDatabase(allow_test=False):
-            return
+            return  # pragma: no cover
 
         # Configure logging for PDF generation (disable "info" messages)
         logging.getLogger('fontTools').setLevel(logging.WARNING)
         logging.getLogger('weasyprint').setLevel(logging.WARNING)
 
-        # Create entries for default report templates
+        with maintenance_mode_on():
-        if (
+            self.create_reports()
-            InvenTree.ready.canAppAccessDatabase(allow_test=False)
+
-            and not InvenTree.ready.isImportingData()
+        set_maintenance_mode(False)
-        ):
+
+    def create_reports(self):
+        """Create default report templates."""
         try:
             self.create_default_test_reports()
             self.create_default_build_reports()
@@ -59,9 +63,7 @@ class ReportConfig(AppConfig):
             ProgrammingError,
         ):
             # Database might not yet be ready
-                warnings.warn(
+            warnings.warn('Database was not ready for creating reports', stacklevel=2)
-                    'Database was not ready for creating reports', stacklevel=2
-                )
 
     def create_default_reports(self, model, reports):
         """Copy default report files across to the media directory."""

tasks.py

@@ -369,10 +369,9 @@ def migrate(c):
     print('Running InvenTree database migrations...')
     print('========================================')
 
-    manage(c, 'makemigrations')
+    # Run custom management command which wraps migrations in "maintenance mode"
-    manage(c, 'migrate --noinput')
+    manage(c, 'runmigrations', pty=True)
     manage(c, 'migrate --run-syncdb')
-    manage(c, 'check')
 
     print('========================================')
     print('InvenTree database migrations completed!')