Auto migrating (#3741)

* base structure for updates

* add base structure

* add settingscheck

* update docstring

* only load plugins if needed

* fix misstyping

* run migration

* check if there are open migrations

* log open migration

* add more logging

* patch in fore reloading on unload

* only run if database is ready

* check every 5 minutes

* remove non implemented feautres  from desc

* add command flag to makr if cmmand runs as worker

* Add tests for migrations

* factor mmigration plan into own function

* Add print statements

* add initial migrations for tests

* remove last assertation

* cleanup migrations after run

* add flag to accept empty source code files

* the flag is enough for reporting

* fix test

* do not run migrations on sqlite3

* make sure migrations don't fail if no plan ran

* increase coverage for  migration

* spell fix

* check for migrations daily

* add a migration check after plugins are installed

InvenTree/InvenTree/apps.py

@@ -12,10 +12,9 @@ from django.db import transaction
 from django.db.utils import IntegrityError
 
 import InvenTree.tasks
+from InvenTree.config import get_setting
 from InvenTree.ready import canAppAccessDatabase, isInTestMode
 
-from .config import get_setting
-
 logger = logging.getLogger("inventree")
 
 
@@ -24,8 +23,18 @@ class InvenTreeConfig(AppConfig):
     name = 'InvenTree'
 
     def ready(self):
-        """Setup background tasks and update exchange rates."""
+        """Run system wide setup init steps.
+
+        Like:
+        - Checking if migrations should be run
+        - Cleaning up tasks
+        - Starting regular tasks
+        - Updateing exchange rates
+        - Collecting notification mehods
+        - Adding users set in the current environment
+        """
         if canAppAccessDatabase() or settings.TESTING_ENV:
+            InvenTree.tasks.check_for_migrations(worker=False)
 
             self.remove_obsolete_tasks()
 

InvenTree/InvenTree/migrations/0001_initial.py

@@ -0,0 +1,12 @@
+# Generated by Django 3.2.15 on 2022-10-03 18:57
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+    ]
+
+    operations = [
+    ]

InvenTree/InvenTree/tasks.py

@@ -15,10 +15,17 @@ from django.conf import settings
 from django.core import mail as django_mail
 from django.core.exceptions import AppRegistryNotReady
 from django.core.management import call_command
-from django.db.utils import OperationalError, ProgrammingError
+from django.db import DEFAULT_DB_ALIAS, connections
+from django.db.migrations.executor import MigrationExecutor
+from django.db.utils import (NotSupportedError, OperationalError,
+                             ProgrammingError)
 from django.utils import timezone
 
 import requests
+from maintenance_mode.core import (get_maintenance_mode, maintenance_mode_on,
+                                   set_maintenance_mode)
+
+from InvenTree.config import get_setting
 
 logger = logging.getLogger("inventree")
 
@@ -506,3 +513,74 @@ def send_email(subject, body, recipients, from_email=None, html_message=None):
         fail_silently=False,
         html_message=html_message
     )
+
+
+@scheduled_task(ScheduledTask.DAILY)
+def check_for_migrations(worker: bool = True):
+    """Checks if migrations are needed.
+
+    If the setting auto_update is enabled we will start updateing.
+    """
+    # Test if auto-updates are enabled
+    if not get_setting('INVENTREE_AUTO_UPDATE', 'auto_update'):
+        return
+
+    from plugin import registry
+
+    plan = get_migration_plan()
+
+    # Check if there are any open migrations
+    if not plan:
+        logger.info('There are no open migrations')
+        return
+
+    logger.info('There are open migrations')
+
+    # Log open migrations
+    for migration in plan:
+        logger.info(migration[0])
+
+    # Set the application to maintenance mode - no access from now on.
+    logger.info('Going into maintenance')
+    set_maintenance_mode(True)
+    logger.info('Mainentance mode is on now')
+
+    # Check if we are worker - go kill all other workers then.
+    # Only the frontend workers run updates.
+    if worker:
+        logger.info('Current process is a worker - shutting down cluster')
+
+    # Ok now we are ready to go ahead!
+    # To be sure we are in maintenance this is wrapped
+    with maintenance_mode_on():
+        logger.info('Starting migrations')
+        print('Starting migrations')
+
+        try:
+            call_command('migrate', interactive=False)
+        except NotSupportedError as e:  # pragma: no cover
+            if settings.DATABASES['default']['ENGINE'] != 'django.db.backends.sqlite3':
+                raise e
+            logger.error(f'Error during migrations: {e}')
+
+        print('Migrations done')
+        logger.info('Ran migrations')
+
+    # Make sure we are out of maintenance again
+    logger.info('Checking InvenTree left maintenance mode')
+    if get_maintenance_mode():
+
+        logger.warning('Mainentance was still on - releasing now')
+        set_maintenance_mode(False)
+        logger.info('Released out of maintenance')
+
+    # We should be current now - triggering full reload to make sure all models
+    # are loaded fully in their new state.
+    registry.reload_plugins(full_reload=True, force_reload=True)
+
+
+def get_migration_plan():
+    """Returns a list of migrations which are needed to be run."""
+    executor = MigrationExecutor(connections[DEFAULT_DB_ALIAS])
+    plan = executor.migration_plan(executor.loader.graph.leaf_nodes())
+    return plan

InvenTree/InvenTree/test_tasks.py

@@ -1,7 +1,11 @@
 """Unit tests for task management."""
 
+import os
 from datetime import timedelta
 
+from django.conf import settings
+from django.core.management import call_command
+from django.db.utils import NotSupportedError
 from django.test import TestCase
 from django.utils import timezone
 
@@ -117,3 +121,32 @@ class InvenTreeTaskTests(TestCase):
         response = InvenTreeSetting.get_setting('_INVENTREE_LATEST_VERSION')
         self.assertNotEqual(response, '')
         self.assertTrue(bool(response))
+
+    def test_task_check_for_migrations(self):
+        """Test the task check_for_migrations."""
+        # Update disabled
+        InvenTree.tasks.check_for_migrations()
+
+        # Update enabled - no migrations
+        os.environ['INVENTREE_AUTO_UPDATE'] = 'True'
+        InvenTree.tasks.check_for_migrations()
+
+        # Create migration
+        self.assertEqual(len(InvenTree.tasks.get_migration_plan()), 0)
+        call_command('makemigrations', ['InvenTree', '--empty'], interactive=False)
+        self.assertEqual(len(InvenTree.tasks.get_migration_plan()), 1)
+
+        # Run with migrations - catch no foreigner error
+        try:
+            InvenTree.tasks.check_for_migrations()
+        except NotSupportedError as e:  # pragma: no cover
+            if settings.DATABASES['default']['ENGINE'] != 'django.db.backends.sqlite3':
+                raise e
+
+        # Cleanup
+        try:
+            migration_name = InvenTree.tasks.get_migration_plan()[0][0].name + '.py'
+            migration_path = settings.BASE_DIR / 'InvenTree' / 'migrations' / migration_name
+            migration_path.unlink()
+        except IndexError:  # pragma: no cover
+            pass

InvenTree/config_template.yaml

@@ -119,6 +119,10 @@ plugins_enabled: False
 #plugin_file: '/path/to/plugins.txt'
 #plugin_dir: '/path/to/plugins/'
 
+# Set this variable to True to enable auto-migrations
+# Alternatively, use the environment variable INVENTREE_AUTO_UPDATE
+auto_update: False
+
 # Allowed hosts (see ALLOWED_HOSTS in Django settings documentation)
 # A list of strings representing the host/domain names that this Django site can serve.
 # Default behaviour is to allow all hosts (THIS IS NOT SECURE!)

InvenTree/plugin/registry.py

@@ -162,8 +162,12 @@ class PluginsRegistry:
 
         logger.info('Finished loading plugins')
 
-    def unload_plugins(self):
+    def unload_plugins(self, force_reload: bool = False):
-        """Unload and deactivate all IntegrationPlugins."""
+        """Unload and deactivate all IntegrationPlugins.
+
+        Args:
+            force_reload (bool, optional): Also reload base apps. Defaults to False.
+        """
 
         logger.info('Start unloading plugins')
 
@@ -176,7 +180,7 @@ class PluginsRegistry:
         self._clean_registry()
 
         # deactivate all integrations
-        self._deactivate_plugins()
+        self._deactivate_plugins(force_reload=force_reload)
 
         # remove maintenance
         if not _maintenance:
@@ -184,11 +188,12 @@ class PluginsRegistry:
 
         logger.info('Finished unloading plugins')
 
-    def reload_plugins(self, full_reload: bool = False):
+    def reload_plugins(self, full_reload: bool = False, force_reload: bool = False):
         """Safely reload.
 
         Args:
             full_reload (bool, optional): Reload everything - including plugin mechanism. Defaults to False.
+            force_reload (bool, optional): Also reload base apps. Defaults to False.
         """
         # Do not reload whe currently loading
         if self.is_loading:
@@ -197,8 +202,8 @@ class PluginsRegistry:
         logger.info('Start reloading plugins')
 
         with maintenance_mode_on():
-            self.unload_plugins()
+            self.unload_plugins(force_reload=force_reload)
-            self.load_plugins(full_reload)
+            self.load_plugins(full_reload=full_reload)
 
         logger.info('Finished reloading plugins')
 
@@ -470,9 +475,13 @@ class PluginsRegistry:
         self.activate_plugin_app(plugins, force_reload=force_reload, full_reload=full_reload)
         self.activate_plugin_url(plugins, force_reload=force_reload, full_reload=full_reload)
 
-    def _deactivate_plugins(self):
+    def _deactivate_plugins(self, force_reload: bool = False):
-        """Run deactivation functions for all plugins."""
+        """Run deactivation functions for all plugins.
-        self.deactivate_plugin_app()
+
+        Args:
+            force_reload (bool, optional): Also reload base apps. Defaults to False.
+        """
+        self.deactivate_plugin_app(force_reload=force_reload)
         self.deactivate_plugin_schedule()
         self.deactivate_plugin_settings()
     # endregion
@@ -655,8 +664,12 @@ class PluginsRegistry:
             plugin_path = plugin.__module__.split('.')[0]
         return plugin_path
 
-    def deactivate_plugin_app(self):
+    def deactivate_plugin_app(self, force_reload: bool = False):
-        """Deactivate AppMixin plugins - some magic required."""
+        """Deactivate AppMixin plugins - some magic required.
+
+        Args:
+            force_reload (bool, optional): Also reload base apps. Defaults to False.
+        """
         # unregister models from admin
         for plugin_path in self.installed_apps:
             models = []  # the modelrefs need to be collected as poping an item in a iter is not welcomed
@@ -693,7 +706,7 @@ class PluginsRegistry:
 
         # reset load flag and reload apps
         settings.INTEGRATION_APPS_LOADED = False
-        self._reload_apps()
+        self._reload_apps(force_reload=force_reload)
 
         # update urls to remove the apps from the site admin
         self._update_urls()

InvenTree/plugin/serializers.py

@@ -10,6 +10,7 @@ from django.utils.translation import gettext_lazy as _
 from rest_framework import serializers
 
 from common.serializers import GenericReferencedSettingSerializer
+from InvenTree.tasks import check_for_migrations, offload_task
 from plugin.models import NotificationUserSetting, PluginConfig, PluginSetting
 
 
@@ -156,6 +157,9 @@ class PluginConfigInstallSerializer(serializers.Serializer):
             with open(settings.PLUGIN_FILE, "a") as plugin_file:
                 plugin_file.write(f'{" ".join(install_name)}  # Installed {timezone.now()} by {str(self.context["request"].user)}\n')
 
+        # Check for migrations
+        offload_task(check_for_migrations, worker=True)
+
         return ret
 
 

tasks.py

@@ -620,4 +620,4 @@ def coverage(c):
     ))
 
     # Generate coverage report
-    c.run('coverage html')
+    c.run('coverage html -i')