Migrate plugin tables (#11648)

* Prevent creation of PluginConfig during migrations

* Refactor data import process

- Split into multiple separate steps

* Load plugins during data load / dump

- Required, otherwise we cannot dump the data

* Refactor export_records

- Use temporary file
- Cleanup docstring

* Force apps check on second validation step

* Improve import sequencing

* Update CI script

* Update migration docs

* CI pipeline for running import/export test

* Fix workflow naming

* Fix env vars

* Add placeholder script

* Fix matrix env vars

* Fix missing env var

* Install required packages

* Fix typo

* Tweak tasks.py

* Install dummy plugin as part of the

* Updated CI workflow

* Validate exported data

* Additional CI process

* Log mandatory plugins to INFO

* Force global setting

* Refactor CI pipeline

* Tweak file test

* Workflow updates

* Enable auto-update

* Test if import/export test should run

* Trigger if tasks.py changes

src/backend/InvenTree/InvenTree/ready.py

@@ -125,6 +125,7 @@ def isGeneratingSchema():
         'qcluster',
         'check',
         'shell',
+        'help',
     ]
 
     if any(cmd in sys.argv for cmd in excluded_commands):
@@ -132,12 +133,14 @@ def isGeneratingSchema():
 
     included_commands = [
         'schema',
+        'spectactular',
         # schema adjacent calls
         'export_settings_definitions',
         'export_tags',
         'export_filters',
         'export_report_context',
     ]
+
     if any(cmd in sys.argv for cmd in included_commands):
         return True
 
@@ -185,11 +188,38 @@ def isInMainThread():
     return not isInWorkerThread()
 
 
+def readOnlyCommands():
+    """Return a list of read-only management commands which should not trigger database writes."""
+    return [
+        'help',
+        'check',
+        'shell',
+        'sqlflush',
+        'list_apps',
+        'wait_for_db',
+        'spectactular',
+        'makemessages',
+        'collectstatic',
+        'showmigrations',
+        'compilemessages',
+    ]
+
+
+def isReadOnlyCommand():
+    """Return True if the current command is a read-only command, which should not trigger any database writes."""
+    return any(cmd in sys.argv for cmd in readOnlyCommands())
+
+
 def canAppAccessDatabase(
     allow_test: bool = False, allow_plugins: bool = False, allow_shell: bool = False
 ):
     """Returns True if the apps.py file can access database records.
 
+    Arguments:
+        allow_test: If True, override checks and allow database access during testing mode
+        allow_plugins: If True, override checks and allow database access during plugin loading
+        allow_shell: If True, override checks and allow database access during shell sessions
+
     There are some circumstances where we don't want the ready function in apps.py
     to touch the database
     """
@@ -198,7 +228,7 @@ def canAppAccessDatabase(
         return False
 
     # Prevent database access if we are importing data
-    if isImportingData():
+    if not allow_plugins and isImportingData():
         return False
 
     # Prevent database access if we are rebuilding data
@@ -212,13 +242,13 @@ def canAppAccessDatabase(
     # If any of the following management commands are being executed,
     # prevent custom "on load" code from running!
     excluded_commands = [
-        'check',
-        'createsuperuser',
-        'wait_for_db',
-        'makemessages',
         'compilemessages',
-        'spectactular',
+        'createsuperuser',
         'collectstatic',
+        'makemessages',
+        'spectactular',
+        'wait_for_db',
+        'check',
     ]
 
     if not allow_shell:

src/backend/InvenTree/InvenTree/settings.py

@@ -194,6 +194,9 @@ PLUGINS_MANDATORY = get_setting(
     'INVENTREE_PLUGINS_MANDATORY', 'plugins_mandatory', typecast=list, default_value=[]
 )
 
+if PLUGINS_MANDATORY:
+    logger.info('Mandatory plugins: %s', PLUGINS_MANDATORY)
+
 PLUGINS_INSTALL_DISABLED = get_boolean_setting(
     'INVENTREE_PLUGIN_NOINSTALL', 'plugin_noinstall', False
 )

src/backend/InvenTree/plugin/registry.py

@@ -222,13 +222,18 @@ class PluginsRegistry:
         import InvenTree.ready
         from plugin.models import PluginConfig
 
-        if InvenTree.ready.isImportingData():
-            return None
+        # Under certain circumstances, we want to avoid creating new PluginConfig instances in the database
+        can_create = (
+            InvenTree.ready.canAppAccessDatabase(
+                allow_plugins=False, allow_shell=True, allow_test=True
+            )
+            and not InvenTree.ready.isReadOnlyCommand()
+        )
 
         try:
             cfg = PluginConfig.objects.filter(key=slug).first()
 
-            if not cfg:
+            if not cfg and can_create:
                 logger.debug(
                     "get_plugin_config: Creating new PluginConfig for '%s'", slug
                 )

src/backend/InvenTree/report/apps.py

@@ -49,6 +49,9 @@ class ReportConfig(AppConfig):
         if not InvenTree.ready.canAppAccessDatabase(allow_test=False):
             return  # pragma: no cover
 
+        if InvenTree.ready.isReadOnlyCommand():
+            return  # pragma: no cover
+
         with maintenance_mode_on():
             try:
                 self.create_default_labels()

src/backend/InvenTree/users/models.py

@@ -24,7 +24,7 @@ from rest_framework.authtoken.models import Token as AuthToken
 import InvenTree.helpers
 import InvenTree.models
 from common.settings import get_global_setting
-from InvenTree.ready import isImportingData
+from InvenTree.ready import isImportingData, isReadOnlyCommand
 
 from .ruleset import RULESET_CHOICES, get_ruleset_models
 
@@ -463,7 +463,7 @@ class Owner(models.Model):
 def create_owner(sender, instance, **kwargs):
     """Callback function to create a new owner instance after either a new group or user instance is saved."""
     # Ignore during data import process to avoid data duplication
-    if not isImportingData():
+    if not isReadOnlyCommand() and not isImportingData():
         Owner.create(obj=instance)
 
 
@@ -600,8 +600,8 @@ class UserProfile(InvenTree.models.MetadataMixin):
 @receiver(post_save, sender=User)
 def create_or_update_user_profile(sender, instance, created, **kwargs):
     """Create or update user profile when user is saved."""
-    # Disable profile creation if importing data from file
-    if isImportingData():
+    # Disable profile creation if importing data from file or running a read-only command
+    if isReadOnlyCommand() or isImportingData():
         return
 
     if created: