Enhancements for recort import/export (#11630)

* Add management command to list installed apps

* Add metadata to exported data file

* Validate metadata for imported file

* Update CHANGELOG.md

* Update docs

* Use internal codes

* Refactor and add more metadata

* Adjust github action workflow

* Run with --force option to setup demo dataset

.github/actions/migration/action.yaml

@@ -13,5 +13,5 @@ runs:
           invoke export-records -f data.json
           python3 ./src/backend/InvenTree/manage.py flush --noinput
           invoke migrate
-          invoke import-records -c -f data.json
-          invoke import-records -c -f data.json
+          invoke import-records -c -f data.json --force --strict
+          invoke import-records -c -f data.json --force --strict

CHANGELOG.md

@@ -25,6 +25,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
+- [#11630](https://github.com/inventree/InvenTree/pull/11630) enhances the `import_records` and `export_records` system commands, by adding a metadata entry to the exported data file to allow for compatibility checks during data import.
+
 ### Removed
 
 - [#11581](https://github.com/inventree/InvenTree/pull/11581) removes the ability to specify arbitrary filters when performing bulk operations via the API. This functionality represented a significant security risk, and was not required for any existing use cases. Bulk operations now only work with a provided list of primary keys.

docs/docs/start/migrate.md

@@ -67,6 +67,10 @@ invoke import-records -c -f data.json
 !!! warning "Character Encoding"
 	If the character encoding of the data file does not exactly match the target database, the import operation may not succeed. In this case, some manual editing of the database JSON file may be required.
 
+```
+{{ invoke_commands('import-records --help') }}
+```
+
 ### Copy Media Files
 
 Any media files (images, documents, etc) that were stored in the original database must be copied to the new database. In a typical InvenTree installation, these files are stored in the `media` subdirectory of the InvenTree data location.

src/backend/InvenTree/InvenTree/management/commands/list_apps.py

@@ -0,0 +1,29 @@
+"""Custom management command to list all installed apps.
+
+This is used to determine which apps are installed,
+including any apps which are defined for plugins.
+"""
+
+from django.core.management.base import BaseCommand
+
+
+class Command(BaseCommand):
+    """List all installed apps."""
+
+    def handle(self, *args, **kwargs):
+        """List all installed apps.
+
+        Note that this function outputs in a particular format,
+        which is expected by the calling code in tasks.py
+        """
+        from django.apps import apps
+
+        app_list = []
+
+        for app in apps.get_app_configs():
+            app_list.append(app.name)
+
+        app_list.sort()
+
+        self.stdout.write(f'Installed Apps: {len(app_list)}')
+        self.stdout.write('>>> ' + ','.join(app_list) + ' <<<')

src/backend/InvenTree/InvenTree/ready.py

@@ -119,6 +119,7 @@ def isGeneratingSchema():
         'collectstatic',
         'makemessages',
         'wait_for_db',
+        'list_apps',
         'gunicorn',
         'sqlflush',
         'qcluster',

tasks.py

@@ -18,6 +18,14 @@ from invoke import Collection, task
 from invoke.exceptions import UnexpectedExit
 
 
+def safe_value(fnc):
+    """Helper function to safely get value from function, catching import exceptions."""
+    try:
+        return fnc()
+    except (ModuleNotFoundError, ImportError):
+        return wrap_color('N/A', '93')  # Yellow color for "Not Available"
+
+
 def is_true(x):
     """Shortcut function to determine if a value "looks" like a boolean."""
     return str(x).strip().lower() in ['1', 'y', 'yes', 't', 'true', 'on']
@@ -45,6 +53,51 @@ def is_debug_environment():
     )
 
 
+def get_django_version():
+    """Return the current Django version."""
+    from src.backend.InvenTree.InvenTree.version import (
+        inventreeDjangoVersion,  # type: ignore[import]
+    )
+
+    return safe_value(inventreeDjangoVersion)
+
+
+def get_inventree_version():
+    """Return the current InvenTree version."""
+    from src.backend.InvenTree.InvenTree.version import (
+        inventreeVersion,  # type: ignore[import]
+    )
+
+    return safe_value(inventreeVersion)
+
+
+def get_inventree_api_version():
+    """Return the current InvenTree API version."""
+    from src.backend.InvenTree.InvenTree.version import (
+        inventreeApiVersion,  # type: ignore[import]
+    )
+
+    return safe_value(inventreeApiVersion)
+
+
+def get_commit_hash():
+    """Return the current git commit hash."""
+    from src.backend.InvenTree.InvenTree.version import (
+        inventreeCommitHash,  # type: ignore[import]
+    )
+
+    return safe_value(inventreeCommitHash)
+
+
+def get_commit_date():
+    """Return the current git commit date."""
+    from src.backend.InvenTree.InvenTree.version import (
+        inventreeCommitDate,  # type: ignore[import]
+    )
+
+    return safe_value(inventreeCommitDate)
+
+
 def get_version_vals():
     """Get values from the VERSION file."""
     version_file = local_dir().joinpath('VERSION')
@@ -249,7 +302,7 @@ def main():
 # endregion
 
 
-def apps():
+def builtin_apps():
     """Returns a list of installed apps."""
     return [
         'build',
@@ -384,7 +437,9 @@ if __name__ in ['__main__', 'tasks']:
     main()
 
 
-def run(c, cmd, path: Optional[Path] = None, pty=False, env=None):
+def run(
+    c, cmd, path: Optional[Path] = None, pty: bool = False, hide: bool = False, env=None
+):
     """Runs a given command a given path.
 
     Args:
@@ -392,20 +447,23 @@ def run(c, cmd, path: Optional[Path] = None, pty=False, env=None):
         cmd: Command to run.
         path: Path to run the command in.
         pty (bool, optional): Run an interactive session. Defaults to False.
+        hide (bool, optional): Hide the command output. Defaults to False.
         env (dict, optional): Environment variables to pass to the command. Defaults to None.
     """
     env = env or {}
     path = path or local_dir()
 
     try:
-        c.run(f'cd "{path}" && {cmd}', pty=pty, env=env)
+        result = c.run(f'cd "{path}" && {cmd}', pty=pty, env=env, hide=hide)
     except UnexpectedExit as e:
         error(f"ERROR: InvenTree command failed: '{cmd}'")
         warning('- Refer to the error messages in the log above for more information')
         raise e
 
+    return result
 
-def manage(c, cmd, pty: bool = False, env=None):
+
+def manage(c, cmd, pty: bool = False, env=None, **kwargs):
     """Runs a given command against django's "manage.py" script.
 
     Args:
@@ -414,7 +472,23 @@ def manage(c, cmd, pty: bool = False, env=None):
         pty (bool, optional): Run an interactive session. Defaults to False.
         env (dict, optional): Environment variables to pass to the command. Defaults to None.
     """
-    run(c, f'python3 manage.py {cmd}', manage_py_dir(), pty, env)
+    return run(
+        c, f'python3 manage.py {cmd}', manage_py_dir(), pty=pty, env=env, **kwargs
+    )
+
+
+def installed_apps(c) -> list[str]:
+    """Returns a list of all installed apps, including plugins."""
+    result = manage(c, 'list_apps', pty=False, hide=True)
+    output = result.stdout.strip()
+
+    # Look for the expected pattern
+    match = re.findall(r'>>> (.*) <<<', output)
+
+    if not match:
+        raise ValueError(f"Unexpected output from 'list_apps' command: {output}")
+
+    return match[0].split(',')
 
 
 def run_install(
@@ -1030,7 +1104,20 @@ def export_records(
     with open(tmpfile, encoding='utf-8') as f_in:
         data = json.loads(f_in.read())
 
-    data_out = []
+    data_out = [
+        {
+            'metadata': True,
+            'comment': 'This file contains a dump of the InvenTree database',
+            'exported_at': datetime.datetime.now().isoformat(),
+            'exported_at_utc': datetime.datetime.utcnow().isoformat(),
+            'source_version': get_inventree_version(),
+            'api_version': get_inventree_api_version(),
+            'django_version': get_django_version(),
+            'python_version': python_version(),
+            'source_commit': get_commit_hash(),
+            'installed_apps': installed_apps(c),
+        }
+    ]
 
     for entry in data:
         model_name = entry.get('model', None)
@@ -1060,17 +1147,77 @@ def export_records(
     success('Data export completed')
 
 
+def validate_import_metadata(c, metadata: dict, strict: bool = False) -> bool:
+    """Validate the metadata associated with an import file.
+
+    Arguments:
+        c: The context or connection object
+        metadata (dict): The metadata to validate
+        strict (bool): If True, the import process will fail if any issues are detected.
+    """
+    info('Validating import metadata...')
+
+    valid = True
+
+    def metadata_issue(message: str):
+        """Handle an issue with the metadata."""
+        nonlocal valid
+        valid = False
+
+        if strict:
+            error(f'INVE-E16 Data Import Error: {message}')
+            sys.exit(1)
+        else:
+            warning(f'INVE-W13 Data Import Warning: {message}')
+
+    if not metadata:
+        metadata_issue(
+            'No metadata found in the import file - cannot validate source version'
+        )
+        return False
+
+    source_version = metadata.get('source_version')
+
+    if source_version != get_inventree_version():
+        metadata_issue(
+            f"Source version '{source_version}' does not match the current InvenTree version '{get_inventree_version()}' - this may lead to issues with the import process"
+        )
+
+    local_apps = set(installed_apps(c))
+    source_apps = set(metadata.get('installed_apps', []))
+
+    for app in source_apps:
+        if app not in local_apps:
+            metadata_issue(
+                f"Source app '{app}' is not installed in the current environment - this may break the import process"
+            )
+
+    if valid:
+        success('Metadata validation succeeded - no issues detected')
+
+    return valid
+
+
 @task(
     help={
         'filename': 'Input filename',
         'clear': 'Clear existing data before import',
+        'force': 'Force deletion of existing data without confirmation (only applies if --clear is set)',
+        'strict': 'Strict mode - fail if any issues are detected with the metadata (default = False)',
         'retain_temp': 'Retain temporary files at end of process (default = False)',
+        'ignore_nonexistent': 'Ignore non-existent database models (default = False)',
     },
     pre=[wait],
     post=[rebuild_models, rebuild_thumbnails],
 )
 def import_records(
-    c, filename='data.json', clear: bool = False, retain_temp: bool = False
+    c,
+    filename='data.json',
+    clear: bool = False,
+    retain_temp: bool = False,
+    strict: bool = False,
+    force: bool = False,
+    ignore_nonexistent: bool = False,
 ):
     """Import database records from a file."""
     # Get an absolute path to the supplied filename
@@ -1083,7 +1230,7 @@ def import_records(
         sys.exit(1)
 
     if clear:
-        delete_data(c, force=True, migrate=True)
+        delete_data(c, force=force, migrate=True)
 
     info(f"Importing database records from '{target}'")
 
@@ -1104,7 +1251,14 @@ def import_records(
     auth_data = []
     load_data = []
 
+    # A dict containing metadata associated with the data file
+    metadata = {}
+
     for entry in data:
+        if entry.get('metadata', False):
+            metadata = entry
+            continue
+
         if 'model' in entry:
             # Clear out any permissions specified for a group
             if entry['model'] == 'auth.group':
@@ -1123,6 +1277,9 @@ def import_records(
             warning('WARNING: Invalid entry in data file')
             print(entry)
 
+    # Check the metadata associated with the imported data
+    validate_import_metadata(c, metadata, strict=strict)
+
     # Write the auth file data
     with open(authfile, 'w', encoding='utf-8') as f_out:
         f_out.write(json.dumps(auth_data, indent=2))
@@ -1131,17 +1288,25 @@ def import_records(
     with open(datafile, 'w', encoding='utf-8') as f_out:
         f_out.write(json.dumps(load_data, indent=2))
 
+    # A set of content types to exclude from the import process
     excludes = content_excludes(allow_auth=False)
 
     # Import auth models first
     info('Importing user auth data...')
     cmd = f"loaddata '{authfile}'"
+
+    if ignore_nonexistent:
+        cmd += ' --ignorenonexistent'
+
     manage(c, cmd, pty=True)
 
     # Import everything else next
     info('Importing database records...')
     cmd = f"loaddata '{datafile}' -i {excludes}"
 
+    if ignore_nonexistent:
+        cmd += ' --ignorenonexistent'
+
     manage(c, cmd, pty=True)
 
     if not retain_temp:
@@ -1388,7 +1553,7 @@ def test(
 
     pty = not disable_pty
 
-    tested_apps = ' '.join(apps())
+    tested_apps = ' '.join(builtin_apps())
 
     cmd = 'test'
 
@@ -1475,7 +1640,9 @@ def setup_test(
 
     # Load data
     info('Loading database records ...')
-    import_records(c, filename=template_dir.joinpath('inventree_data.json'), clear=True)
+    import_records(
+        c, filename=template_dir.joinpath('inventree_data.json'), clear=True, force=True
+    )
 
     # Copy media files
     src = template_dir.joinpath('media')
@@ -1597,7 +1764,6 @@ def export_definitions(c, basedir: str = ''):
 @task(default=True)
 def version(c):
     """Show the current version of InvenTree."""
-    import src.backend.InvenTree.InvenTree.version as InvenTreeVersion  # type: ignore[import]
     from src.backend.InvenTree.InvenTree.config import (  # type: ignore[import]
         get_backup_dir,
         get_config_file,
@@ -1606,13 +1772,6 @@ def version(c):
         get_static_dir,
     )
 
-    def get_value(fnc):
-        """Helper function to safely get value from function, catching import exceptions."""
-        try:
-            return fnc()
-        except (ModuleNotFoundError, ImportError):
-            return wrap_color('ENVIRONMENT ERROR', '91')
-
     # Gather frontend version information
     _, node, yarn = node_available(versions=True)
 
@@ -1645,17 +1804,17 @@ Invoke Tool {invoke_path}
 
 Installation paths:
 Base        {local_dir()}
-Config      {get_value(get_config_file)}
-Plugin File {get_value(get_plugin_file) or NOT_SPECIFIED}
-Media       {get_value(lambda: get_media_dir(error=False)) or NOT_SPECIFIED}
-Static      {get_value(lambda: get_static_dir(error=False)) or NOT_SPECIFIED}
-Backup      {get_value(lambda: get_backup_dir(error=False)) or NOT_SPECIFIED}
+Config      {safe_value(get_config_file)}
+Plugin File {safe_value(get_plugin_file) or NOT_SPECIFIED}
+Media       {safe_value(lambda: get_media_dir(error=False)) or NOT_SPECIFIED}
+Static      {safe_value(lambda: get_static_dir(error=False)) or NOT_SPECIFIED}
+Backup      {safe_value(lambda: get_backup_dir(error=False)) or NOT_SPECIFIED}
 
 Versions:
-InvenTree   {InvenTreeVersion.inventreeVersion()}
-API         {InvenTreeVersion.inventreeApiVersion()}
+InvenTree   {get_inventree_version() or NOT_SPECIFIED}
+API         {get_inventree_api_version() or NOT_SPECIFIED}
 Python      {python_version()}
-Django      {get_value(InvenTreeVersion.inventreeDjangoVersion)}
+Django      {get_django_version() or NOT_SPECIFIED}
 Node        {node if node else NA}
 Yarn        {yarn if yarn else NA}
 
@@ -1663,8 +1822,8 @@ Environment:
 Platform    {platform}
 Debug       {is_debug_environment()}
 
-Commit hash: {InvenTreeVersion.inventreeCommitHash()}
-Commit date: {InvenTreeVersion.inventreeCommitDate()}"""
+Commit hash: {get_commit_hash() or NOT_SPECIFIED}
+Commit date: {get_commit_date() or NOT_SPECIFIED}"""
     )
     if is_pkg_installer_by_path():
         print(