Adds bulk-import option for "import-records" task (#12792)

* skip plugin events if importing or migrating data

* Add bulkloaddata option

* Skip signals if importing

* Improve bulkloaddata command

* Optionally rebuild thumbnails

* enhancements for import_records task

* cache natural key references in bulkloaddata

* wrap export_records in @state_logger

* Reduce file size of exported data

* Added docs

* Test bulk workflow as part of CI

* Add progress bar for data import

* fix for import workflow bug

* Separately test bulk import workflow

* Exercise --prettify option

* Additional CI checks for content excludes

* Allow plugin loading for list_apps

* Additional CI unit tests

* Test for importing with conflicting records

* path fixes

* Adjust test conditions

tasks.py

@@ -1205,9 +1205,11 @@ def update(
         'exclude_plugins': 'Exclude plugin data from the output file (default = False)',
         'include_sso': 'Include SSO token data in the output file (default = False)',
         'include_session': 'Include user session data in the output file (default = False)',
+        'prettify': 'Pretty-print the output file with indentation (default = False)',
         'verbose': 'Print verbose output from management commands',
     }
 )
+@state_logger
 def export_records(
     c,
     filename='data.json',
@@ -1218,6 +1220,7 @@ def export_records(
     exclude_plugins: bool = False,
     include_sso: bool = False,
     include_session: bool = False,
+    prettify: bool = False,
     verbose: bool = False,
 ):
     """Export all database records to a file."""
@@ -1243,7 +1246,10 @@ def export_records(
     with tempfile.NamedTemporaryFile(
         suffix='.json', encoding='utf-8', mode='w+t', delete=True
     ) as tmpfile:
-        cmd = f"dumpdata --natural-foreign --indent 2 --output '{tmpfile.name}' {excludes}"
+        cmd = f"dumpdata --natural-foreign --output '{tmpfile.name}' {excludes}"
+
+        if prettify:
+            cmd += ' --indent 2'
 
         # Dump data to temporary file
         manage(c, cmd, pty=True, verbose=verbose)
@@ -1258,7 +1264,7 @@ def export_records(
             'metadata': True,
             'comment': 'This file contains a dump of the InvenTree database',
             'exported_at': datetime.datetime.now().isoformat(),
-            'exported_at_utc': datetime.datetime.utcnow().isoformat(),
+            'exported_at_utc': datetime.datetime.now(datetime.UTC).isoformat(),
             'source_version': get_inventree_version(),
             'api_version': get_inventree_api_version(),
             'django_version': get_django_version(),
@@ -1287,7 +1293,7 @@ def export_records(
 
     # Write the processed data to file
     with open(target, 'w', encoding='utf-8') as f_out:
-        f_out.write(json.dumps(data_out, indent=2))
+        f_out.write(json.dumps(data_out, indent=2 if prettify else None))
 
     success('Data export completed')
 
@@ -1358,10 +1364,15 @@ def validate_import_metadata(
         'exclude_plugins': 'Exclude plugin data from the import process (default = False)',
         'skip_migrations': 'Skip the migration step after clearing data (default = False)',
         'verbose': 'Print verbose output from management commands',
+        'bulk': 'Use the faster bulkloaddata command instead of loaddata (default = False)',
+        'ignore_conflicts': 'Skip records that violate a unique constraint, instead of raising an error (requires --bulk, default = False)',
+        'rebuild_trees': 'Rebuild MPTT tree structures after import (default = True)',
+        'rebuild_images': 'Rebuild image thumbnails after import (default = True)',
     },
     pre=[wait],
-    post=[rebuild_models, rebuild_thumbnails],
+    post=[],
 )
+@state_logger
 def import_records(
     c,
     filename='data.json',
@@ -1371,6 +1382,10 @@ def import_records(
     ignore_nonexistent: bool = False,
     skip_migrations: bool = False,
     verbose: bool = False,
+    bulk: bool = False,
+    ignore_conflicts: bool = False,
+    rebuild_trees: bool = True,
+    rebuild_images: bool = True,
 ):
     """Import database records from a file."""
     # Get an absolute path to the supplied filename
@@ -1383,6 +1398,10 @@ def import_records(
         error(f"ERROR: File '{target}' does not exist")
         sys.exit(1)
 
+    if ignore_conflicts and not bulk:
+        warning('--ignore-conflicts has no effect without --bulk - ignoring')
+        ignore_conflicts = False
+
     if clear:
         delete_data(c, force=True, migrate=True, verbose=verbose)
 
@@ -1416,6 +1435,8 @@ def import_records(
         """Helper function to save data to a temporary file, and then load into the database."""
         nonlocal ignore_nonexistent
         nonlocal verbose
+        nonlocal bulk
+        nonlocal ignore_conflicts
         nonlocal c
 
         # Skip if there is no data to load
@@ -1429,7 +1450,9 @@ def import_records(
         ) as f_out:
             f_out.write(json.dumps(data, indent=2))
 
-        cmd = f'loaddata {f_out.name} -v 0 --force-color'
+        cmd = (
+            f'{"bulkloaddata" if bulk else "loaddata"} {f_out.name} -v 0 --force-color'
+        )
 
         if app:
             cmd += f' --app {app}'
@@ -1437,9 +1460,12 @@ def import_records(
         if ignore_nonexistent:
             cmd += ' --ignorenonexistent'
 
+        if bulk and ignore_conflicts:
+            cmd += ' --ignore-conflicts'
+
         # A set of content types to exclude from the import process
         if excludes:
-            cmd += f' -i {excludes}'
+            cmd += f' {excludes}'
 
         manage(c, cmd, pty=True, verbose=verbose)
 
@@ -1452,17 +1478,17 @@ def import_records(
 
         if model := entry.get('model', None):
             # Clear out any permissions specified for a group
+            # (these are regenerated after import)
             if model == 'auth.group':
                 entry['fields']['permissions'] = []
 
             # Clear out any permissions specified for a user
+            # (these are regenerated after import)
             if model == 'auth.user':
                 entry['fields']['user_permissions'] = []
 
             # Handle certain model types separately, to ensure they are loaded in the correct order
-            if model.startswith('auth.'):
-                auth_data.append(entry)
-            if model.startswith('users.'):
+            if model.startswith(('auth.', 'users.')):
                 auth_data.append(entry)
             elif model.startswith('common.'):
                 common_data.append(entry)
@@ -1498,6 +1524,12 @@ def import_records(
 
     load_data('remaining', all_data, excludes=content_excludes(allow_auth=False))
 
+    if rebuild_trees:
+        rebuild_models(c)
+
+    if rebuild_images:
+        rebuild_thumbnails(c)
+
     success('Data import completed')