Adds bulk-import option for "import-records" task (#12792)

* skip plugin events if importing or migrating data

* Add bulkloaddata option

* Skip signals if importing

* Improve bulkloaddata command

* Optionally rebuild thumbnails

* enhancements for import_records task

* cache natural key references in bulkloaddata

* wrap export_records in @state_logger

* Reduce file size of exported data

* Added docs

* Test bulk workflow as part of CI

* Add progress bar for data import

* fix for import workflow bug

* Separately test bulk import workflow

* Exercise --prettify option

* Additional CI checks for content excludes

* Allow plugin loading for list_apps

* Additional CI unit tests

* Test for importing with conflicting records

* path fixes

* Adjust test conditions

.github/scripts/check_exported_data.py

@@ -10,6 +10,14 @@ In reads the exported data file, to ensure that:
 - The file contains the expected plugin configuration
 - The file contains the expected plugin database records
 
+It can also optionally check the presence / absence of several categories of
+data which 'export-records' can include or exclude via --include-x /
+--exclude-x flags (email logs, API tokens, SSO app/token data, user sessions,
+and non-empty group/user permissions) - pass e.g. '--check-email include' or
+'--check-email exclude' to assert that category was (or was not) found in the
+exported data. Any '--check-x' option which is *not* passed is simply not
+checked at all (not even implicitly assumed absent) - so existing invocations
+which don't pass any of them keep working unchanged.
 """
 
 PLUGIN_KEY = 'dummy_app_plugin'
@@ -19,10 +27,112 @@ import argparse
 import json
 import os
 
+
+def check_category(
+    data: list[dict], label: str, model_names: list[str], expect: str | None
+):
+    """Check that all of the given model names are present / absent as expected.
+
+    Arguments:
+        data: The loaded (parsed) exported data file.
+        label: A human-readable label for this category, for error messages.
+        model_names: The Django model labels (e.g. 'common.emailmessage')
+            which make up this category.
+        expect: 'include' - at least one entry for *each* model name must be
+            present. 'exclude' - *no* entry for *any* model name may be
+            present. None - this category was not requested to be checked;
+            do nothing.
+    """
+    if expect is None:
+        return
+
+    expect_present = expect == 'include'
+
+    counts = dict.fromkeys(model_names, 0)
+
+    for entry in data:
+        model = entry.get('model', None)
+        if model in counts:
+            counts[model] += 1
+
+    if expect_present:
+        for model, count in counts.items():
+            if count == 0:
+                print(f"Error: Expected '{label}' data ('{model}') was not found")
+                exit(1)
+        print(f"Found expected '{label}' data ({counts})")
+    else:
+        for model, count in counts.items():
+            if count > 0:
+                print(
+                    f"Error: '{label}' data ('{model}') was found, but should have been excluded ({count} record(s))"
+                )
+                exit(1)
+        print(f"Confirmed '{label}' data was correctly excluded")
+
+
+def check_permissions(data: list[dict], expect: str | None):
+    """Check that auth.group / auth.user permission fields are stripped or preserved as expected.
+
+    Arguments:
+        data: The loaded (parsed) exported data file.
+        expect: 'include' - at least one auth.group / auth.user entry must
+            have non-empty permissions. 'exclude' - all such entries must have
+            empty permissions. None - not checked; do nothing.
+    """
+    if expect is None:
+        return
+
+    expect_present = expect == 'include'
+
+    group_perms = [
+        entry['fields'].get('permissions', [])
+        for entry in data
+        if entry.get('model') == 'auth.group'
+    ]
+    user_perms = [
+        entry['fields'].get('user_permissions', [])
+        for entry in data
+        if entry.get('model') == 'auth.user'
+    ]
+
+    any_group_perms = any(group_perms)
+    any_user_perms = any(user_perms)
+
+    if expect_present:
+        if not any_group_perms and not any_user_perms:
+            print(
+                'Error: Expected at least one auth.group / auth.user entry with '
+                'non-empty permissions, but all were empty'
+            )
+            exit(1)
+        print('Found expected non-empty group/user permissions')
+    else:
+        if any_group_perms or any_user_perms:
+            print(
+                'Error: Found non-empty group/user permissions, but they should '
+                'have been stripped'
+            )
+            exit(1)
+        print('Confirmed group/user permissions were correctly stripped')
+
+
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Check exported data file')
     parser.add_argument('datafile', help='Path to the exported data file (JSON)')
 
+    # Plugin data is checked unconditionally below (it always has been) - this
+    # just controls which direction is expected, mirroring export-records' own
+    # --exclude-plugins flag (plugin data is included by default).
+    parser.add_argument('--exclude-plugins', action='store_true')
+
+    # The remaining categories are only checked when explicitly requested -
+    # pass 'include' or 'exclude' to assert that direction, or omit the flag
+    # entirely to skip checking that category (the default, for backwards
+    # compatibility with existing invocations that don't pass any of these).
+    for flag in ('email', 'tokens', 'sso', 'session', 'permissions'):
+        parser.add_argument(f'--check-{flag}', choices=['include', 'exclude'])
+
     args = parser.parse_args()
 
     if not os.path.isfile(args.datafile):
@@ -84,18 +194,45 @@ if __name__ == '__main__':
         )
         exit(1)
 
-    if not found_plugin_config:
-        print(f'Error: No plugin configuration found for plugin "{PLUGIN_KEY}"')
-        exit(1)
-
-    # Check the extracted plugin records
-    expected_keys = ['alpha', 'beta', 'gamma', 'delta']
-
-    for key in expected_keys:
-        if key not in plugin_data_records:
-            print(
-                f'Error: Expected plugin record with key "{key}" not found in exported data'
-            )
+    # Plugin data is included by default (export-records only excludes it when
+    # given --exclude-plugins), so preserve that as the default expectation here
+    if not args.exclude_plugins:
+        if not found_plugin_config:
+            print(f'Error: No plugin configuration found for plugin "{PLUGIN_KEY}"')
             exit(1)
 
+        # Check the extracted plugin records
+        expected_keys = ['alpha', 'beta', 'gamma', 'delta']
+
+        for key in expected_keys:
+            if key not in plugin_data_records:
+                print(
+                    f'Error: Expected plugin record with key "{key}" not found in exported data'
+                )
+                exit(1)
+    elif found_plugin_config:
+        print('Error: Plugin data was found, but should have been excluded')
+        exit(1)
+    else:
+        print('Confirmed plugin data was correctly excluded')
+
+    # Content-excludes checks - only run for '--check-x' flags that were actually passed
+    check_category(
+        data, 'email', ['common.emailmessage', 'common.emailthread'], args.check_email
+    )
+    check_category(data, 'tokens', ['users.apitoken'], args.check_tokens)
+    check_category(
+        data,
+        'sso',
+        ['socialaccount.socialapp', 'socialaccount.socialtoken'],
+        args.check_sso,
+    )
+    check_category(
+        data,
+        'session',
+        ['sessions.session', 'usersessions.usersession'],
+        args.check_session,
+    )
+    check_permissions(data, args.check_permissions)
+
     print('All checks passed successfully!')

.github/scripts/seed_content_excludes_data.py

@@ -0,0 +1,112 @@
+"""Script to seed test data for the 'content-excludes' export CI job.
+
+'export-records' can optionally include/exclude several categories of data
+(email logs, API tokens, SSO app/token data, user sessions, and group/user
+permissions) via --include-x / --exclude-x flags. Toggling one of those flags
+only proves anything if the source database actually contains a row in that
+category to begin with - otherwise "the export doesn't contain it" is true
+regardless of whether the flag/exclusion logic works at all.
+
+This script creates exactly one row in each such category, so the
+import_export.yaml workflow's content-excludes job can meaningfully assert
+both "included when asked for" and "excluded by default".
+
+Intended to be run from 'src/backend/InvenTree', e.g.:
+    cd src/backend/InvenTree && python ../../../.github/scripts/seed_content_excludes_data.py
+"""
+
+import os
+import sys
+
+sys.path.insert(0, os.getcwd())
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'InvenTree.settings')
+
+import django
+
+django.setup()
+
+from django.contrib.auth import get_user_model
+from django.contrib.auth.models import Group, Permission
+from django.contrib.sessions.backends.db import SessionStore
+
+from allauth.socialaccount.models import SocialAccount, SocialApp, SocialToken
+from allauth.usersessions.models import UserSession
+
+from common.models import EmailMessage, Priority
+from users.models import ApiToken
+
+User = get_user_model()
+
+
+def main():
+    """Seed one row of test data in each optional export/import category."""
+    user = User.objects.filter(is_superuser=True).first()
+
+    if user is None:
+        print('Error: no superuser found - run `invoke dev.setup-test` first')
+        sys.exit(1)
+
+    # Ensure at least one group has a non-empty permission set, so toggling
+    # --include-permissions has something real to include/strip. Doesn't need
+    # to be a group the superuser belongs to - InvenTree's RuleSet groups
+    # already have permissions assigned regardless of membership.
+    group = Group.objects.first()
+
+    if group is None:
+        print('Error: no groups found - run `invoke dev.setup-test` first')
+        sys.exit(1)
+
+    if not group.permissions.exists():
+        group.permissions.add(Permission.objects.first())
+        print(f"- Added a permission to group '{group.name}' (was empty)")
+    else:
+        print(f"- Group '{group.name}' already has permissions")
+
+    # Email log entry (thread is auto-created by EmailMessage.save() if omitted)
+    EmailMessage.objects.get_or_create(
+        subject='CI content-excludes test email',
+        defaults={
+            'body': 'CI content-excludes test email body',
+            'to': 'ci-recipient@example.com',
+            'sender': 'ci-sender@example.com',
+            'priority': Priority.NORMAL,
+        },
+    )
+    print('- Created email log entry')
+
+    # API token
+    ApiToken.objects.get_or_create(user=user, name='ci-content-excludes-token')
+    print('- Created API token')
+
+    # SSO application + linked account + token
+    app, _ = SocialApp.objects.get_or_create(
+        provider='google',
+        name='CI Content-Excludes Test App',
+        defaults={'client_id': 'ci-test-client-id'},
+    )
+    account, _ = SocialAccount.objects.get_or_create(
+        user=user, provider='google', uid='ci-test-external-uid'
+    )
+    SocialToken.objects.get_or_create(
+        app=app, account=account, defaults={'token': 'ci-test-token-value'}
+    )
+    print('- Created SSO application, account and token')
+
+    # A real, properly-encoded session (avoids writing an undecodable session_data blob)
+    store = SessionStore()
+    store['ci_content_excludes_test'] = True
+    store.create()
+    print('- Created session entry')
+
+    # allauth user-session record (tracked separately from the raw Session table)
+    UserSession.objects.get_or_create(
+        session_key='ci-content-excludes-user-session',
+        defaults={'user': user, 'ip': '127.0.0.1', 'user_agent': 'ci-test-agent'},
+    )
+    print('- Created user session entry')
+
+    print('Content-excludes seed data created successfully')
+
+
+if __name__ == '__main__':
+    main()

.github/workflows/import_export.yaml

@@ -58,9 +58,10 @@ jobs:
             server:
               - .github/workflows/import_export.yaml
               - .github/scripts/check_exported_data.py
+              - .github/scripts/seed_content_excludes_data.py
               - 'src/backend/**'
               - 'tasks.py'
-  test:
+  import-export:
     runs-on: ubuntu-latest
     needs: paths-filter
     if: needs.paths-filter.outputs.server == 'true' || contains(github.event.pull_request.labels.*.name, 'full-run')
@@ -112,9 +113,227 @@ jobs:
           test -f /home/runner/work/InvenTree/test_inventree_db.sqlite3 || (echo "Sqlite database not created" && exit 1)
       - name: Import Sqlite Dataset
         run: |
+          # Run two imports back-to-back to ensure that the import process is idempotent
+          invoke import-records -c -f ${{ env.DATA_FILE }} --strict
           invoke import-records -c -f ${{ env.DATA_FILE }} --strict
           cd src/backend/InvenTree && python manage.py check_dummy_data
-      - name: Export Sqlite Dataset
+          invoke export-records -o -f ${{ env.DATA_FILE }}
+          python ../../../.github/scripts/check_exported_data.py ${{ env.DATA_FILE }}
+      - name: Bulk Import Sqlite Dataset
+        run: |
+          # Ensure that the 'bulk' import process works as expected
+          invoke import-records -c -f ${{ env.DATA_FILE }} --strict --bulk
+          cd src/backend/InvenTree && python manage.py check_dummy_data
+          invoke export-records -o -f ${{ env.DATA_FILE }} --prettify
+          python ../../../.github/scripts/check_exported_data.py ${{ env.DATA_FILE }}
+
+  content-excludes:
+    # Ensure that 'export-records' correctly includes / excludes each optional
+    # category of data (email logs, API tokens, SSO app/token data, user
+    # sessions, and group/user permissions) according to its --include-x /
+    # --exclude-x flags. Separate from the 'test' job above since it exercises
+    # a different axis of behaviour (export content, not the import/export
+    # round-trip) and doesn't need the Sqlite half at all.
+    runs-on: ubuntu-latest
+    needs: paths-filter
+    if: needs.paths-filter.outputs.server == 'true' || contains(github.event.pull_request.labels.*.name, 'full-run')
+
+    services:
+      postgres:
+        image: postgres:17
+        env:
+          POSTGRES_USER: inventree
+          POSTGRES_PASSWORD: password
+        ports:
+          - 5432:5432
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+      - name: Environment Setup
+        uses: ./.github/actions/setup
+        with:
+          apt-dependency: gettext poppler-utils libpq-dev
+          pip-dependency: psycopg
+          update: true
+          static: false
+      - name: Setup Postgres Database
+        run: |
+          invoke migrate
+          invoke dev.setup-test -i
+      - name: Create Plugin Data
+        run: |
+          pip install -U inventree-dummy-app-plugin==0.1.0
+          invoke migrate
+          cd src/backend/InvenTree && python manage.py create_dummy_data
+      - name: Seed Content-Excludes Test Data
+        run: |
+          # Creates one row in each optional export category (email log, API
+          # token, SSO app/token, session, group permissions), so that toggling
+          # the corresponding flag below has real data to prove it actually works
+          cd src/backend/InvenTree
+          python ../../../.github/scripts/seed_content_excludes_data.py
+      - name: Export - All Optional Categories Included
+        run: |
+          invoke export-records -o -f ${{ env.DATA_FILE }} --include-email --include-permissions --include-tokens --include-sso --include-session
+          python .github/scripts/check_exported_data.py ${{ env.DATA_FILE }} \
+            --check-email include --check-tokens include --check-sso include \
+            --check-session include --check-permissions include
+      - name: Export - All Optional Categories Excluded
+        run: |
+          invoke export-records -o -f ${{ env.DATA_FILE }} --exclude-plugins
+          python .github/scripts/check_exported_data.py ${{ env.DATA_FILE }} --exclude-plugins \
+            --check-email exclude --check-tokens exclude --check-sso exclude \
+            --check-session exclude --check-permissions exclude
+
+  plugin-absent:
+    # Ensure that importing data referencing a plugin's own models degrades
+    # gracefully (skipping just those records) when that plugin isn't
+    # installed on the target - and fails loudly without --ignore-nonexistent.
+    # See docs/docs/start/migrate.md's "Importing Plugin Data" section,
+    # condition 1 ("the plugin code must be present in the new installation").
+    #
+    # Note: --strict is deliberately *not* used for the imports below,
+    # as the source metadata's installed_apps list includes the plugin.
+    runs-on: ubuntu-latest
+    needs: paths-filter
+    if: needs.paths-filter.outputs.server == 'true' || contains(github.event.pull_request.labels.*.name, 'full-run')
+
+    services:
+      postgres:
+        image: postgres:17
+        env:
+          POSTGRES_USER: inventree
+          POSTGRES_PASSWORD: password
+        ports:
+          - 5432:5432
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+      - name: Environment Setup
+        uses: ./.github/actions/setup
+        with:
+          apt-dependency: gettext poppler-utils libpq-dev
+          pip-dependency: psycopg
+          update: true
+          static: false
+      - name: Setup Postgres Database
+        run: |
+          invoke migrate
+          invoke dev.setup-test -i
+      - name: Create Plugin Data
+        run: |
+          pip install -U inventree-dummy-app-plugin==0.1.0
+          invoke migrate
+          cd src/backend/InvenTree && python manage.py create_dummy_data
+      - name: Export Postgres Dataset (plugin installed)
         run: |
           invoke export-records -o -f ${{ env.DATA_FILE }}
-          python .github/scripts/check_exported_data.py ${{ env.DATA_FILE }}
+      - name: Uninstall Plugin
+        run: |
+          pip uninstall -y inventree-dummy-app-plugin
+      - name: Update Environment Variables for Sqlite
+        run: |
+          echo "INVENTREE_DB_ENGINE=sqlite" >> $GITHUB_ENV
+          echo "INVENTREE_DB_NAME=/home/runner/work/InvenTree/test_inventree_db.sqlite3" >> $GITHUB_ENV
+      - name: Setup Sqlite Database (plugin not installed)
+        run: |
+          invoke migrate
+          test -f /home/runner/work/InvenTree/test_inventree_db.sqlite3 || (echo "Sqlite database not created" && exit 1)
+      - name: Import Without --ignore-nonexistent Should Fail
+        run: |
+          if invoke import-records -c -f ${{ env.DATA_FILE }}; then
+            echo "ERROR: import-records succeeded without --ignore-nonexistent, but the plugin is not installed on this target - it should have failed"
+            exit 1
+          fi
+          echo "Confirmed: import correctly failed without --ignore-nonexistent"
+      - name: Import With --ignore-nonexistent Should Succeed
+        run: |
+          invoke import-records -c -f ${{ env.DATA_FILE }} --ignore-nonexistent
+          cd src/backend/InvenTree
+          python manage.py shell -c "
+          from part.models import Part
+          count = Part.objects.count()
+          assert count > 0, 'Expected core Part data to be imported'
+          print(f'Confirmed {count} Part record(s) imported despite the missing plugin')
+          "
+      - name: Bulk Import With --ignore-nonexistent Should Also Succeed
+        run: |
+          invoke import-records -c -f ${{ env.DATA_FILE }} --ignore-nonexistent --bulk
+          cd src/backend/InvenTree
+          python manage.py shell -c "
+          from part.models import Part
+          count = Part.objects.count()
+          assert count > 0, 'Expected core Part data to be imported'
+          print(f'Confirmed {count} Part record(s) imported despite the missing plugin (bulk)')
+          "
+
+  bulk-conflicts:
+    # Check for expected conflict behaviour when re-importing a dataset into a database that already contains that dataset.
+    runs-on: ubuntu-latest
+    needs: paths-filter
+    if: needs.paths-filter.outputs.server == 'true' || contains(github.event.pull_request.labels.*.name, 'full-run')
+
+    env:
+      INVENTREE_DB_ENGINE: sqlite
+      INVENTREE_DB_NAME: /home/runner/work/InvenTree/test_inventree_bulk_conflicts_db.sqlite3
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+      - name: Environment Setup
+        uses: ./.github/actions/setup
+        with:
+          apt-dependency: gettext poppler-utils libpq-dev
+          pip-dependency: psycopg
+          update: true
+          static: false
+      - name: Setup Sqlite Database
+        run: |
+          invoke migrate
+          invoke dev.setup-test -i
+      - name: Export Dataset
+        run: |
+          invoke export-records -o -f ${{ env.DATA_FILE }}
+          cd src/backend/InvenTree
+          python manage.py shell -c "
+          from part.models import Part
+          print(Part.objects.count())
+          " | tail -n 1 > /home/runner/work/InvenTree/test_inventree_baseline_part_count.txt
+          echo "Baseline Part count: $(cat /home/runner/work/InvenTree/test_inventree_baseline_part_count.txt)"
+      - name: Bulk Re-Import Without --ignore-conflicts Should Fail
+        run: |
+          # Deliberately no -c/--clear - the database already contains this
+          # exact data, so every row bulk_create() tries to insert conflicts
+          # with one already there
+          if invoke import-records -f ${{ env.DATA_FILE }} --skip-migrations --strict --bulk; then
+            echo "ERROR: bulk import succeeded against a database with conflicting rows - it should have failed without --ignore-conflicts"
+            exit 1
+          fi
+          echo "Confirmed: bulk import correctly failed on conflicting rows without --ignore-conflicts"
+      - name: Bulk Re-Import With --ignore-conflicts Should Succeed
+        run: |
+          invoke import-records -f ${{ env.DATA_FILE }} --skip-migrations --strict --bulk --ignore-conflicts
+          cd src/backend/InvenTree
+          python manage.py shell -c "
+          from part.models import Part
+          print(Part.objects.count())
+          " | tail -n 1 > /home/runner/work/InvenTree/test_inventree_after_part_count.txt
+          BASELINE=$(cat /home/runner/work/InvenTree/test_inventree_baseline_part_count.txt)
+          AFTER=$(cat /home/runner/work/InvenTree/test_inventree_after_part_count.txt)
+          echo "Part count: baseline=$BASELINE, after --ignore-conflicts re-import=$AFTER"
+          if [ "$BASELINE" != "$AFTER" ]; then
+            echo "ERROR: Part count changed after --ignore-conflicts re-import (expected conflicting rows to be skipped, not duplicated or lost)"
+            exit 1
+          fi
+          echo "Confirmed: --ignore-conflicts skipped every conflicting row without duplicating or losing any data"

docs/docs/start/migrate.md

@@ -31,6 +31,9 @@ This will create JSON file at the specified location which contains all database
 !!! info "Specifying filename"
     The filename of the exported file can be specified using the `-f` option. To see all available options, run `invoke export-records --help`
 
+!!! info "File Size"
+    By default the exported file is written as compact JSON, to keep its size down. Add the `-p` / `--prettify` option to pretty-print the output with indentation, which is easier to read manually but can roughly double the file size for a large database.
+
 ```
 {{ invoke_commands('export-records --help') }}
 ```
@@ -67,6 +70,16 @@ invoke import-records -c -f data.json
 !!! warning "Character Encoding"
 	If the character encoding of the data file does not exactly match the target database, the import operation may not succeed. In this case, some manual editing of the database JSON file may be required.
 
+!!! tip "Faster Imports"
+    For very large datasets, add the `-b` / `--bulk` option to use a faster import path (the `bulkloaddata` management command) which inserts records in large batches and skips per-record signal processing, rather than saving each record individually:
+
+    ```
+    invoke import-records -c -b -f data.json
+    ```
+
+!!! tip "Strict Metadata Validation"
+    By default, a mismatch between the source and target InvenTree versions (see the "Database Versions" warning above) only produces a warning, and the import continues. Add the `-s` / `--strict` option to fail immediately instead, if you want to guarantee the versions match exactly before any data is written.
+
 ```
 {{ invoke_commands('import-records --help') }}
 ```
@@ -220,6 +233,9 @@ When running the `import-records` command, the import process will also attempt
 2. The plugin *version* must be the same in both installations. If the plugin version is different, then the database schema may be different, and thus the import process may fail.
 3. The InvenTree software version must be the same in both installations. If the InvenTree version is different, then the database schema may be different, and thus the import process may fail.
 
+!!! tip "Skipping Missing Data"
+    If the import file references a plugin (or any other model) that cannot be matched to the current installation - for example, condition 1 above is not met - add the `-i` / `--ignore-nonexistent` option to skip those records instead of failing the entire import.
+
 If all of the above conditions are met, then the plugin data *should* be imported correctly into the new database. To achieve this reliably, the following process steps are implemented in the `import-records` command:
 
 1. The database is cleaned of all existing records (if the `-c` option is used).

src/backend/InvenTree/InvenTree/management/commands/bulkloaddata.py

@@ -0,0 +1,213 @@
+"""Custom management command to load fixtures faster using bulk_create()."""
+
+import time
+from contextlib import contextmanager
+
+from django.core.management.base import CommandError
+from django.core.management.commands.loaddata import Command as LoadDataCommand
+from django.core.serializers import base as serializers_base
+from django.db import DatabaseError, IntegrityError, connections, router
+
+import structlog
+from tqdm import tqdm
+
+logger = structlog.get_logger('inventree')
+
+DEFAULT_BATCH_SIZE = 500
+
+# Number of records to process between progress bar updates. Calling tqdm's
+# update() for every single record adds measurable overhead of its own (well
+# beyond just the display refresh) once there are a million or more of them -
+# so update it in batches instead.
+PROGRESS_UPDATE_INTERVAL = 100
+
+
+class Command(LoadDataCommand):
+    """Load fixtures using bulk_create() for improved performance.
+
+    Behaves like the built-in 'loaddata' command, with two differences to be
+    aware of:
+
+    - pre_save / post_save signals are not sent, and Model.save() / full_clean()
+      are bypassed entirely (this is a Django bulk_create() limitation).
+    - Multi-table inheritance is not supported by bulk_create() and will fail
+      loudly rather than being silently mishandled. Natural-key foreign key /
+      many-to-many resolution *is* supported (falling back to an individual,
+      non-bulk save for any row that needs it - see save_obj()) and is further
+      sped up by caching each resolved natural key for the life of the command
+      (see _cached_natural_keys()), since 'export_records' uses
+      --natural-foreign and a large fixture can have many rows referencing the
+      same handful of natural-keyed objects (e.g. stock.StockItemTracking.user
+      -> auth.User) - without caching, each one costs a separate DB query.
+
+    Based on the django forum thread:
+    - https://forum.djangoproject.com/t/feature-proposal-faster-fixture-loading-via-loaddata-command/36972/21
+    """
+
+    def add_arguments(self, parser):
+        """Add bulkloaddata-specific arguments, on top of loaddata's own."""
+        super().add_arguments(parser)
+        parser.add_argument(
+            '--batch-size',
+            type=int,
+            default=DEFAULT_BATCH_SIZE,
+            help=f'Number of records per bulk_create() batch (default: {DEFAULT_BATCH_SIZE})',
+        )
+        parser.add_argument(
+            '--ignore-conflicts',
+            action='store_true',
+            help='Skip records that violate a unique constraint, instead of raising an error',
+        )
+
+    def handle(self, *fixture_labels, **options):
+        """Store bulk-loading options before delegating to the base command."""
+        self.batch_size = options['batch_size']
+        self.ignore_conflicts = options['ignore_conflicts']
+        self.pending_objs = {}
+        self.pending_progress = 0
+
+        connection = connections[options['database']]
+        self.query_count = 0
+
+        def count_queries(execute, sql, params, many, context):
+            """Count every query executed against this connection, without the overhead of recording each query's SQL text (unlike e.g. CaptureQueriesContext)."""
+            self.query_count += 1
+            return execute(sql, params, many, context)
+
+        start_time = time.monotonic()
+
+        with (
+            connection.execute_wrapper(count_queries),
+            self._cached_natural_keys(),
+            tqdm(desc='Importing', unit=' records') as self.progress,
+        ):
+            super().handle(*fixture_labels, **options)
+            self.progress.update(self.pending_progress)
+
+        elapsed = time.monotonic() - start_time
+
+        if self.verbosity >= 1:
+            self.stdout.write(
+                f'Executed {self.query_count} database queries in {elapsed:.2f}s'
+            )
+
+    @contextmanager
+    def _cached_natural_keys(self):
+        """Cache natural-key foreign key resolutions for the duration of this block.
+
+        Django's deserializer (deserialize_fk_value) issues a fresh DB query
+        every time it resolves a natural-key FK reference, with no caching of
+        its own. A fixture with many rows referencing the same handful of
+        natural-keyed objects (e.g. thousands of stock.StockItemTracking rows
+        all pointing at a few auth.User accounts) would otherwise cost one
+        query per row instead of one query per distinct value.
+        """
+        cache = {}
+        original = serializers_base.deserialize_fk_value
+
+        def cached_deserialize_fk_value(
+            field, field_value, using, handle_forward_references
+        ):
+            default_manager = field.remote_field.model._default_manager
+
+            is_natural_key = (
+                field_value is not None
+                and hasattr(default_manager, 'get_by_natural_key')
+                and hasattr(field_value, '__iter__')
+                and not isinstance(field_value, str)
+            )
+
+            if not is_natural_key:
+                # Plain (non natural-key) FK values never reach a DB query in
+                # the first place - nothing to cache, delegate as normal
+                return original(field, field_value, using, handle_forward_references)
+
+            cache_key = (field.remote_field.model, using, tuple(field_value))
+
+            if cache_key in cache:
+                return cache[cache_key]
+
+            value = original(field, field_value, using, handle_forward_references)
+
+            # Only cache a fully-resolved value - a deferred lookup (the
+            # referenced object doesn't exist yet) may well succeed on a later
+            # call, once that object has actually been saved.
+            if value is not serializers_base.DEFER_FIELD:
+                cache[cache_key] = value
+
+            return value
+
+        serializers_base.deserialize_fk_value = cached_deserialize_fk_value
+        try:
+            yield
+        finally:
+            serializers_base.deserialize_fk_value = original
+
+    def save_obj(self, obj):
+        """Buffer an object for bulk insertion, instead of saving it immediately."""
+        self.pending_progress += 1
+        if self.pending_progress >= PROGRESS_UPDATE_INTERVAL:
+            self.progress.update(self.pending_progress)
+            self.pending_progress = 0
+
+        if (
+            obj.object._meta.app_config in self.excluded_apps
+            or type(obj.object) in self.excluded_models
+        ):
+            return False
+
+        if not router.allow_migrate_model(self.using, obj.object.__class__):
+            return False
+
+        self.models.add(obj.object.__class__)
+
+        if obj.deferred_fields:
+            # This object has an unresolved forward reference (e.g. a natural-key
+            # FK to an object that has not been saved yet - 'export_records' uses
+            # --natural-foreign, and e.g. auth.User defines a natural_key(), so
+            # this does occur in practice). It cannot be bulk_create()'d as-is, since
+            # the deferred field would be written as blank/null. The base loaddata()
+            # command already resolves and saves objects like this individually,
+            # via save_deferred_fields(), once every fixture file has been buffered
+            # (see 'handle' -> 'loaddata') - so just hand it off for that, rather
+            # than also bulk-inserting it here with the field left unresolved.
+            self.objs_with_deferred_fields.append(obj)
+        else:
+            self.pending_objs.setdefault(obj.object.__class__, []).append(obj)
+
+        return True
+
+    def flush_pending(self):
+        """Bulk-create every object buffered so far, grouped by model."""
+        for model, objs in self.pending_objs.items():
+            try:
+                model._default_manager.db_manager(self.using).bulk_create(
+                    [obj.object for obj in objs],
+                    batch_size=self.batch_size,
+                    ignore_conflicts=self.ignore_conflicts,
+                )
+            except (DatabaseError, IntegrityError, ValueError) as e:
+                e.args = (
+                    f'Could not bulk-create {len(objs)} object(s) of {model._meta.label}: {e}',
+                )
+                raise
+
+            # bulk_create() cannot populate many-to-many relations - apply them here,
+            # same as DeserializedObject.save() does for the non-bulk path.
+            for obj in objs:
+                if obj.m2m_data:
+                    for accessor_name, values in obj.m2m_data.items():
+                        getattr(obj.object, accessor_name).set(values)
+                    obj.m2m_data = None
+
+        self.pending_objs = {}
+
+    def load_label(self, fixture_label):
+        """Load one fixture label, then flush the records it buffered."""
+        super().load_label(fixture_label)
+        try:
+            self.flush_pending()
+        except Exception as e:
+            if not isinstance(e, CommandError):
+                e.args = (f"Problem installing fixture '{fixture_label}': {e}",)
+            raise

src/backend/InvenTree/InvenTree/ready.py

@@ -49,7 +49,7 @@ def isWaitingForDatabase():
 
 def isImportingData():
     """Returns True if the database is currently importing (or exporting) data, e.g. 'loaddata' command is performed."""
-    return any(x in sys.argv for x in ['flush', 'loaddata', 'dumpdata'])
+    return any(x in sys.argv for x in ['flush', 'loaddata', 'bulkloaddata', 'dumpdata'])
 
 
 def isRunningMigrations():
@@ -285,7 +285,7 @@ def canAppAccessDatabase(
         excluded_commands.append('test')
 
     if not allow_plugins:
-        excluded_commands.extend(['collectplugins'])
+        excluded_commands.extend(['collectplugins', 'list_apps'])
 
     return all(cmd not in sys.argv for cmd in excluded_commands)
 

src/backend/InvenTree/InvenTree/test_commands.py

@@ -7,6 +7,7 @@ from pathlib import Path
 from django.conf import settings
 from django.contrib.auth.models import User
 from django.core.management import call_command
+from django.db import IntegrityError
 from django.test import TestCase
 
 from opentelemetry.instrumentation.sqlite3 import SQLite3Instrumentor
@@ -116,6 +117,50 @@ class CommandTestCase(TestCase):
         self.assertEqual(output, 'done')
         self.assertEqual(my_admin3.authenticator_set.all().count(), 0)
 
+    def test_bulkloaddata(self):
+        """Test the bulkloaddata command."""
+        from django.contrib.contenttypes.models import ContentType
+        from django.core import serializers
+
+        # ContentType has no custom signals and a real unique_together constraint
+        # (app_label, model), making it a convenient, side-effect-free test model.
+        entries = [
+            ContentType.objects.create(app_label='bulkloaddata_test', model=f'model{i}')
+            for i in range(5)
+        ]
+        pks = [e.pk for e in entries]
+        data = serializers.serialize('json', entries)
+        ContentType.objects.filter(pk__in=pks).delete()
+
+        tmp_file = get_testfolder_dir().joinpath('bulkloaddata_test.json')
+        tmp_file.write_text(data, encoding='utf-8')
+
+        try:
+            # Basic load - all records should be recreated
+            call_command('bulkloaddata', str(tmp_file), verbosity=0)
+            self.assertEqual(ContentType.objects.filter(pk__in=pks).count(), 5)
+
+            # Re-loading without --ignore-conflicts should raise (unique app_label/model)
+            with self.assertRaises(IntegrityError):
+                call_command('bulkloaddata', str(tmp_file), verbosity=0)
+
+            # Re-loading with --ignore-conflicts should succeed, without duplicating rows
+            call_command(
+                'bulkloaddata', str(tmp_file), verbosity=0, ignore_conflicts=True
+            )
+            self.assertEqual(ContentType.objects.filter(pk__in=pks).count(), 5)
+
+            # A small batch size should still load every record correctly
+            ContentType.objects.filter(pk__in=pks).delete()
+            call_command('bulkloaddata', str(tmp_file), verbosity=0, batch_size=2)
+            models = set(
+                ContentType.objects.filter(pk__in=pks).values_list('model', flat=True)
+            )
+            self.assertEqual(models, {e.model for e in entries})
+        finally:
+            ContentType.objects.filter(pk__in=pks).delete()
+            tmp_file.unlink(missing_ok=True)
+
     def test_backup_metadata(self):
         """Test the backup metadata functions."""
         from InvenTree.backup import (

src/backend/InvenTree/order/models.py

@@ -4287,6 +4287,9 @@ class TransferOrderAllocation(models.Model):
 
 def _touch_order_updated_at(instance):
     """Bump updated_at on the parent order without triggering a full save."""
+    if InvenTree.ready.isRunningMigrations() or InvenTree.ready.isImportingData():
+        # Do not touch the order during migrations or data import
+        return
     if not InvenTree.ready.canAppAccessDatabase(allow_test=True):
         return
     instance.order.__class__.objects.filter(pk=instance.order_id).update(

src/backend/InvenTree/plugin/base/event/events.py

@@ -14,7 +14,7 @@ from opentelemetry import trace
 
 import InvenTree.exceptions
 from common.settings import get_global_setting
-from InvenTree.ready import canAppAccessDatabase, isImportingData
+from InvenTree.ready import canAppAccessDatabase, isImportingData, isRunningMigrations
 from InvenTree.tasks import bulk_offload_task, offload_task
 from plugin import PluginMixinEnum
 from plugin.registry import registry
@@ -227,6 +227,10 @@ def process_event(plugin_slug, event, *args, **kwargs):
     This function is run by the background worker process.
     This function may queue multiple functions to be handled by the background worker.
     """
+    if isRunningMigrations() or isImportingData():
+        # Do not trigger events during migrations or data import
+        return
+
     plugin = registry.get_plugin(plugin_slug, active=True)
 
     if plugin is None:  # pragma: no cover
@@ -293,6 +297,10 @@ def allow_table_event(table_name):
 @receiver(post_save)
 def after_save(sender, instance, created, **kwargs):
     """Trigger an event whenever a database entry is saved."""
+    if isRunningMigrations() or isImportingData():
+        # Do not trigger events during migrations or data import
+        return
+
     table = sender.objects.model._meta.db_table
 
     instance_id = getattr(instance, 'id', None)
@@ -312,6 +320,10 @@ def after_save(sender, instance, created, **kwargs):
 @receiver(post_delete)
 def after_delete(sender, instance, **kwargs):
     """Trigger an event whenever a database entry is deleted."""
+    if isRunningMigrations() or isImportingData():
+        # Do not trigger events during migrations or data import
+        return
+
     table = sender.objects.model._meta.db_table
 
     if not allow_table_event(table):

tasks.py

@@ -1205,9 +1205,11 @@ def update(
         'exclude_plugins': 'Exclude plugin data from the output file (default = False)',
         'include_sso': 'Include SSO token data in the output file (default = False)',
         'include_session': 'Include user session data in the output file (default = False)',
+        'prettify': 'Pretty-print the output file with indentation (default = False)',
         'verbose': 'Print verbose output from management commands',
     }
 )
+@state_logger
 def export_records(
     c,
     filename='data.json',
@@ -1218,6 +1220,7 @@ def export_records(
     exclude_plugins: bool = False,
     include_sso: bool = False,
     include_session: bool = False,
+    prettify: bool = False,
     verbose: bool = False,
 ):
     """Export all database records to a file."""
@@ -1243,7 +1246,10 @@ def export_records(
     with tempfile.NamedTemporaryFile(
         suffix='.json', encoding='utf-8', mode='w+t', delete=True
     ) as tmpfile:
-        cmd = f"dumpdata --natural-foreign --indent 2 --output '{tmpfile.name}' {excludes}"
+        cmd = f"dumpdata --natural-foreign --output '{tmpfile.name}' {excludes}"
+
+        if prettify:
+            cmd += ' --indent 2'
 
         # Dump data to temporary file
         manage(c, cmd, pty=True, verbose=verbose)
@@ -1258,7 +1264,7 @@ def export_records(
             'metadata': True,
             'comment': 'This file contains a dump of the InvenTree database',
             'exported_at': datetime.datetime.now().isoformat(),
-            'exported_at_utc': datetime.datetime.utcnow().isoformat(),
+            'exported_at_utc': datetime.datetime.now(datetime.UTC).isoformat(),
             'source_version': get_inventree_version(),
             'api_version': get_inventree_api_version(),
             'django_version': get_django_version(),
@@ -1287,7 +1293,7 @@ def export_records(
 
     # Write the processed data to file
     with open(target, 'w', encoding='utf-8') as f_out:
-        f_out.write(json.dumps(data_out, indent=2))
+        f_out.write(json.dumps(data_out, indent=2 if prettify else None))
 
     success('Data export completed')
 
@@ -1358,10 +1364,15 @@ def validate_import_metadata(
         'exclude_plugins': 'Exclude plugin data from the import process (default = False)',
         'skip_migrations': 'Skip the migration step after clearing data (default = False)',
         'verbose': 'Print verbose output from management commands',
+        'bulk': 'Use the faster bulkloaddata command instead of loaddata (default = False)',
+        'ignore_conflicts': 'Skip records that violate a unique constraint, instead of raising an error (requires --bulk, default = False)',
+        'rebuild_trees': 'Rebuild MPTT tree structures after import (default = True)',
+        'rebuild_images': 'Rebuild image thumbnails after import (default = True)',
     },
     pre=[wait],
-    post=[rebuild_models, rebuild_thumbnails],
+    post=[],
 )
+@state_logger
 def import_records(
     c,
     filename='data.json',
@@ -1371,6 +1382,10 @@ def import_records(
     ignore_nonexistent: bool = False,
     skip_migrations: bool = False,
     verbose: bool = False,
+    bulk: bool = False,
+    ignore_conflicts: bool = False,
+    rebuild_trees: bool = True,
+    rebuild_images: bool = True,
 ):
     """Import database records from a file."""
     # Get an absolute path to the supplied filename
@@ -1383,6 +1398,10 @@ def import_records(
         error(f"ERROR: File '{target}' does not exist")
         sys.exit(1)
 
+    if ignore_conflicts and not bulk:
+        warning('--ignore-conflicts has no effect without --bulk - ignoring')
+        ignore_conflicts = False
+
     if clear:
         delete_data(c, force=True, migrate=True, verbose=verbose)
 
@@ -1416,6 +1435,8 @@ def import_records(
         """Helper function to save data to a temporary file, and then load into the database."""
         nonlocal ignore_nonexistent
         nonlocal verbose
+        nonlocal bulk
+        nonlocal ignore_conflicts
         nonlocal c
 
         # Skip if there is no data to load
@@ -1429,7 +1450,9 @@ def import_records(
         ) as f_out:
             f_out.write(json.dumps(data, indent=2))
 
-        cmd = f'loaddata {f_out.name} -v 0 --force-color'
+        cmd = (
+            f'{"bulkloaddata" if bulk else "loaddata"} {f_out.name} -v 0 --force-color'
+        )
 
         if app:
             cmd += f' --app {app}'
@@ -1437,9 +1460,12 @@ def import_records(
         if ignore_nonexistent:
             cmd += ' --ignorenonexistent'
 
+        if bulk and ignore_conflicts:
+            cmd += ' --ignore-conflicts'
+
         # A set of content types to exclude from the import process
         if excludes:
-            cmd += f' -i {excludes}'
+            cmd += f' {excludes}'
 
         manage(c, cmd, pty=True, verbose=verbose)
 
@@ -1452,17 +1478,17 @@ def import_records(
 
         if model := entry.get('model', None):
             # Clear out any permissions specified for a group
+            # (these are regenerated after import)
             if model == 'auth.group':
                 entry['fields']['permissions'] = []
 
             # Clear out any permissions specified for a user
+            # (these are regenerated after import)
             if model == 'auth.user':
                 entry['fields']['user_permissions'] = []
 
             # Handle certain model types separately, to ensure they are loaded in the correct order
-            if model.startswith('auth.'):
-                auth_data.append(entry)
-            if model.startswith('users.'):
+            if model.startswith(('auth.', 'users.')):
                 auth_data.append(entry)
             elif model.startswith('common.'):
                 common_data.append(entry)
@@ -1498,6 +1524,12 @@ def import_records(
 
     load_data('remaining', all_data, excludes=content_excludes(allow_auth=False))
 
+    if rebuild_trees:
+        rebuild_models(c)
+
+    if rebuild_images:
+        rebuild_thumbnails(c)
+
     success('Data import completed')