inventree/InvenTree
2
0
Fork 0
mirror of https://github.com/inventree/InvenTree.git synced 2026-09-11 06:59:04 +00:00
Code Activity

Adds bulk-import option for "import-records" task (#12792)

Browse Source 
* skip plugin events if importing or migrating data

* Add bulkloaddata option

* Skip signals if importing

* Improve bulkloaddata command

* Optionally rebuild thumbnails

* enhancements for import_records task

* cache natural key references in bulkloaddata

* wrap export_records in @state_logger

* Reduce file size of exported data

* Added docs

* Test bulk workflow as part of CI

* Add progress bar for data import

* fix for import workflow bug

* Separately test bulk import workflow

* Exercise --prettify option

* Additional CI checks for content excludes

* Allow plugin loading for list_apps

* Additional CI unit tests

* Test for importing with conflicting records

* path fixes

* Adjust test conditions
This commit is contained in:
Oliver
2026-09-06 19:58:52 +10:00
committed by GitHub
parent 321489a41b
commit 1218af7366
10 changed files with 816 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/scripts/check_exported_data.py
+149 -12
View File
@@ -10,6 +10,14 @@ In reads the exported data file, to ensure that:
- The file contains the expected plugin configuration
- The file contains the expected plugin database records
It can also optionally check the presence / absence of several categories of
data which 'export-records' can include or exclude via --include-x /
--exclude-x flags (email logs, API tokens, SSO app/token data, user sessions,
and non-empty group/user permissions) - pass e.g. '--check-email include' or
'--check-email exclude' to assert that category was (or was not) found in the
exported data. Any '--check-x' option which is *not* passed is simply not
checked at all (not even implicitly assumed absent) - so existing invocations
which don't pass any of them keep working unchanged.
"""
PLUGIN_KEY = 'dummy_app_plugin'
@@ -19,10 +27,112 @@ import argparse
import json
import os
def check_category(
data: list[dict], label: str, model_names: list[str], expect: str | None
):
"""Check that all of the given model names are present / absent as expected.
Arguments:
data: The loaded (parsed) exported data file.
label: A human-readable label for this category, for error messages.
model_names: The Django model labels (e.g. 'common.emailmessage')
which make up this category.
expect: 'include' - at least one entry for *each* model name must be
present. 'exclude' - *no* entry for *any* model name may be
present. None - this category was not requested to be checked;
do nothing.
"""
if expect is None:
return
expect_present = expect == 'include'
counts = dict.fromkeys(model_names, 0)
for entry in data:
model = entry.get('model', None)
if model in counts:
counts[model] += 1
if expect_present:
for model, count in counts.items():
if count == 0:
print(f"Error: Expected '{label}' data ('{model}') was not found")
exit(1)
print(f"Found expected '{label}' data ({counts})")
else:
for model, count in counts.items():
if count > 0:
print(
f"Error: '{label}' data ('{model}') was found, but should have been excluded ({count} record(s))"
)
exit(1)
print(f"Confirmed '{label}' data was correctly excluded")
def check_permissions(data: list[dict], expect: str | None):
"""Check that auth.group / auth.user permission fields are stripped or preserved as expected.
Arguments:
data: The loaded (parsed) exported data file.
expect: 'include' - at least one auth.group / auth.user entry must
have non-empty permissions. 'exclude' - all such entries must have
empty permissions. None - not checked; do nothing.
"""
if expect is None:
return
expect_present = expect == 'include'
group_perms = [
entry['fields'].get('permissions', [])
for entry in data
if entry.get('model') == 'auth.group'
]
user_perms = [
entry['fields'].get('user_permissions', [])
for entry in data
if entry.get('model') == 'auth.user'
]
any_group_perms = any(group_perms)
any_user_perms = any(user_perms)
if expect_present:
if not any_group_perms and not any_user_perms:
print(
'Error: Expected at least one auth.group / auth.user entry with '
'non-empty permissions, but all were empty'
)
exit(1)
print('Found expected non-empty group/user permissions')
else:
if any_group_perms or any_user_perms:
print(
'Error: Found non-empty group/user permissions, but they should '
'have been stripped'
)
exit(1)
print('Confirmed group/user permissions were correctly stripped')
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Check exported data file')
parser.add_argument('datafile', help='Path to the exported data file (JSON)')
# Plugin data is checked unconditionally below (it always has been) - this
# just controls which direction is expected, mirroring export-records' own
# --exclude-plugins flag (plugin data is included by default).
parser.add_argument('--exclude-plugins', action='store_true')
# The remaining categories are only checked when explicitly requested -
# pass 'include' or 'exclude' to assert that direction, or omit the flag
# entirely to skip checking that category (the default, for backwards
# compatibility with existing invocations that don't pass any of these).
for flag in ('email', 'tokens', 'sso', 'session', 'permissions'):
parser.add_argument(f'--check-{flag}', choices=['include', 'exclude'])
args = parser.parse_args()
if not os.path.isfile(args.datafile):
@@ -84,18 +194,45 @@ if __name__ == '__main__':
)
exit(1)
if not found_plugin_config:
print(f'Error: No plugin configuration found for plugin "{PLUGIN_KEY}"')
exit(1)
# Check the extracted plugin records
expected_keys = ['alpha', 'beta', 'gamma', 'delta']
for key in expected_keys:
if key not in plugin_data_records:
print(
f'Error: Expected plugin record with key "{key}" not found in exported data'
)
# Plugin data is included by default (export-records only excludes it when
# given --exclude-plugins), so preserve that as the default expectation here
if not args.exclude_plugins:
if not found_plugin_config:
print(f'Error: No plugin configuration found for plugin "{PLUGIN_KEY}"')
exit(1)
# Check the extracted plugin records
expected_keys = ['alpha', 'beta', 'gamma', 'delta']
for key in expected_keys:
if key not in plugin_data_records:
print(
f'Error: Expected plugin record with key "{key}" not found in exported data'
)
exit(1)
elif found_plugin_config:
print('Error: Plugin data was found, but should have been excluded')
exit(1)
else:
print('Confirmed plugin data was correctly excluded')
# Content-excludes checks - only run for '--check-x' flags that were actually passed
check_category(
data, 'email', ['common.emailmessage', 'common.emailthread'], args.check_email
)
check_category(data, 'tokens', ['users.apitoken'], args.check_tokens)
check_category(
data,
'sso',
['socialaccount.socialapp', 'socialaccount.socialtoken'],
args.check_sso,
)
check_category(
data,
'session',
['sessions.session', 'usersessions.usersession'],
args.check_session,
)
check_permissions(data, args.check_permissions)
print('All checks passed successfully!')
.github/scripts/seed_content_excludes_data.py
+112
View File
@@ -0,0 +1,112 @@
"""Script to seed test data for the 'content-excludes' export CI job.
'export-records' can optionally include/exclude several categories of data
(email logs, API tokens, SSO app/token data, user sessions, and group/user
permissions) via --include-x / --exclude-x flags. Toggling one of those flags
only proves anything if the source database actually contains a row in that
category to begin with - otherwise "the export doesn't contain it" is true
regardless of whether the flag/exclusion logic works at all.
This script creates exactly one row in each such category, so the
import_export.yaml workflow's content-excludes job can meaningfully assert
both "included when asked for" and "excluded by default".
Intended to be run from 'src/backend/InvenTree', e.g.:
cd src/backend/InvenTree && python ../../../.github/scripts/seed_content_excludes_data.py
"""
import os
import sys
sys.path.insert(0, os.getcwd())
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'InvenTree.settings')
import django
django.setup()
from django.contrib.auth import get_user_model
from django.contrib.auth.models import Group, Permission
from django.contrib.sessions.backends.db import SessionStore
from allauth.socialaccount.models import SocialAccount, SocialApp, SocialToken
from allauth.usersessions.models import UserSession
from common.models import EmailMessage, Priority
from users.models import ApiToken
User = get_user_model()
def main():
"""Seed one row of test data in each optional export/import category."""
user = User.objects.filter(is_superuser=True).first()
if user is None:
print('Error: no superuser found - run `invoke dev.setup-test` first')
sys.exit(1)
# Ensure at least one group has a non-empty permission set, so toggling
# --include-permissions has something real to include/strip. Doesn't need
# to be a group the superuser belongs to - InvenTree's RuleSet groups
# already have permissions assigned regardless of membership.
group = Group.objects.first()
if group is None:
print('Error: no groups found - run `invoke dev.setup-test` first')
sys.exit(1)
if not group.permissions.exists():
group.permissions.add(Permission.objects.first())
print(f"- Added a permission to group '{group.name}' (was empty)")
else:
print(f"- Group '{group.name}' already has permissions")
# Email log entry (thread is auto-created by EmailMessage.save() if omitted)
EmailMessage.objects.get_or_create(
subject='CI content-excludes test email',
defaults={
'body': 'CI content-excludes test email body',
'to': 'ci-recipient@example.com',
'sender': 'ci-sender@example.com',
'priority': Priority.NORMAL,
},
)
print('- Created email log entry')
# API token
ApiToken.objects.get_or_create(user=user, name='ci-content-excludes-token')
print('- Created API token')
# SSO application + linked account + token
app, _ = SocialApp.objects.get_or_create(
provider='google',
name='CI Content-Excludes Test App',
defaults={'client_id': 'ci-test-client-id'},
)
account, _ = SocialAccount.objects.get_or_create(
user=user, provider='google', uid='ci-test-external-uid'
)
SocialToken.objects.get_or_create(
app=app, account=account, defaults={'token': 'ci-test-token-value'}
)
print('- Created SSO application, account and token')
# A real, properly-encoded session (avoids writing an undecodable session_data blob)
store = SessionStore()
store['ci_content_excludes_test'] = True
store.create()
print('- Created session entry')
# allauth user-session record (tracked separately from the raw Session table)
UserSession.objects.get_or_create(
session_key='ci-content-excludes-user-session',
defaults={'user': user, 'ip': '127.0.0.1', 'user_agent': 'ci-test-agent'},
)
print('- Created user session entry')
print('Content-excludes seed data created successfully')
if __name__ == '__main__':
main()
.github/workflows/import_export.yaml
+222 -3
View File
@@ -58,9 +58,10 @@ jobs:
server:
- .github/workflows/import_export.yaml
- .github/scripts/check_exported_data.py
- .github/scripts/seed_content_excludes_data.py
- 'src/backend/**'
- 'tasks.py'
test:
import-export:
runs-on: ubuntu-latest
needs: paths-filter
if: needs.paths-filter.outputs.server == 'true' || contains(github.event.pull_request.labels.*.name, 'full-run')
@@ -112,9 +113,227 @@ jobs:
test -f /home/runner/work/InvenTree/test_inventree_db.sqlite3 || (echo "Sqlite database not created" && exit 1)
- name: Import Sqlite Dataset
run: |
# Run two imports back-to-back to ensure that the import process is idempotent
invoke import-records -c -f ${{ env.DATA_FILE }} --strict
invoke import-records -c -f ${{ env.DATA_FILE }} --strict
cd src/backend/InvenTree && python manage.py check_dummy_data
- name: Export Sqlite Dataset
invoke export-records -o -f ${{ env.DATA_FILE }}
python ../../../.github/scripts/check_exported_data.py ${{ env.DATA_FILE }}
- name: Bulk Import Sqlite Dataset
run: |
# Ensure that the 'bulk' import process works as expected
invoke import-records -c -f ${{ env.DATA_FILE }} --strict --bulk
cd src/backend/InvenTree && python manage.py check_dummy_data
invoke export-records -o -f ${{ env.DATA_FILE }} --prettify
python ../../../.github/scripts/check_exported_data.py ${{ env.DATA_FILE }}
content-excludes:
# Ensure that 'export-records' correctly includes / excludes each optional
# category of data (email logs, API tokens, SSO app/token data, user
# sessions, and group/user permissions) according to its --include-x /
# --exclude-x flags. Separate from the 'test' job above since it exercises
# a different axis of behaviour (export content, not the import/export
# round-trip) and doesn't need the Sqlite half at all.
runs-on: ubuntu-latest
needs: paths-filter
if: needs.paths-filter.outputs.server == 'true' || contains(github.event.pull_request.labels.*.name, 'full-run')
services:
postgres:
image: postgres:17
env:
POSTGRES_USER: inventree
POSTGRES_PASSWORD: password
ports:
- 5432:5432
steps:
- name: Checkout code
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0
persist-credentials: false
- name: Environment Setup
uses: ./.github/actions/setup
with:
apt-dependency: gettext poppler-utils libpq-dev
pip-dependency: psycopg
update: true
static: false
- name: Setup Postgres Database
run: |
invoke migrate
invoke dev.setup-test -i
- name: Create Plugin Data
run: |
pip install -U inventree-dummy-app-plugin==0.1.0
invoke migrate
cd src/backend/InvenTree && python manage.py create_dummy_data
- name: Seed Content-Excludes Test Data
run: |
# Creates one row in each optional export category (email log, API
# token, SSO app/token, session, group permissions), so that toggling
# the corresponding flag below has real data to prove it actually works
cd src/backend/InvenTree
python ../../../.github/scripts/seed_content_excludes_data.py
- name: Export - All Optional Categories Included
run: |
invoke export-records -o -f ${{ env.DATA_FILE }} --include-email --include-permissions --include-tokens --include-sso --include-session
python .github/scripts/check_exported_data.py ${{ env.DATA_FILE }} \
--check-email include --check-tokens include --check-sso include \
--check-session include --check-permissions include
- name: Export - All Optional Categories Excluded
run: |
invoke export-records -o -f ${{ env.DATA_FILE }} --exclude-plugins
python .github/scripts/check_exported_data.py ${{ env.DATA_FILE }} --exclude-plugins \
--check-email exclude --check-tokens exclude --check-sso exclude \
--check-session exclude --check-permissions exclude
plugin-absent:
# Ensure that importing data referencing a plugin's own models degrades
# gracefully (skipping just those records) when that plugin isn't
# installed on the target - and fails loudly without --ignore-nonexistent.
# See docs/docs/start/migrate.md's "Importing Plugin Data" section,
# condition 1 ("the plugin code must be present in the new installation").
#
# Note: --strict is deliberately *not* used for the imports below,
# as the source metadata's installed_apps list includes the plugin.
runs-on: ubuntu-latest
needs: paths-filter
if: needs.paths-filter.outputs.server == 'true' || contains(github.event.pull_request.labels.*.name, 'full-run')
services:
postgres:
image: postgres:17
env:
POSTGRES_USER: inventree
POSTGRES_PASSWORD: password
ports:
- 5432:5432
steps:
- name: Checkout code
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0
persist-credentials: false
- name: Environment Setup
uses: ./.github/actions/setup
with:
apt-dependency: gettext poppler-utils libpq-dev
pip-dependency: psycopg
update: true
static: false
- name: Setup Postgres Database
run: |
invoke migrate
invoke dev.setup-test -i
- name: Create Plugin Data
run: |
pip install -U inventree-dummy-app-plugin==0.1.0
invoke migrate
cd src/backend/InvenTree && python manage.py create_dummy_data
- name: Export Postgres Dataset (plugin installed)
run: |
invoke export-records -o -f ${{ env.DATA_FILE }}
python .github/scripts/check_exported_data.py ${{ env.DATA_FILE }}
- name: Uninstall Plugin
run: |
pip uninstall -y inventree-dummy-app-plugin
- name: Update Environment Variables for Sqlite
run: |
echo "INVENTREE_DB_ENGINE=sqlite" >> $GITHUB_ENV
echo "INVENTREE_DB_NAME=/home/runner/work/InvenTree/test_inventree_db.sqlite3" >> $GITHUB_ENV
- name: Setup Sqlite Database (plugin not installed)
run: |
invoke migrate
test -f /home/runner/work/InvenTree/test_inventree_db.sqlite3 || (echo "Sqlite database not created" && exit 1)
- name: Import Without --ignore-nonexistent Should Fail
run: |
if invoke import-records -c -f ${{ env.DATA_FILE }}; then
echo "ERROR: import-records succeeded without --ignore-nonexistent, but the plugin is not installed on this target - it should have failed"
exit 1
fi
echo "Confirmed: import correctly failed without --ignore-nonexistent"
- name: Import With --ignore-nonexistent Should Succeed
run: |
invoke import-records -c -f ${{ env.DATA_FILE }} --ignore-nonexistent
cd src/backend/InvenTree
python manage.py shell -c "
from part.models import Part
count = Part.objects.count()
assert count > 0, 'Expected core Part data to be imported'
print(f'Confirmed {count} Part record(s) imported despite the missing plugin')
"
- name: Bulk Import With --ignore-nonexistent Should Also Succeed
run: |
invoke import-records -c -f ${{ env.DATA_FILE }} --ignore-nonexistent --bulk
cd src/backend/InvenTree
python manage.py shell -c "
from part.models import Part
count = Part.objects.count()
assert count > 0, 'Expected core Part data to be imported'
print(f'Confirmed {count} Part record(s) imported despite the missing plugin (bulk)')
"
bulk-conflicts:
# Check for expected conflict behaviour when re-importing a dataset into a database that already contains that dataset.
runs-on: ubuntu-latest
needs: paths-filter
if: needs.paths-filter.outputs.server == 'true' || contains(github.event.pull_request.labels.*.name, 'full-run')
env:
INVENTREE_DB_ENGINE: sqlite
INVENTREE_DB_NAME: /home/runner/work/InvenTree/test_inventree_bulk_conflicts_db.sqlite3
steps:
- name: Checkout code
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0
persist-credentials: false
- name: Environment Setup
uses: ./.github/actions/setup
with:
apt-dependency: gettext poppler-utils libpq-dev
pip-dependency: psycopg
update: true
static: false
- name: Setup Sqlite Database
run: |
invoke migrate
invoke dev.setup-test -i
- name: Export Dataset
run: |
invoke export-records -o -f ${{ env.DATA_FILE }}
cd src/backend/InvenTree
python manage.py shell -c "
from part.models import Part
print(Part.objects.count())
" | tail -n 1 > /home/runner/work/InvenTree/test_inventree_baseline_part_count.txt
echo "Baseline Part count: $(cat /home/runner/work/InvenTree/test_inventree_baseline_part_count.txt)"
- name: Bulk Re-Import Without --ignore-conflicts Should Fail
run: |
# Deliberately no -c/--clear - the database already contains this
# exact data, so every row bulk_create() tries to insert conflicts
# with one already there
if invoke import-records -f ${{ env.DATA_FILE }} --skip-migrations --strict --bulk; then
echo "ERROR: bulk import succeeded against a database with conflicting rows - it should have failed without --ignore-conflicts"
exit 1
fi
echo "Confirmed: bulk import correctly failed on conflicting rows without --ignore-conflicts"
- name: Bulk Re-Import With --ignore-conflicts Should Succeed
run: |
invoke import-records -f ${{ env.DATA_FILE }} --skip-migrations --strict --bulk --ignore-conflicts
cd src/backend/InvenTree
python manage.py shell -c "
from part.models import Part
print(Part.objects.count())
" | tail -n 1 > /home/runner/work/InvenTree/test_inventree_after_part_count.txt
BASELINE=$(cat /home/runner/work/InvenTree/test_inventree_baseline_part_count.txt)
AFTER=$(cat /home/runner/work/InvenTree/test_inventree_after_part_count.txt)
echo "Part count: baseline=$BASELINE, after --ignore-conflicts re-import=$AFTER"
if [ "$BASELINE" != "$AFTER" ]; then
echo "ERROR: Part count changed after --ignore-conflicts re-import (expected conflicting rows to be skipped, not duplicated or lost)"
exit 1
fi
echo "Confirmed: --ignore-conflicts skipped every conflicting row without duplicating or losing any data"