mirror of
https://github.com/inventree/InvenTree.git
synced 2026-09-10 06:37:17 +00:00
Adds bulk-import option for "import-records" task (#12792)
* skip plugin events if importing or migrating data * Add bulkloaddata option * Skip signals if importing * Improve bulkloaddata command * Optionally rebuild thumbnails * enhancements for import_records task * cache natural key references in bulkloaddata * wrap export_records in @state_logger * Reduce file size of exported data * Added docs * Test bulk workflow as part of CI * Add progress bar for data import * fix for import workflow bug * Separately test bulk import workflow * Exercise --prettify option * Additional CI checks for content excludes * Allow plugin loading for list_apps * Additional CI unit tests * Test for importing with conflicting records * path fixes * Adjust test conditions
This commit is contained in:
Oliver
@@ -10,6 +10,14 @@ In reads the exported data file, to ensure that:
|
||||
- The file contains the expected plugin configuration
|
||||
- The file contains the expected plugin database records
|
||||
|
||||
It can also optionally check the presence / absence of several categories of
|
||||
data which 'export-records' can include or exclude via --include-x /
|
||||
--exclude-x flags (email logs, API tokens, SSO app/token data, user sessions,
|
||||
and non-empty group/user permissions) - pass e.g. '--check-email include' or
|
||||
'--check-email exclude' to assert that category was (or was not) found in the
|
||||
exported data. Any '--check-x' option which is *not* passed is simply not
|
||||
checked at all (not even implicitly assumed absent) - so existing invocations
|
||||
which don't pass any of them keep working unchanged.
|
||||
"""
|
||||
|
||||
PLUGIN_KEY = 'dummy_app_plugin'
|
||||
@@ -19,10 +27,112 @@ import argparse
|
||||
import json
|
||||
import os
|
||||
|
||||
|
||||
def check_category(
|
||||
data: list[dict], label: str, model_names: list[str], expect: str | None
|
||||
):
|
||||
"""Check that all of the given model names are present / absent as expected.
|
||||
|
||||
Arguments:
|
||||
data: The loaded (parsed) exported data file.
|
||||
label: A human-readable label for this category, for error messages.
|
||||
model_names: The Django model labels (e.g. 'common.emailmessage')
|
||||
which make up this category.
|
||||
expect: 'include' - at least one entry for *each* model name must be
|
||||
present. 'exclude' - *no* entry for *any* model name may be
|
||||
present. None - this category was not requested to be checked;
|
||||
do nothing.
|
||||
"""
|
||||
if expect is None:
|
||||
return
|
||||
|
||||
expect_present = expect == 'include'
|
||||
|
||||
counts = dict.fromkeys(model_names, 0)
|
||||
|
||||
for entry in data:
|
||||
model = entry.get('model', None)
|
||||
if model in counts:
|
||||
counts[model] += 1
|
||||
|
||||
if expect_present:
|
||||
for model, count in counts.items():
|
||||
if count == 0:
|
||||
print(f"Error: Expected '{label}' data ('{model}') was not found")
|
||||
exit(1)
|
||||
print(f"Found expected '{label}' data ({counts})")
|
||||
else:
|
||||
for model, count in counts.items():
|
||||
if count > 0:
|
||||
print(
|
||||
f"Error: '{label}' data ('{model}') was found, but should have been excluded ({count} record(s))"
|
||||
)
|
||||
exit(1)
|
||||
print(f"Confirmed '{label}' data was correctly excluded")
|
||||
|
||||
|
||||
def check_permissions(data: list[dict], expect: str | None):
|
||||
"""Check that auth.group / auth.user permission fields are stripped or preserved as expected.
|
||||
|
||||
Arguments:
|
||||
data: The loaded (parsed) exported data file.
|
||||
expect: 'include' - at least one auth.group / auth.user entry must
|
||||
have non-empty permissions. 'exclude' - all such entries must have
|
||||
empty permissions. None - not checked; do nothing.
|
||||
"""
|
||||
if expect is None:
|
||||
return
|
||||
|
||||
expect_present = expect == 'include'
|
||||
|
||||
group_perms = [
|
||||
entry['fields'].get('permissions', [])
|
||||
for entry in data
|
||||
if entry.get('model') == 'auth.group'
|
||||
]
|
||||
user_perms = [
|
||||
entry['fields'].get('user_permissions', [])
|
||||
for entry in data
|
||||
if entry.get('model') == 'auth.user'
|
||||
]
|
||||
|
||||
any_group_perms = any(group_perms)
|
||||
any_user_perms = any(user_perms)
|
||||
|
||||
if expect_present:
|
||||
if not any_group_perms and not any_user_perms:
|
||||
print(
|
||||
'Error: Expected at least one auth.group / auth.user entry with '
|
||||
'non-empty permissions, but all were empty'
|
||||
)
|
||||
exit(1)
|
||||
print('Found expected non-empty group/user permissions')
|
||||
else:
|
||||
if any_group_perms or any_user_perms:
|
||||
print(
|
||||
'Error: Found non-empty group/user permissions, but they should '
|
||||
'have been stripped'
|
||||
)
|
||||
exit(1)
|
||||
print('Confirmed group/user permissions were correctly stripped')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description='Check exported data file')
|
||||
parser.add_argument('datafile', help='Path to the exported data file (JSON)')
|
||||
|
||||
# Plugin data is checked unconditionally below (it always has been) - this
|
||||
# just controls which direction is expected, mirroring export-records' own
|
||||
# --exclude-plugins flag (plugin data is included by default).
|
||||
parser.add_argument('--exclude-plugins', action='store_true')
|
||||
|
||||
# The remaining categories are only checked when explicitly requested -
|
||||
# pass 'include' or 'exclude' to assert that direction, or omit the flag
|
||||
# entirely to skip checking that category (the default, for backwards
|
||||
# compatibility with existing invocations that don't pass any of these).
|
||||
for flag in ('email', 'tokens', 'sso', 'session', 'permissions'):
|
||||
parser.add_argument(f'--check-{flag}', choices=['include', 'exclude'])
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if not os.path.isfile(args.datafile):
|
||||
@@ -84,18 +194,45 @@ if __name__ == '__main__':
|
||||
)
|
||||
exit(1)
|
||||
|
||||
if not found_plugin_config:
|
||||
print(f'Error: No plugin configuration found for plugin "{PLUGIN_KEY}"')
|
||||
exit(1)
|
||||
|
||||
# Check the extracted plugin records
|
||||
expected_keys = ['alpha', 'beta', 'gamma', 'delta']
|
||||
|
||||
for key in expected_keys:
|
||||
if key not in plugin_data_records:
|
||||
print(
|
||||
f'Error: Expected plugin record with key "{key}" not found in exported data'
|
||||
)
|
||||
# Plugin data is included by default (export-records only excludes it when
|
||||
# given --exclude-plugins), so preserve that as the default expectation here
|
||||
if not args.exclude_plugins:
|
||||
if not found_plugin_config:
|
||||
print(f'Error: No plugin configuration found for plugin "{PLUGIN_KEY}"')
|
||||
exit(1)
|
||||
|
||||
# Check the extracted plugin records
|
||||
expected_keys = ['alpha', 'beta', 'gamma', 'delta']
|
||||
|
||||
for key in expected_keys:
|
||||
if key not in plugin_data_records:
|
||||
print(
|
||||
f'Error: Expected plugin record with key "{key}" not found in exported data'
|
||||
)
|
||||
exit(1)
|
||||
elif found_plugin_config:
|
||||
print('Error: Plugin data was found, but should have been excluded')
|
||||
exit(1)
|
||||
else:
|
||||
print('Confirmed plugin data was correctly excluded')
|
||||
|
||||
# Content-excludes checks - only run for '--check-x' flags that were actually passed
|
||||
check_category(
|
||||
data, 'email', ['common.emailmessage', 'common.emailthread'], args.check_email
|
||||
)
|
||||
check_category(data, 'tokens', ['users.apitoken'], args.check_tokens)
|
||||
check_category(
|
||||
data,
|
||||
'sso',
|
||||
['socialaccount.socialapp', 'socialaccount.socialtoken'],
|
||||
args.check_sso,
|
||||
)
|
||||
check_category(
|
||||
data,
|
||||
'session',
|
||||
['sessions.session', 'usersessions.usersession'],
|
||||
args.check_session,
|
||||
)
|
||||
check_permissions(data, args.check_permissions)
|
||||
|
||||
print('All checks passed successfully!')
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
"""Script to seed test data for the 'content-excludes' export CI job.
|
||||
|
||||
'export-records' can optionally include/exclude several categories of data
|
||||
(email logs, API tokens, SSO app/token data, user sessions, and group/user
|
||||
permissions) via --include-x / --exclude-x flags. Toggling one of those flags
|
||||
only proves anything if the source database actually contains a row in that
|
||||
category to begin with - otherwise "the export doesn't contain it" is true
|
||||
regardless of whether the flag/exclusion logic works at all.
|
||||
|
||||
This script creates exactly one row in each such category, so the
|
||||
import_export.yaml workflow's content-excludes job can meaningfully assert
|
||||
both "included when asked for" and "excluded by default".
|
||||
|
||||
Intended to be run from 'src/backend/InvenTree', e.g.:
|
||||
cd src/backend/InvenTree && python ../../../.github/scripts/seed_content_excludes_data.py
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.getcwd())
|
||||
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'InvenTree.settings')
|
||||
|
||||
import django
|
||||
|
||||
django.setup()
|
||||
|
||||
from django.contrib.auth import get_user_model
|
||||
from django.contrib.auth.models import Group, Permission
|
||||
from django.contrib.sessions.backends.db import SessionStore
|
||||
|
||||
from allauth.socialaccount.models import SocialAccount, SocialApp, SocialToken
|
||||
from allauth.usersessions.models import UserSession
|
||||
|
||||
from common.models import EmailMessage, Priority
|
||||
from users.models import ApiToken
|
||||
|
||||
User = get_user_model()
|
||||
|
||||
|
||||
def main():
|
||||
"""Seed one row of test data in each optional export/import category."""
|
||||
user = User.objects.filter(is_superuser=True).first()
|
||||
|
||||
if user is None:
|
||||
print('Error: no superuser found - run `invoke dev.setup-test` first')
|
||||
sys.exit(1)
|
||||
|
||||
# Ensure at least one group has a non-empty permission set, so toggling
|
||||
# --include-permissions has something real to include/strip. Doesn't need
|
||||
# to be a group the superuser belongs to - InvenTree's RuleSet groups
|
||||
# already have permissions assigned regardless of membership.
|
||||
group = Group.objects.first()
|
||||
|
||||
if group is None:
|
||||
print('Error: no groups found - run `invoke dev.setup-test` first')
|
||||
sys.exit(1)
|
||||
|
||||
if not group.permissions.exists():
|
||||
group.permissions.add(Permission.objects.first())
|
||||
print(f"- Added a permission to group '{group.name}' (was empty)")
|
||||
else:
|
||||
print(f"- Group '{group.name}' already has permissions")
|
||||
|
||||
# Email log entry (thread is auto-created by EmailMessage.save() if omitted)
|
||||
EmailMessage.objects.get_or_create(
|
||||
subject='CI content-excludes test email',
|
||||
defaults={
|
||||
'body': 'CI content-excludes test email body',
|
||||
'to': 'ci-recipient@example.com',
|
||||
'sender': 'ci-sender@example.com',
|
||||
'priority': Priority.NORMAL,
|
||||
},
|
||||
)
|
||||
print('- Created email log entry')
|
||||
|
||||
# API token
|
||||
ApiToken.objects.get_or_create(user=user, name='ci-content-excludes-token')
|
||||
print('- Created API token')
|
||||
|
||||
# SSO application + linked account + token
|
||||
app, _ = SocialApp.objects.get_or_create(
|
||||
provider='google',
|
||||
name='CI Content-Excludes Test App',
|
||||
defaults={'client_id': 'ci-test-client-id'},
|
||||
)
|
||||
account, _ = SocialAccount.objects.get_or_create(
|
||||
user=user, provider='google', uid='ci-test-external-uid'
|
||||
)
|
||||
SocialToken.objects.get_or_create(
|
||||
app=app, account=account, defaults={'token': 'ci-test-token-value'}
|
||||
)
|
||||
print('- Created SSO application, account and token')
|
||||
|
||||
# A real, properly-encoded session (avoids writing an undecodable session_data blob)
|
||||
store = SessionStore()
|
||||
store['ci_content_excludes_test'] = True
|
||||
store.create()
|
||||
print('- Created session entry')
|
||||
|
||||
# allauth user-session record (tracked separately from the raw Session table)
|
||||
UserSession.objects.get_or_create(
|
||||
session_key='ci-content-excludes-user-session',
|
||||
defaults={'user': user, 'ip': '127.0.0.1', 'user_agent': 'ci-test-agent'},
|
||||
)
|
||||
print('- Created user session entry')
|
||||
|
||||
print('Content-excludes seed data created successfully')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user