mirror of
https://github.com/inventree/InvenTree.git
synced 2026-09-10 06:37:17 +00:00
Adds bulk-import option for "import-records" task (#12792)
* skip plugin events if importing or migrating data * Add bulkloaddata option * Skip signals if importing * Improve bulkloaddata command * Optionally rebuild thumbnails * enhancements for import_records task * cache natural key references in bulkloaddata * wrap export_records in @state_logger * Reduce file size of exported data * Added docs * Test bulk workflow as part of CI * Add progress bar for data import * fix for import workflow bug * Separately test bulk import workflow * Exercise --prettify option * Additional CI checks for content excludes * Allow plugin loading for list_apps * Additional CI unit tests * Test for importing with conflicting records * path fixes * Adjust test conditions
This commit is contained in:
Oliver
@@ -10,6 +10,14 @@ In reads the exported data file, to ensure that:
|
||||
- The file contains the expected plugin configuration
|
||||
- The file contains the expected plugin database records
|
||||
|
||||
It can also optionally check the presence / absence of several categories of
|
||||
data which 'export-records' can include or exclude via --include-x /
|
||||
--exclude-x flags (email logs, API tokens, SSO app/token data, user sessions,
|
||||
and non-empty group/user permissions) - pass e.g. '--check-email include' or
|
||||
'--check-email exclude' to assert that category was (or was not) found in the
|
||||
exported data. Any '--check-x' option which is *not* passed is simply not
|
||||
checked at all (not even implicitly assumed absent) - so existing invocations
|
||||
which don't pass any of them keep working unchanged.
|
||||
"""
|
||||
|
||||
PLUGIN_KEY = 'dummy_app_plugin'
|
||||
@@ -19,10 +27,112 @@ import argparse
|
||||
import json
|
||||
import os
|
||||
|
||||
|
||||
def check_category(
|
||||
data: list[dict], label: str, model_names: list[str], expect: str | None
|
||||
):
|
||||
"""Check that all of the given model names are present / absent as expected.
|
||||
|
||||
Arguments:
|
||||
data: The loaded (parsed) exported data file.
|
||||
label: A human-readable label for this category, for error messages.
|
||||
model_names: The Django model labels (e.g. 'common.emailmessage')
|
||||
which make up this category.
|
||||
expect: 'include' - at least one entry for *each* model name must be
|
||||
present. 'exclude' - *no* entry for *any* model name may be
|
||||
present. None - this category was not requested to be checked;
|
||||
do nothing.
|
||||
"""
|
||||
if expect is None:
|
||||
return
|
||||
|
||||
expect_present = expect == 'include'
|
||||
|
||||
counts = dict.fromkeys(model_names, 0)
|
||||
|
||||
for entry in data:
|
||||
model = entry.get('model', None)
|
||||
if model in counts:
|
||||
counts[model] += 1
|
||||
|
||||
if expect_present:
|
||||
for model, count in counts.items():
|
||||
if count == 0:
|
||||
print(f"Error: Expected '{label}' data ('{model}') was not found")
|
||||
exit(1)
|
||||
print(f"Found expected '{label}' data ({counts})")
|
||||
else:
|
||||
for model, count in counts.items():
|
||||
if count > 0:
|
||||
print(
|
||||
f"Error: '{label}' data ('{model}') was found, but should have been excluded ({count} record(s))"
|
||||
)
|
||||
exit(1)
|
||||
print(f"Confirmed '{label}' data was correctly excluded")
|
||||
|
||||
|
||||
def check_permissions(data: list[dict], expect: str | None):
|
||||
"""Check that auth.group / auth.user permission fields are stripped or preserved as expected.
|
||||
|
||||
Arguments:
|
||||
data: The loaded (parsed) exported data file.
|
||||
expect: 'include' - at least one auth.group / auth.user entry must
|
||||
have non-empty permissions. 'exclude' - all such entries must have
|
||||
empty permissions. None - not checked; do nothing.
|
||||
"""
|
||||
if expect is None:
|
||||
return
|
||||
|
||||
expect_present = expect == 'include'
|
||||
|
||||
group_perms = [
|
||||
entry['fields'].get('permissions', [])
|
||||
for entry in data
|
||||
if entry.get('model') == 'auth.group'
|
||||
]
|
||||
user_perms = [
|
||||
entry['fields'].get('user_permissions', [])
|
||||
for entry in data
|
||||
if entry.get('model') == 'auth.user'
|
||||
]
|
||||
|
||||
any_group_perms = any(group_perms)
|
||||
any_user_perms = any(user_perms)
|
||||
|
||||
if expect_present:
|
||||
if not any_group_perms and not any_user_perms:
|
||||
print(
|
||||
'Error: Expected at least one auth.group / auth.user entry with '
|
||||
'non-empty permissions, but all were empty'
|
||||
)
|
||||
exit(1)
|
||||
print('Found expected non-empty group/user permissions')
|
||||
else:
|
||||
if any_group_perms or any_user_perms:
|
||||
print(
|
||||
'Error: Found non-empty group/user permissions, but they should '
|
||||
'have been stripped'
|
||||
)
|
||||
exit(1)
|
||||
print('Confirmed group/user permissions were correctly stripped')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description='Check exported data file')
|
||||
parser.add_argument('datafile', help='Path to the exported data file (JSON)')
|
||||
|
||||
# Plugin data is checked unconditionally below (it always has been) - this
|
||||
# just controls which direction is expected, mirroring export-records' own
|
||||
# --exclude-plugins flag (plugin data is included by default).
|
||||
parser.add_argument('--exclude-plugins', action='store_true')
|
||||
|
||||
# The remaining categories are only checked when explicitly requested -
|
||||
# pass 'include' or 'exclude' to assert that direction, or omit the flag
|
||||
# entirely to skip checking that category (the default, for backwards
|
||||
# compatibility with existing invocations that don't pass any of these).
|
||||
for flag in ('email', 'tokens', 'sso', 'session', 'permissions'):
|
||||
parser.add_argument(f'--check-{flag}', choices=['include', 'exclude'])
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if not os.path.isfile(args.datafile):
|
||||
@@ -84,18 +194,45 @@ if __name__ == '__main__':
|
||||
)
|
||||
exit(1)
|
||||
|
||||
if not found_plugin_config:
|
||||
print(f'Error: No plugin configuration found for plugin "{PLUGIN_KEY}"')
|
||||
exit(1)
|
||||
|
||||
# Check the extracted plugin records
|
||||
expected_keys = ['alpha', 'beta', 'gamma', 'delta']
|
||||
|
||||
for key in expected_keys:
|
||||
if key not in plugin_data_records:
|
||||
print(
|
||||
f'Error: Expected plugin record with key "{key}" not found in exported data'
|
||||
)
|
||||
# Plugin data is included by default (export-records only excludes it when
|
||||
# given --exclude-plugins), so preserve that as the default expectation here
|
||||
if not args.exclude_plugins:
|
||||
if not found_plugin_config:
|
||||
print(f'Error: No plugin configuration found for plugin "{PLUGIN_KEY}"')
|
||||
exit(1)
|
||||
|
||||
# Check the extracted plugin records
|
||||
expected_keys = ['alpha', 'beta', 'gamma', 'delta']
|
||||
|
||||
for key in expected_keys:
|
||||
if key not in plugin_data_records:
|
||||
print(
|
||||
f'Error: Expected plugin record with key "{key}" not found in exported data'
|
||||
)
|
||||
exit(1)
|
||||
elif found_plugin_config:
|
||||
print('Error: Plugin data was found, but should have been excluded')
|
||||
exit(1)
|
||||
else:
|
||||
print('Confirmed plugin data was correctly excluded')
|
||||
|
||||
# Content-excludes checks - only run for '--check-x' flags that were actually passed
|
||||
check_category(
|
||||
data, 'email', ['common.emailmessage', 'common.emailthread'], args.check_email
|
||||
)
|
||||
check_category(data, 'tokens', ['users.apitoken'], args.check_tokens)
|
||||
check_category(
|
||||
data,
|
||||
'sso',
|
||||
['socialaccount.socialapp', 'socialaccount.socialtoken'],
|
||||
args.check_sso,
|
||||
)
|
||||
check_category(
|
||||
data,
|
||||
'session',
|
||||
['sessions.session', 'usersessions.usersession'],
|
||||
args.check_session,
|
||||
)
|
||||
check_permissions(data, args.check_permissions)
|
||||
|
||||
print('All checks passed successfully!')
|
||||
|
||||
Reference in New Issue
Block a user