From 4b3b03ed4b75230ad13a2df3769b522d63ab85aa Mon Sep 17 00:00:00 2001 From: Oliver Date: Fri, 10 Apr 2026 07:58:53 +1000 Subject: [PATCH] Invoke verbosity (#11706) * Reduce verbosity of invoke tasks - Suppress some django messages which are not useful to most users - Verbosity can be added with --verbose flag * Further improvements * Better messaging * Extra options * No! --- .../management/commands/wait_for_db.py | 34 ++- src/frontend/vite.config.ts | 2 +- tasks.py | 206 ++++++++++++------ 3 files changed, 170 insertions(+), 72 deletions(-) diff --git a/src/backend/InvenTree/InvenTree/management/commands/wait_for_db.py b/src/backend/InvenTree/InvenTree/management/commands/wait_for_db.py index 68b33adfff..84fcb2ebff 100644 --- a/src/backend/InvenTree/InvenTree/management/commands/wait_for_db.py +++ b/src/backend/InvenTree/InvenTree/management/commands/wait_for_db.py @@ -13,25 +13,39 @@ class Command(BaseCommand): def handle(self, *args, **kwargs): """Wait till the database is ready.""" - self.stdout.write('Waiting for database...') - connected = False + verbose = int(kwargs.get('verbosity', 0)) > 0 + attempts = kwargs.get('attempts', 10) - while not connected: - time.sleep(2) + if verbose: + self.stdout.write('Waiting for database connection...') + self.stdout.flush() + + while not connected and attempts > 0: + attempts -= 1 try: connection.ensure_connection() connected = True - except OperationalError as e: - self.stdout.write(f'Could not connect to database: {e}') - except ImproperlyConfigured as e: - self.stdout.write(f'Improperly configured: {e}') + except (OperationalError, ImproperlyConfigured): + if verbose: + self.stdout.write('Database connection failed, retrying ...') + self.stdout.flush() else: if not connection.is_usable(): - self.stdout.write('Database configuration is not usable') + if verbose: + self.stdout.write('Database configuration is not usable') + self.stdout.flush() if connected: - self.stdout.write('Database connection successful!') + if verbose: + self.stdout.write('Database connection successful!') + self.stdout.flush() + else: + time.sleep(1) + + if not connected: + self.stderr.write('Failed to connect to database after multiple attempts') + self.stderr.flush() diff --git a/src/frontend/vite.config.ts b/src/frontend/vite.config.ts index a658d92aa7..627a1ade6e 100644 --- a/src/frontend/vite.config.ts +++ b/src/frontend/vite.config.ts @@ -13,7 +13,7 @@ import { __INVENTREE_VERSION_INFO__ } from './version-info'; const IS_IN_WSL = platform().includes('WSL') || release().includes('WSL'); if (IS_IN_WSL) { - console.log('WSL detected: using polling for file system events'); + console.debug('WSL detected: using polling for file system events'); } // Output directory for the built files diff --git a/tasks.py b/tasks.py index b937e768c9..d9380bea12 100644 --- a/tasks.py +++ b/tasks.py @@ -9,6 +9,7 @@ import shutil import subprocess import sys import tempfile +import time from functools import wraps from pathlib import Path from platform import python_version @@ -200,9 +201,13 @@ def state_logger(fn=None, method_name=None): do_log = is_debug_environment() if do_log: info(f'# task | {func.method_name} | start') + + t1 = time.time() func(c, *args, **kwargs) + t2 = time.time() + if do_log: - info(f'# task | {func.method_name} | done') + info(f'# task | {func.method_name} | done | elapsed: {t2 - t1:.2f}s') return wrapped @@ -464,7 +469,7 @@ def run( return result -def manage(c, cmd, pty: bool = False, env=None, **kwargs): +def manage(c, cmd, pty: bool = False, env=None, verbose: bool = False, **kwargs): """Runs a given command against django's "manage.py" script. Args: @@ -472,7 +477,14 @@ def manage(c, cmd, pty: bool = False, env=None, **kwargs): cmd: Django command to run. pty (bool, optional): Run an interactive session. Defaults to False. env (dict, optional): Environment variables to pass to the command. Defaults to None. + verbose (bool, optional): Print verbose output from the command. Defaults to False. """ + if verbose: + info(f'Running command: python3 manage.py {cmd}') + cmd += ' -v 1' + else: + cmd += ' -v 0' + return run( c, f'python3 manage.py {cmd}', manage_py_dir(), pty=pty, env=env, **kwargs ) @@ -499,8 +511,19 @@ def run_install( run_preflight=True, version_check=False, pinned=True, + verbose: bool = False, ): - """Run the installation of python packages from a requirements file.""" + """Run the installation of python packages from a requirements file. + + Arguments: + c: Command line context. + uv: Whether to use UV (experimental package manager) instead of pip. + install_file: Path to the requirements file to install from. + run_preflight: Whether to run the preflight installation step (installing pip/uv itself). Default is True. + version_check: Whether to check for a version-specific requirements file. Default is False. + pinned: Whether to use the --require-hashes option when installing packages. Default is True. + verbose: Whether to print verbose output from pip install commands. Default is False. + """ if version_check: # Test if there is a version specific requirements file sys_ver_s = python_version().split('.') @@ -518,25 +541,28 @@ def run_install( # Install required Python packages with PIP if not uv: + # Optionally run preflight first if run_preflight: run( c, - 'pip3 install --no-cache-dir --disable-pip-version-check -U pip setuptools', + f'pip3 install --no-cache-dir --disable-pip-version-check -U pip setuptools {"" if verbose else "--quiet"}', ) + info('Installed package manager') + run( c, - f'pip3 install --no-cache-dir --disable-pip-version-check -U {"--require-hashes" if pinned else ""} -r {install_file}', + f'pip3 install --no-cache-dir --disable-pip-version-check -U {"--require-hashes" if pinned else ""} -r {install_file} {"" if verbose else "--quiet"}', ) else: if run_preflight: run( c, - 'pip3 install --no-cache-dir --disable-pip-version-check -U uv setuptools', + f'pip3 install --no-cache-dir --disable-pip-version-check -U uv setuptools {"" if verbose else "--quiet"}', ) info('Installed package manager') run( c, - f'uv pip install -U {"--require-hashes" if pinned else ""} -r {install_file}', + f'uv pip install -U {"--require-hashes" if pinned else ""} -r {install_file} {"" if verbose else "--quiet"}', ) @@ -605,28 +631,34 @@ def check_file_existence(filename: Path, overwrite: bool = False): sys.exit(1) -@task +@task(help={'verbose': 'Print verbose output from the command'}) @state_logger -def wait(c): +def wait(c, verbose: bool = False): """Wait until the database connection is ready.""" - info('Waiting for database connection...') - return manage(c, 'wait_for_db') + return manage(c, 'wait_for_db', verbose=verbose) # Install tasks # region tasks -@task(help={'uv': 'Use UV (experimental package manager)'}) +@task( + help={ + 'uv': 'Use UV (experimental package manager)', + 'verbose': 'Print verbose output from installation commands', + } +) @state_logger -def plugins(c, uv=False): +def plugins(c, uv: bool = False, verbose: bool = False): """Installs all plugins as specified in 'plugins.txt'.""" from src.backend.InvenTree.InvenTree.config import ( # type: ignore[import] get_plugin_file, ) - run_install(c, uv, get_plugin_file(), run_preflight=False, pinned=False) + run_install( + c, uv, get_plugin_file(), run_preflight=False, pinned=False, verbose=verbose + ) # Collect plugin static files - manage(c, 'collectplugins') + manage(c, 'collectplugins', verbose=verbose) @task( @@ -634,29 +666,51 @@ def plugins(c, uv=False): 'uv': 'Use UV package manager (experimental)', 'skip_plugins': 'Skip plugin installation', 'dev': 'Install development requirements instead of production requirements', + 'verbose': 'Print verbose output from pip install commands', } ) @state_logger -def install(c, uv=False, skip_plugins=False, dev=False): - """Installs required python packages.""" +def install( + c, + uv: bool = False, + skip_plugins: bool = False, + dev: bool = False, + verbose: bool = False, +): + """Install required python packages for InvenTree. + + Arguments: + c: Command line context. + uv: Use UV package manager (experimental) instead of pip. Default is False. + skip_plugins: Skip plugin installation. Default is False. + dev: Install development requirements instead of production requirements. Default is False. + verbose: Print verbose output from pip install commands. Default is False. + """ + info('Installing required python packages...') + if dev: run_install( c, uv, local_dir().joinpath('src/backend/requirements-dev.txt'), version_check=True, + verbose=verbose, ) success('Dependency installation complete') return # Ensure path is relative to *this* directory run_install( - c, uv, local_dir().joinpath('src/backend/requirements.txt'), version_check=True + c, + uv, + local_dir().joinpath('src/backend/requirements.txt'), + version_check=True, + verbose=verbose, ) # Run plugins install if not skip_plugins: - plugins(c, uv=uv) + plugins(c, uv=uv, verbose=verbose) # Compile license information lic_path = manage_py_dir().joinpath('InvenTree', 'licenses.txt') @@ -668,24 +722,26 @@ def install(c, uv=False, skip_plugins=False, dev=False): success('Dependency installation complete') -@task(help={'tests': 'Set up test dataset at the end'}) -def setup_dev(c, tests=False): +@task( + help={ + 'tests': 'Set up test dataset at the end', + 'verbose': 'Print verbose output from commands', + } +) +def setup_dev(c, tests: bool = False, verbose: bool = False): """Sets up everything needed for the dev environment.""" # Install required Python packages with PIP - install(c, uv=False, skip_plugins=True, dev=True) + install(c, uv=False, skip_plugins=True, dev=True, verbose=verbose) # Install pre-commit hook info('Installing pre-commit for checks before git commits...') run(c, 'pre-commit install') - - # Update all the hooks run(c, 'pre-commit autoupdate') - success('pre-commit set up complete') # Set up test-data if flag is set if tests: - setup_test(c) + setup_test(c, verbose=verbose) # Setup / maintenance tasks @@ -725,7 +781,7 @@ def rebuild_thumbnails(c): @state_logger def clean_settings(c): """Clean the setting tables of old settings.""" - info('Cleaning old settings from the database') + info('Cleaning old settings from the database...') manage(c, 'clean_settings') success('Settings cleaned successfully') @@ -797,12 +853,12 @@ def translate(c, ignore_static=False, no_frontend=False): success('Translation files built successfully') -@task +@task(help={'verbose': 'Print verbose output'}) @state_logger('backend_trans') -def backend_trans(c): +def backend_trans(c, verbose: bool = False): """Compile backend Django translation files.""" - info('Compiling backend translations') - manage(c, 'compilemessages') + info('Compiling backend translations...') + manage(c, 'compilemessages', verbose=verbose) success('Backend translations compiled successfully') @@ -944,20 +1000,33 @@ def listbackups(c): manage(c, 'listbackups') -@task(pre=[wait], post=[rebuild_models, rebuild_thumbnails]) +@task( + pre=[wait], + post=[rebuild_models, rebuild_thumbnails], + help={ + 'verbose': 'Print verbose output from migration commands', + 'detect': 'Detect and create new migrations based on changes to models', + }, +) @state_logger -def migrate(c): +def migrate(c, detect: bool = True, verbose: bool = False): """Performs database migrations. This is a critical step if the database schema have been altered! """ info('Running InvenTree database migrations...') - # Run custom management command which wraps migrations in "maintenance mode" - manage(c, 'makemigrations') - manage(c, 'runmigrations', pty=True) - manage(c, 'migrate --run-syncdb') - manage(c, 'remove_stale_contenttypes --include-stale-apps --no-input', pty=True) + if detect: + manage(c, 'makemigrations', verbose=verbose) + + manage(c, 'runmigrations', pty=True, verbose=verbose) + manage(c, 'migrate --run-syncdb', verbose=verbose) + manage( + c, + 'remove_stale_contenttypes --include-stale-apps --no-input', + pty=True, + verbose=verbose, + ) success('InvenTree database migrations completed') @@ -978,6 +1047,7 @@ def showmigrations(c, app=''): 'no_frontend': 'Skip frontend compilation/download step', 'skip_static': 'Skip static file collection step', 'uv': 'Use UV (experimental package manager)', + 'verbose': 'Print verbose output from installation commands', }, ) @state_logger @@ -990,6 +1060,7 @@ def update( no_frontend: bool = False, skip_static: bool = False, uv: bool = False, + verbose: bool = False, ): """Update InvenTree installation. @@ -1009,7 +1080,7 @@ def update( info('Updating InvenTree installation...') # Ensure required components are installed - install(c, uv=uv) + install(c, uv=uv, verbose=verbose) # Skip backend translation compilation on docker, unless explicitly requested. # Users can also forcefully disable the step via `--no-backend`. @@ -1019,7 +1090,7 @@ def update( else: info('Skipping backend translation compilation (INVENTREE_DOCKER flag set)') else: - backend_trans(c) + backend_trans(c, verbose=verbose) if not skip_backup: backup(c) @@ -1052,7 +1123,7 @@ def update( # Note: frontend has already been compiled if required static(c, frontend=False) - success('InvenTree update complete!') + success('InvenTree update complete') # Data tasks @@ -1066,8 +1137,8 @@ def update( 'exclude_plugins': 'Exclude plugin data from the output file (default = False)', 'include_sso': 'Include SSO token data in the output file (default = False)', 'include_session': 'Include user session data in the output file (default = False)', - }, - pre=[wait], + 'verbose': 'Print verbose output from management commands', + } ) def export_records( c, @@ -1079,6 +1150,7 @@ def export_records( exclude_plugins: bool = False, include_sso: bool = False, include_session: bool = False, + verbose: bool = False, ): """Export all database records to a file.""" # Get an absolute path to the file @@ -1088,6 +1160,8 @@ def export_records( info(f"Exporting database records to file '{target}'") + wait(c, verbose=verbose) + check_file_existence(target, overwrite) excludes = content_excludes( @@ -1104,8 +1178,7 @@ def export_records( cmd = f"dumpdata --natural-foreign --indent 2 --output '{tmpfile.name}' {excludes}" # Dump data to temporary file - manage(c, cmd, pty=True) - + manage(c, cmd, pty=True, verbose=verbose) info('Running data post-processing step...') # Post-process the file, to remove any "permissions" specified for a user or group @@ -1217,6 +1290,7 @@ def validate_import_metadata( 'ignore_nonexistent': 'Ignore non-existent database models (default = False)', 'exclude_plugins': 'Exclude plugin data from the import process (default = False)', 'skip_migrations': 'Skip the migration step after clearing data (default = False)', + 'verbose': 'Print verbose output from management commands', }, pre=[wait], post=[rebuild_models, rebuild_thumbnails], @@ -1230,6 +1304,7 @@ def import_records( exclude_plugins: bool = False, ignore_nonexistent: bool = False, skip_migrations: bool = False, + verbose: bool = False, ): """Import database records from a file.""" # Get an absolute path to the supplied filename @@ -1243,10 +1318,10 @@ def import_records( sys.exit(1) if clear: - delete_data(c, force=True, migrate=True) + delete_data(c, force=True, migrate=True, verbose=verbose) if not skip_migrations: - migrate(c) + migrate(c, verbose=verbose) info(f"Importing database records from '{target}'") @@ -1274,6 +1349,7 @@ def import_records( ) -> tempfile.NamedTemporaryFile: """Helper function to save data to a temporary file, and then load into the database.""" nonlocal ignore_nonexistent + nonlocal verbose nonlocal c # Skip if there is no data to load @@ -1299,7 +1375,7 @@ def import_records( if excludes: cmd += f' -i {excludes}' - manage(c, cmd, pty=True) + manage(c, cmd, pty=True, verbose=verbose) # Iterate through each entry in the provided data file, and separate out into different categories based on the model type for entry in data: @@ -1363,22 +1439,22 @@ def import_records( help={ 'force': 'Force deletion of all data without confirmation', 'migrate': 'Run migrations before deleting data (default = False)', + 'verbose': 'Print verbose output from management commands', } ) -def delete_data(c, force: bool = False, migrate: bool = False): +def delete_data(c, force: bool = False, migrate: bool = False, verbose: bool = False): """Delete all database records! Warning: This will REALLY delete all records in the database!! """ - info('Deleting all data from InvenTree database...') + info('Deleting existing data from InvenTree database...') if migrate: - manage(c, 'migrate --run-syncdb') + manage(c, 'migrate --run-syncdb', verbose=verbose) - if force: - manage(c, 'flush --noinput') - else: - manage(c, 'flush') + manage(c, f'flush{" --noinput" if force else ""}', verbose=verbose) + + success('Existing data deleted') @task(post=[rebuild_models, rebuild_thumbnails]) @@ -1640,6 +1716,7 @@ def test( 'validate_files': 'Validate media files are correctly copied', 'use_ssh': 'Use SSH protocol for cloning the demo dataset (requires SSH key)', 'branch': 'Specify branch of demo-dataset to clone (default = main)', + 'verbose': 'Print verbose output from management commands', } ) def setup_test( @@ -1648,6 +1725,7 @@ def setup_test( dev=False, validate_files=False, use_ssh=False, + verbose=False, path='inventree-demo-dataset', branch='main', ): @@ -1657,13 +1735,12 @@ def setup_test( ) if not ignore_update: - update(c) + update(c, verbose=verbose) template_dir = local_dir().joinpath(path) # Remove old data directory if template_dir.exists(): - info('Removing old data ...') run(c, f'rm {template_dir} -r') URL = 'https://github.com/inventree/demo-dataset' @@ -1678,11 +1755,16 @@ def setup_test( # Make sure migrations are done - might have just deleted sqlite database if not ignore_update: - migrate(c) + migrate(c, verbose=verbose) # Load data info('Loading database records ...') - import_records(c, filename=template_dir.joinpath('inventree_data.json'), clear=True) + import_records( + c, + filename=template_dir.joinpath('inventree_data.json'), + clear=True, + verbose=verbose, + ) # Copy media files src = template_dir.joinpath('media') @@ -1718,7 +1800,7 @@ def setup_test( # Set up development setup if flag is set if dev: - setup_dev(c) + setup_dev(c, verbose=verbose) @task( @@ -1929,7 +2011,7 @@ def frontend_build(c): Args: c: Context variable """ - info('Building frontend') + info('Building frontend...') yarn(c, 'yarn run build') def write_info(path: Path, content: str): @@ -1957,6 +2039,8 @@ def frontend_build(c): except Exception: warning('Failed to write frontend version marker') + success('Frontend build complete') + @task def frontend_server(c):