From 7c28bf1f64245fbdf3408671e87c936b818cb7da Mon Sep 17 00:00:00 2001 From: Matthias Mair Date: Sat, 11 Jun 2022 02:07:57 +0200 Subject: [PATCH] Clean up tasks (#3175) * Remove shell command Fixes #3157 * Also run translation stats on updates * remove style command * remove import_fixtures command * remove check * fix docstirngs regarding D415 * move function up * move task * move task * move tasks * move task * add section comments * Revert "remove import_fixtures command" This reverts commit e202ff2b805c12f7b858aa2bdb90b43eff465e68. --- tasks.py | 458 +++++++++++++++++++++++++++---------------------------- 1 file changed, 222 insertions(+), 236 deletions(-) diff --git a/tasks.py b/tasks.py index fdec7b4fcf..36e8a20678 100644 --- a/tasks.py +++ b/tasks.py @@ -26,238 +26,8 @@ def apps(): ] -def localDir(): - """Returns the directory of *THIS* file. - - Used to ensure that the various scripts always run - in the correct directory. - """ - return os.path.dirname(os.path.abspath(__file__)) - - -def managePyDir(): - """Returns the directory of the manage.py file""" - return os.path.join(localDir(), 'InvenTree') - - -def managePyPath(): - """Return the path of the manage.py file""" - return os.path.join(managePyDir(), 'manage.py') - - -def manage(c, cmd, pty: bool = False): - """Runs a given command against django's "manage.py" script. - - Args: - c: Command line context. - cmd: Django command to run. - pty (bool, optional): Run an interactive session. Defaults to False. - """ - c.run('cd "{path}" && python3 manage.py {cmd}'.format( - path=managePyDir(), - cmd=cmd - ), pty=pty) - - -@task -def plugins(c): - """Installs all plugins as specified in 'plugins.txt'""" - from InvenTree.InvenTree.config import get_plugin_file - - plugin_file = get_plugin_file() - - print(f"Installing plugin packages from '{plugin_file}'") - - # Install the plugins - c.run(f"pip3 install --disable-pip-version-check -U -r '{plugin_file}'") - - -@task(post=[plugins]) -def install(c): - """Installs required python packages""" - print("Installing required python packages from 'requirements.txt'") - - # Install required Python packages with PIP - c.run('pip3 install --no-cache-dir --disable-pip-version-check -U -r requirements.txt') - - -@task -def setup_dev(c): - """Sets up everything needed for the dev enviroment""" - print("Installing required python packages from 'requirements.txt'") - - # Install required Python packages with PIP - c.run('pip3 install -U -r requirements.txt') - - # Install pre-commit hook - c.run('pre-commit install') - - # Update all the hooks - c.run('pre-commit autoupdate') - - -@task -def shell(c): - """Open a python shell with access to the InvenTree database models.""" - manage(c, 'shell', pty=True) - - -@task -def superuser(c): - """Create a superuser/admin account for the database.""" - manage(c, 'createsuperuser', pty=True) - - -@task -def check(c): - """Check validity of django codebase""" - manage(c, "check") - - -@task -def wait(c): - """Wait until the database connection is ready""" - return manage(c, "wait_for_db") - - -@task(pre=[wait]) -def worker(c): - """Run the InvenTree background worker process""" - manage(c, 'qcluster', pty=True) - - -@task -def rebuild_models(c): - """Rebuild database models with MPTT structures""" - manage(c, "rebuild_models", pty=True) - - -@task -def rebuild_thumbnails(c): - """Rebuild missing image thumbnails""" - manage(c, "rebuild_thumbnails", pty=True) - - -@task -def clean_settings(c): - """Clean the setting tables of old settings""" - manage(c, "clean_settings") - - -@task(help={'mail': 'mail of the user whos MFA should be disabled'}) -def remove_mfa(c, mail=''): - """Remove MFA for a user""" - if not mail: - print('You must provide a users mail') - - manage(c, f"remove_mfa {mail}") - - -@task(post=[rebuild_models, rebuild_thumbnails]) -def migrate(c): - """Performs database migrations. - - This is a critical step if the database schema have been altered! - """ - print("Running InvenTree database migrations...") - print("========================================") - - manage(c, "makemigrations") - manage(c, "migrate --noinput") - manage(c, "migrate --run-syncdb") - manage(c, "check") - - print("========================================") - print("InvenTree database migrations completed!") - - -@task -def static(c): - """Copies required static files to the STATIC_ROOT directory, as per Django requirements.""" - manage(c, "prerender") - manage(c, "collectstatic --no-input") - - -@task -def translate_stats(c): - """Collect translation stats. - - The file generated from this is needed for the UI. - """ - path = os.path.join('InvenTree', 'script', 'translation_stats.py') - c.run(f'python3 {path}') - - -@task(post=[translate_stats, static]) -def translate(c): - """Rebuild translation source files. (Advanced use only!) - - Note: This command should not be used on a local install, - it is performed as part of the InvenTree translation toolchain. - """ - # Translate applicable .py / .html / .js files - manage(c, "makemessages --all -e py,html,js --no-wrap") - manage(c, "compilemessages") - - -@task(pre=[install, migrate, static, clean_settings]) -def update(c): - """Update InvenTree installation. - - This command should be invoked after source code has been updated, - e.g. downloading new code from GitHub. - - The following tasks are performed, in order: - - - install - - migrate - - translate_stats - - static - - clean_settings - """ - # Recompile the translation files (.mo) - # We do not run 'invoke translate' here, as that will touch the source (.po) files too! - manage(c, 'compilemessages', pty=True) - - -@task -def style(c): - """Run PEP style checks against InvenTree sourcecode""" - print("Running PEP style checks...") - c.run('flake8 InvenTree tasks.py') - - -@task -def test(c, database=None): - """Run unit-tests for InvenTree codebase.""" - # Run sanity check on the django install - manage(c, 'check') - - # Run coverage tests - manage(c, 'test', pty=True) - - -@task -def coverage(c): - """Run code-coverage of the InvenTree codebase, using the 'coverage' code-analysis tools. - - Generates a code coverage report (available in the htmlcov directory) - """ - # Run sanity check on the django install - manage(c, 'check') - - # Run coverage tests - c.run('coverage run {manage} test {apps}'.format( - manage=managePyPath(), - apps=' '.join(apps()) - )) - - # Generate coverage report - c.run('coverage html') - - def content_excludes(): - """Returns a list of content types to exclude from import/export""" + """Returns a list of content types to exclude from import/export.""" excludes = [ "contenttypes", "auth.permission", @@ -282,6 +52,179 @@ def content_excludes(): return output +def localDir(): + """Returns the directory of *THIS* file. + + Used to ensure that the various scripts always run + in the correct directory. + """ + return os.path.dirname(os.path.abspath(__file__)) + + +def managePyDir(): + """Returns the directory of the manage.py file.""" + return os.path.join(localDir(), 'InvenTree') + + +def managePyPath(): + """Return the path of the manage.py file.""" + return os.path.join(managePyDir(), 'manage.py') + + +def manage(c, cmd, pty: bool = False): + """Runs a given command against django's "manage.py" script. + + Args: + c: Command line context. + cmd: Django command to run. + pty (bool, optional): Run an interactive session. Defaults to False. + """ + c.run('cd "{path}" && python3 manage.py {cmd}'.format( + path=managePyDir(), + cmd=cmd + ), pty=pty) + + +# Install tasks +@task +def plugins(c): + """Installs all plugins as specified in 'plugins.txt'.""" + from InvenTree.InvenTree.config import get_plugin_file + + plugin_file = get_plugin_file() + + print(f"Installing plugin packages from '{plugin_file}'") + + # Install the plugins + c.run(f"pip3 install --disable-pip-version-check -U -r '{plugin_file}'") + + +@task(post=[plugins]) +def install(c): + """Installs required python packages.""" + print("Installing required python packages from 'requirements.txt'") + + # Install required Python packages with PIP + c.run('pip3 install --no-cache-dir --disable-pip-version-check -U -r requirements.txt') + + +@task +def setup_dev(c): + """Sets up everything needed for the dev enviroment.""" + print("Installing required python packages from 'requirements.txt'") + + # Install required Python packages with PIP + c.run('pip3 install -U -r requirements.txt') + + # Install pre-commit hook + c.run('pre-commit install') + + # Update all the hooks + c.run('pre-commit autoupdate') + + +# Setup / maintenance tasks +@task +def superuser(c): + """Create a superuser/admin account for the database.""" + manage(c, 'createsuperuser', pty=True) + + +@task +def rebuild_models(c): + """Rebuild database models with MPTT structures.""" + manage(c, "rebuild_models", pty=True) + + +@task +def rebuild_thumbnails(c): + """Rebuild missing image thumbnails.""" + manage(c, "rebuild_thumbnails", pty=True) + + +@task +def clean_settings(c): + """Clean the setting tables of old settings.""" + manage(c, "clean_settings") + + +@task(help={'mail': 'mail of the user whos MFA should be disabled'}) +def remove_mfa(c, mail=''): + """Remove MFA for a user.""" + if not mail: + print('You must provide a users mail') + + manage(c, f"remove_mfa {mail}") + + +@task +def static(c): + """Copies required static files to the STATIC_ROOT directory, as per Django requirements.""" + manage(c, "prerender") + manage(c, "collectstatic --no-input") + + +@task +def translate_stats(c): + """Collect translation stats. + + The file generated from this is needed for the UI. + """ + path = os.path.join('InvenTree', 'script', 'translation_stats.py') + c.run(f'python3 {path}') + + +@task(post=[translate_stats, static]) +def translate(c): + """Rebuild translation source files. Advanced use only! + + Note: This command should not be used on a local install, + it is performed as part of the InvenTree translation toolchain. + """ + # Translate applicable .py / .html / .js files + manage(c, "makemessages --all -e py,html,js --no-wrap") + manage(c, "compilemessages") + + +@task(post=[rebuild_models, rebuild_thumbnails]) +def migrate(c): + """Performs database migrations. + + This is a critical step if the database schema have been altered! + """ + print("Running InvenTree database migrations...") + print("========================================") + + manage(c, "makemigrations") + manage(c, "migrate --noinput") + manage(c, "migrate --run-syncdb") + manage(c, "check") + + print("========================================") + print("InvenTree database migrations completed!") + + +@task(pre=[install, migrate, static, clean_settings, translate_stats]) +def update(c): + """Update InvenTree installation. + + This command should be invoked after source code has been updated, + e.g. downloading new code from GitHub. + + The following tasks are performed, in order: + + - install + - migrate + - static + - clean_settings + - translate_stats + """ + # Recompile the translation files (.mo) + # We do not run 'invoke translate' here, as that will touch the source (.po) files too! + manage(c, 'compilemessages', pty=True) + + +# Data tasks @task(help={ 'filename': "Output filename (default = 'data.json')", 'overwrite': "Overwrite existing files without asking first (default = off/False)", @@ -359,7 +302,7 @@ def export_records(c, filename='data.json', overwrite=False, include_permissions @task(help={'filename': 'Input filename', 'clear': 'Clear existing data before import'}, post=[rebuild_models, rebuild_thumbnails]) def import_records(c, filename='data.json', clear=False): - """Import database records from a file""" + """Import database records from a file.""" # Get an absolute path to the supplied filename if not os.path.isabs(filename): filename = os.path.join(localDir(), filename) @@ -462,6 +405,7 @@ def import_fixtures(c): manage(c, command, pty=True) +# Execution tasks @task(help={'address': 'Server address:port (default=127.0.0.1:8000)'}) def server(c, address="127.0.0.1:8000"): """Launch a (deveopment) server using Django's in-built webserver. @@ -471,9 +415,28 @@ def server(c, address="127.0.0.1:8000"): manage(c, "runserver {address}".format(address=address), pty=True) +@task +def wait(c): + """Wait until the database connection is ready.""" + return manage(c, "wait_for_db") + + +@task(pre=[wait]) +def worker(c): + """Run the InvenTree background worker process.""" + manage(c, 'qcluster', pty=True) + + +# Testing tasks +@task +def render_js_files(c): + """Render templated javascript files (used for static testing).""" + manage(c, "test InvenTree.ci_render_js") + + @task(post=[translate_stats, static, server]) def test_translations(c): - """Add a fictional language to test if each component is ready for translations""" + """Add a fictional language to test if each component is ready for translations.""" import django from django.conf import settings @@ -539,6 +502,29 @@ def test_translations(c): @task -def render_js_files(c): - """Render templated javascript files (used for static testing).""" - manage(c, "test InvenTree.ci_render_js") +def test(c, database=None): + """Run unit-tests for InvenTree codebase.""" + # Run sanity check on the django install + manage(c, 'check') + + # Run coverage tests + manage(c, 'test', pty=True) + + +@task +def coverage(c): + """Run code-coverage of the InvenTree codebase, using the 'coverage' code-analysis tools. + + Generates a code coverage report (available in the htmlcov directory) + """ + # Run sanity check on the django install + manage(c, 'check') + + # Run coverage tests + c.run('coverage run {manage} test {apps}'.format( + manage=managePyPath(), + apps=' '.join(apps()) + )) + + # Generate coverage report + c.run('coverage html')