From 269b269de3a16ec5285b3d5e21991173138ff112 Mon Sep 17 00:00:00 2001 From: Matthias Mair Date: Tue, 18 Oct 2022 07:54:10 +0200 Subject: [PATCH] Fix task register (#3805) * fix schedule registration * add collection step for tasks * make tasks register configurable * extend docs * Also run InvenTree setup in testing * fix import loading method * fix wrong task registration * do not test * do only distinct testing * ignore import error for coverage --- InvenTree/InvenTree/apps.py | 80 +++++++++++------------------------- InvenTree/InvenTree/tasks.py | 80 ++++++++++++++++++++++++++++++++++++ InvenTree/build/tasks.py | 1 + InvenTree/common/tasks.py | 3 ++ InvenTree/order/tasks.py | 4 +- 5 files changed, 110 insertions(+), 58 deletions(-) diff --git a/InvenTree/InvenTree/apps.py b/InvenTree/InvenTree/apps.py index 7190197104..6e2c020373 100644 --- a/InvenTree/InvenTree/apps.py +++ b/InvenTree/InvenTree/apps.py @@ -1,8 +1,10 @@ """AppConfig for inventree app.""" import logging +from importlib import import_module +from pathlib import Path -from django.apps import AppConfig +from django.apps import AppConfig, apps from django.conf import settings from django.contrib.auth import get_user_model from django.core.exceptions import AppRegistryNotReady @@ -23,10 +25,11 @@ class InvenTreeConfig(AppConfig): def ready(self): """Setup background tasks and update exchange rates.""" - if canAppAccessDatabase(): + if canAppAccessDatabase() or settings.TESTING_ENV: self.remove_obsolete_tasks() + self.collect_tasks() self.start_background_tasks() if not isInTestMode(): # pragma: no cover @@ -54,68 +57,31 @@ class InvenTreeConfig(AppConfig): def start_background_tasks(self): """Start all background tests for InvenTree.""" - try: - from django_q.models import Schedule - except AppRegistryNotReady: # pragma: no cover - logger.warning("Cannot start background tasks - app registry not ready") - return logger.info("Starting background tasks...") - # Remove successful task results from the database - InvenTree.tasks.schedule_task( - 'InvenTree.tasks.delete_successful_tasks', - schedule_type=Schedule.DAILY, - ) + for task in InvenTree.tasks.tasks.task_list: + ref_name = f'{task.func.__module__}.{task.func.__name__}' + InvenTree.tasks.schedule_task( + ref_name, + schedule_type=task.interval, + minutes=task.minutes, + ) - # Check for InvenTree updates - InvenTree.tasks.schedule_task( - 'InvenTree.tasks.check_for_updates', - schedule_type=Schedule.DAILY - ) + logger.info("Started background tasks...") - # Heartbeat to let the server know the background worker is running - InvenTree.tasks.schedule_task( - 'InvenTree.tasks.heartbeat', - schedule_type=Schedule.MINUTES, - minutes=15 - ) + def collect_tasks(self): + """Collect all background tasks.""" - # Keep exchange rates up to date - InvenTree.tasks.schedule_task( - 'InvenTree.tasks.update_exchange_rates', - schedule_type=Schedule.DAILY, - ) + for app_name, app in apps.app_configs.items(): + if app_name == 'InvenTree': + continue - # Delete old error messages - InvenTree.tasks.schedule_task( - 'InvenTree.tasks.delete_old_error_logs', - schedule_type=Schedule.DAILY, - ) - - # Delete old notification records - InvenTree.tasks.schedule_task( - 'common.tasks.delete_old_notifications', - schedule_type=Schedule.DAILY, - ) - - # Check for overdue purchase orders - InvenTree.tasks.schedule_task( - 'order.tasks.check_overdue_purchase_orders', - schedule_type=Schedule.DAILY - ) - - # Check for overdue sales orders - InvenTree.tasks.schedule_task( - 'order.tasks.check_overdue_sales_orders', - schedule_type=Schedule.DAILY, - ) - - # Check for overdue build orders - InvenTree.tasks.schedule_task( - 'build.tasks.check_overdue_build_orders', - schedule_type=Schedule.DAILY - ) + if Path(app.path).joinpath('tasks.py').exists(): + try: + import_module(f'{app.module.__package__}.tasks') + except Exception as e: # pragma: no cover + logger.error(f"Error loading tasks for {app_name}: {e}") def update_exchange_rates(self): # pragma: no cover """Update exchange rates each time the server is started. diff --git a/InvenTree/InvenTree/tasks.py b/InvenTree/InvenTree/tasks.py index 3acb4ff3ad..1c856bebba 100644 --- a/InvenTree/InvenTree/tasks.py +++ b/InvenTree/InvenTree/tasks.py @@ -4,7 +4,9 @@ import json import logging import re import warnings +from dataclasses import dataclass from datetime import timedelta +from typing import Callable from django.conf import settings from django.core import mail as django_mail @@ -126,6 +128,79 @@ def offload_task(taskname, *args, force_async=False, force_sync=False, **kwargs) _func(*args, **kwargs) +@dataclass() +class ScheduledTask: + """A scheduled task. + + - interval: The interval at which the task should be run + - minutes: The number of minutes between task runs + - func: The function to be run + """ + + func: Callable + interval: str + minutes: int = None + + MINUTES = "I" + HOURLY = "H" + DAILY = "D" + WEEKLY = "W" + MONTHLY = "M" + QUARTERLY = "Q" + YEARLY = "Y" + TYPE = [MINUTES, HOURLY, DAILY, WEEKLY, MONTHLY, QUARTERLY, YEARLY] + + +class TaskRegister: + """Registery for periodicall tasks.""" + task_list: list[ScheduledTask] = [] + + def register(self, task, schedule, minutes: int = None): + """Register a task with the que.""" + self.task_list.append(ScheduledTask(task, schedule, minutes)) + + +tasks = TaskRegister() + + +def scheduled_task(interval: str, minutes: int = None, tasklist: TaskRegister = None): + """Register the given task as a scheduled task. + + Example: + ```python + @register(ScheduledTask.DAILY) + def my_custom_funciton(): + ... + ``` + + Args: + interval (str): The interval at which the task should be run + minutes (int, optional): The number of minutes between task runs. Defaults to None. + tasklist (TaskRegister, optional): The list the tasks should be registered to. Defaults to None. + + Raises: + ValueError: If decorated object is not callable + ValueError: If interval is not valid + + Returns: + _type_: _description_ + """ + + def _task_wrapper(admin_class): + if not isinstance(admin_class, Callable): + raise ValueError('Wrapped object must be a function') + + if interval not in ScheduledTask.TYPE: + raise ValueError(f'Invalid interval. Must be one of {ScheduledTask.TYPE}') + + _tasks = tasklist if tasklist else tasks + _tasks.register(admin_class, interval, minutes=minutes) + + return admin_class + return _task_wrapper + + +@scheduled_task(ScheduledTask.MINUTES, 15) def heartbeat(): """Simple task which runs at 5 minute intervals, so we can determine that the background worker is actually running. @@ -149,6 +224,7 @@ def heartbeat(): heartbeats.delete() +@scheduled_task(ScheduledTask.DAILY) def delete_successful_tasks(): """Delete successful task logs which are more than a month old.""" try: @@ -168,6 +244,7 @@ def delete_successful_tasks(): results.delete() +@scheduled_task(ScheduledTask.DAILY) def delete_old_error_logs(): """Delete old error logs from the server.""" try: @@ -190,6 +267,7 @@ def delete_old_error_logs(): return +@scheduled_task(ScheduledTask.DAILY) def check_for_updates(): """Check if there is an update for InvenTree.""" try: @@ -232,6 +310,7 @@ def check_for_updates(): ) +@scheduled_task(ScheduledTask.DAILY) def update_exchange_rates(): """Update currency exchange rates.""" try: @@ -273,6 +352,7 @@ def update_exchange_rates(): logger.error(f"Error updating exchange rates: {e}") +@scheduled_task(ScheduledTask.DAILY) def run_backup(): """Run the backup command.""" from common.models import InvenTreeSetting diff --git a/InvenTree/build/tasks.py b/InvenTree/build/tasks.py index fcda1f8bff..6623686155 100644 --- a/InvenTree/build/tasks.py +++ b/InvenTree/build/tasks.py @@ -144,6 +144,7 @@ def notify_overdue_build_order(bo: build.models.Build): trigger_event(event_name, build_order=bo.pk) +@InvenTree.tasks.scheduled_task(InvenTree.tasks.ScheduledTask.DAILY) def check_overdue_build_orders(): """Check if any outstanding BuildOrders have just become overdue diff --git a/InvenTree/common/tasks.py b/InvenTree/common/tasks.py index 56fd3fb04f..e600136560 100644 --- a/InvenTree/common/tasks.py +++ b/InvenTree/common/tasks.py @@ -5,9 +5,12 @@ from datetime import datetime, timedelta from django.core.exceptions import AppRegistryNotReady +from InvenTree.tasks import ScheduledTask, scheduled_task + logger = logging.getLogger('inventree') +@scheduled_task(ScheduledTask.DAILY) def delete_old_notifications(): """Remove old notifications from the database. diff --git a/InvenTree/order/tasks.py b/InvenTree/order/tasks.py index 103fc3cc46..0de2b033ed 100644 --- a/InvenTree/order/tasks.py +++ b/InvenTree/order/tasks.py @@ -6,9 +6,9 @@ from django.utils.translation import gettext_lazy as _ import common.notifications import InvenTree.helpers -import InvenTree.tasks import order.models from InvenTree.status_codes import PurchaseOrderStatus, SalesOrderStatus +from InvenTree.tasks import ScheduledTask, scheduled_task from plugin.events import trigger_event @@ -55,6 +55,7 @@ def notify_overdue_purchase_order(po: order.models.PurchaseOrder): ) +@scheduled_task(ScheduledTask.DAILY) def check_overdue_purchase_orders(): """Check if any outstanding PurchaseOrders have just become overdue: @@ -117,6 +118,7 @@ def notify_overdue_sales_order(so: order.models.SalesOrder): ) +@scheduled_task(ScheduledTask.DAILY) def check_overdue_sales_orders(): """Check if any outstanding SalesOrders have just become overdue