System Health Checks (#12193)

* Add worker health check invoke task

* Increase frequency of heartbeat task

* Adjust default threshold for worker health check

* Add server_health invoke func

src/backend/InvenTree/InvenTree/apps.py

@@ -188,13 +188,10 @@ class InvenTreeConfig(AppConfig):
     @ignore_ready_warning
     def add_heartbeat(self):
         """Ensure there is at least one background task in the queue."""
-        import django_q.models
-
         try:
-            if django_q.models.OrmQ.objects.count() == 0:
-                InvenTree.tasks.offload_task(
-                    InvenTree.tasks.heartbeat, force_async=True, group='heartbeat'
-                )
+            InvenTree.tasks.offload_task(
+                InvenTree.tasks.heartbeat, force_async=True, group='heartbeat'
+            )
         except AppRegistryNotReady:  # pragma: no cover
             pass
         except Exception:

src/backend/InvenTree/InvenTree/tasks.py

@@ -438,21 +438,30 @@ def scheduled_task(
 
 
 @tracer.start_as_current_span('heartbeat')
-@scheduled_task(ScheduledTask.MINUTES, 5)
+@scheduled_task(ScheduledTask.MINUTES, 1)
 def heartbeat():
-    """Simple task which runs at 5 minute intervals, so we can determine that the background worker is actually running.
-
-    (There is probably a less "hacky" way of achieving this)?
-    """
+    """Simple task which runs at 1 minute intervals, so we can determine that the background worker is actually running."""
     try:
         from django_q.models import OrmQ, Success
     except AppRegistryNotReady:  # pragma: no cover
         logger.info('Could not perform heartbeat task - App registry not ready')
         return
 
-    threshold = timezone.now() - timedelta(minutes=30)
+    # Write a timestamp file so that health checks can verify worker liveness
+    # without needing to start a full Django process.
+    import tempfile
+    from pathlib import Path
 
-    # Delete heartbeat results more than half an hour old,
+    try:
+        Path(tempfile.gettempdir()).joinpath('inventree_worker_heartbeat').write_text(
+            str(timezone.now().timestamp())
+        )
+    except Exception:
+        pass
+
+    threshold = timezone.now() - timedelta(minutes=15)
+
+    # Delete heartbeat results more than 15 minutes old,
     # otherwise they just create extra noise
     heartbeats = Success.objects.filter(
         func='InvenTree.tasks.heartbeat', started__lte=threshold