(feature): docker health checks (#12124)

* Add Docker container health checks

* Use invoke tasks for Docker container health checks

Updates server and worker healthchecks to use invoke server-health
and invoke worker-health as requested in #12124 review.

Refs #12124

* Added HTTP host check to server health check

* Add health-check endpoint for Caddyfile

* Simplify health check for Caddy

* Remove test file

---------

Co-authored-by: Oliver <oliver.henry.walters@gmail.com>
Co-authored-by: DESKTOP-691E36A\Administrator <br198064@gmail.com>
Co-authored-by: Matthias Mair <code@mjmair.com>

contrib/container/Caddyfile

@@ -30,6 +30,18 @@
 	}
 }
 
+# Internal-only health check endpoint, on port 9090
+# This only serves to check that the Caddy container is running correctly
+:9090 {
+	handle /api/system/health/* {
+		reverse_proxy {$INVENTREE_SERVER:"http://inventree-server:8000"}
+	}
+
+	handle {
+		respond 404
+	}
+}
+
 # The default server address is configured in the .env file
 # If not specified, the proxy listens for all http/https traffic
 # If you need to listen on multiple addresses, or use a different port, you can modify this section directly

contrib/container/docker-compose.yml

@@ -54,6 +54,12 @@ services:
         volumes:
             # Map 'data' volume such that postgres database is stored externally
             - ${INVENTREE_EXT_VOLUME:?You must specify the 'INVENTREE_EXT_VOLUME' variable in the .env file!}:/var/lib/postgresql/data/:z
+        healthcheck:
+            test: ['CMD-SHELL', 'pg_isready -U "$${POSTGRES_USER}" -d "$${POSTGRES_DB}"']
+            interval: 15s
+            timeout: 5s
+            retries: 10
+            start_period: 20s
         restart: unless-stopped
 
     # redis acts as database cache manager
@@ -69,6 +75,12 @@ services:
             - ${INVENTREE_CACHE_PORT:-6379}
         volumes:
             - ${INVENTREE_EXT_VOLUME}/redis:/data
+        healthcheck:
+            test: ['CMD', 'redis-cli', 'ping']
+            interval: 15s
+            timeout: 5s
+            retries: 10
+            start_period: 10s
         restart: always
 
     # InvenTree web server service
@@ -81,8 +93,10 @@ services:
         expose:
             - ${INVENTREE_WEB_PORT:-8000}
         depends_on:
-            - inventree-db
+            inventree-db:
-            - inventree-cache
+                condition: service_healthy
+            inventree-cache:
+                condition: service_healthy
         env_file:
             - .env
         environment:
@@ -90,6 +104,16 @@ services:
         volumes:
             # Data volume must map to /home/inventree/data
             - ${INVENTREE_EXT_VOLUME}:/home/inventree/data:z
+        healthcheck:
+            test:
+                [
+                    'CMD-SHELL',
+                    'invoke server-health --address "http://localhost:$${INVENTREE_WEB_PORT:-8000}"',
+                ]
+            interval: 20s
+            timeout: 5s
+            retries: 10
+            start_period: 60s
         restart: unless-stopped
 
     # Background worker process handles long-running or periodic tasks
@@ -99,12 +123,19 @@ services:
         container_name: inventree-worker
         command: invoke worker
         depends_on:
-            - inventree-server
+            inventree-server:
+                condition: service_healthy
         env_file:
             - .env
         volumes:
             # Data volume must map to /home/inventree/data
             - ${INVENTREE_EXT_VOLUME}:/home/inventree/data:z
+        healthcheck:
+            test: ['CMD-SHELL', 'invoke worker-health']
+            interval: 60s
+            timeout: 10s
+            retries: 3
+            start_period: 180s
         restart: unless-stopped
 
     # caddy acts as reverse proxy and static file server
@@ -115,12 +146,21 @@ services:
         image: caddy:alpine
         restart: always
         depends_on:
-            - inventree-server
+            inventree-server:
+                condition: service_healthy
+            inventree-worker:
+                condition: service_healthy
         ports:
             - ${INVENTREE_HTTP_PORT:-80}:80
             - ${INVENTREE_HTTPS_PORT:-443}:443
         env_file:
             - .env
+        healthcheck:
+            test: ['CMD', 'wget', '--spider', '-q', 'http://127.0.0.1:9090/api/system/health/']
+            interval: 20s
+            timeout: 5s
+            retries: 10
+            start_period: 20s
         volumes:
             - ./Caddyfile:/etc/caddy/Caddyfile:ro,z
             - ${INVENTREE_EXT_VOLUME}/static:/var/www/static:z

tasks.py

@@ -1666,15 +1666,33 @@ def server_health(c, address: str = 'http://localhost:8000', timeout: int = 5):
     """Check if the web server is healthy by requesting /api/system/health/.
 
     Exits 0 on HTTP 200, 1 otherwise.
-    No Django startup required.
+    Django startup only required when when INVENTREE_SITE_URL is not set
+    and no docker/devcontainer/pkg-installer env vars are set. Django exceptions
+    caught and logged as warnings, but do not cause the health check to fail.
     """
     import urllib.error
+    import urllib.parse
     import urllib.request
 
+    from src.backend.InvenTree.InvenTree.config import (  # type: ignore[import]
+        get_setting,
+    )
+
     url = f'{address.rstrip("/")}/api/system/health/'
+    site_url = None
 
     try:
-        with urllib.request.urlopen(url, timeout=timeout) as response:
+        site_url = get_setting('INVENTREE_SITE_URL', 'site_url', None)
+    except (Exception, SystemExit) as exc:
+        warning(f'Could not determine configured site URL: {exc}')
+
+    request = urllib.request.Request(url)
+
+    if site_url and (hostname := urllib.parse.urlparse(site_url).hostname):
+        request.add_header('Host', hostname)
+
+    try:
+        with urllib.request.urlopen(request, timeout=timeout) as response:
             if response.status == 200:
                 success(f'Server is healthy ({url})')
                 return