From 6f939931ca75d40d08a66b3110bef2f12dd8a2a5 Mon Sep 17 00:00:00 2001 From: Oliver Date: Sat, 15 Feb 2025 08:00:12 +1100 Subject: [PATCH] Docker postgres fix (#9041) * Update docker image - Move from alpine 3.19 to alpine 3.21 - Move from postgres13_client to postgres17_client * Update docker-compose file - Move from postgres:13 to postgres:16 - Move from redis:7.0 to redis:7-alpine * Update docs * Update docker docs * Separate Dockerfile for devcontainer - Debian based (python3.11-bookworm) - Install essential system packages * Instal postgres client * Further devcontainer updates - Bump postgresql image from 13 to 15 - Store psql data in the dev/psql directory - Install required frontend packages * Use --host mode for frontend server * Tweak devcontainer docs * Bump pre commit config file * Revert "Bump pre commit config file" This reverts commit bbfd875ac8ea682bb7e454e5ea349943a887b6f4. --- .devcontainer/Dockerfile | 36 +++++++++++++++++++++ .devcontainer/docker-compose.yml | 13 ++------ .devcontainer/postCreateCommand.sh | 24 +++++++++++--- contrib/container/Dockerfile | 7 ++-- contrib/container/docker-compose.yml | 4 +-- contrib/container/install_build_packages.sh | 2 +- docs/docs/develop/devcontainer.md | 6 ++-- docs/docs/start/docker.md | 15 ++++++++- docs/mkdocs.yml | 1 + tasks.py | 2 +- 10 files changed, 82 insertions(+), 28 deletions(-) create mode 100644 .devcontainer/Dockerfile diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000000..49ed779532 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,36 @@ +# Dockerfile for the InvenTree devcontainer + +# In contrast with the "production" image (which is based on an Alpine image) +# we use a Debian-based image for the devcontainer + +FROM mcr.microsoft.com/devcontainers/python:3.11-bookworm + +# InvenTree paths +ENV INVENTREE_HOME="/home/inventree" +ENV INVENTREE_DATA_DIR="${INVENTREE_HOME}/dev" +ENV INVENTREE_STATIC_ROOT="${INVENTREE_DATA_DIR}/static" +ENV INVENTREE_MEDIA_ROOT="${INVENTREE_DATA_DIR}/media" +ENV INVENTREE_BACKUP_DIR="${INVENTREE_DATA_DIR}/backup" +ENV INVENTREE_PLUGIN_DIR="${INVENTREE_DATA_DIR}/plugins" +ENV INVENTREE_CONFIG_FILE="${INVENTREE_DATA_DIR}/config.yaml" + +COPY contrib/container/init.sh ./ +RUN chmod +x init.sh + +# Install required base packages +RUN apt update && apt install -y \ + python3.11-dev python3.11-venv \ + postgresql-client \ + libldap2-dev libsasl2-dev \ + libpango1.0-0 libcairo2 \ + weasyprint + +# Install packages required for frontend development +RUN apt install -y \ + yarn nodejs npm + +RUN yarn config set network-timeout 600000 -g + +# Install python database connectors + +ENTRYPOINT ["/bin/bash", "./init.sh"] diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 49d5201eec..659f102869 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -1,11 +1,11 @@ services: db: - image: postgres:13 + image: postgres:15 restart: unless-stopped expose: - 5432/tcp volumes: - - inventreedatabase:/var/lib/postgresql/data:z + - ../dev/psql/:/var/lib/postgresql/data:z environment: POSTGRES_DB: inventree POSTGRES_USER: inventree_user @@ -20,11 +20,7 @@ services: inventree: build: context: .. - dockerfile: ../InvenTree/contrib/container/Dockerfile - target: dev - args: - base_image: "mcr.microsoft.com/vscode/devcontainers/base:alpine-3.18" - data_dir: "dev" + dockerfile: .devcontainer/Dockerfile volumes: - ../:/home/inventree:z @@ -44,6 +40,3 @@ services: depends_on: - db - -volumes: - inventreedatabase: diff --git a/.devcontainer/postCreateCommand.sh b/.devcontainer/postCreateCommand.sh index 3da203dddf..8ae901a812 100755 --- a/.devcontainer/postCreateCommand.sh +++ b/.devcontainer/postCreateCommand.sh @@ -7,8 +7,24 @@ git config --global --add safe.directory /home/inventree python3 -m venv /home/inventree/dev/venv --system-site-packages --upgrade-deps . /home/inventree/dev/venv/bin/activate +# remove existing gitconfig created by "Avoiding Dubious Ownership" step +# so that it gets copied from host to the container to have your global +# git config in container +rm -f /home/vscode/.gitconfig + +# Fix issue related to CFFI version mismatch +pip uninstall cffi -y +sudo apt remove --purge -y python3-cffi +pip install --no-cache-dir --force-reinstall --ignore-installed cffi + +# Upgrade pip +python3 -m pip install --upgrade pip + # Ensure the correct invoke is available -pip3 install invoke --ignore-installed -U +pip3 install --ignore-installed --upgrade invoke Pillow + +# install base level packages +pip3 install -Ur contrib/container/requirements.txt # Run initial InvenTree server setup invoke update -s @@ -19,7 +35,5 @@ invoke dev.setup-dev # Install required frontend packages invoke int.frontend-install -# remove existing gitconfig created by "Avoiding Dubious Ownership" step -# so that it gets copied from host to the container to have your global -# git config in container -rm -f /home/vscode/.gitconfig +# Install playwright dependencies +cd src/frontend && sudo npx playwright install-deps diff --git a/contrib/container/Dockerfile b/contrib/container/Dockerfile index 475b6f7604..1cf4f3fe9b 100644 --- a/contrib/container/Dockerfile +++ b/contrib/container/Dockerfile @@ -9,8 +9,7 @@ # - Runs InvenTree web server under django development server # - Monitors source files for any changes, and live-reloads server -ARG base_image=python:3.11-alpine3.18 -FROM ${base_image} AS inventree_base +FROM python:3.11-alpine3.21 AS inventree_base ARG base_image # Build arguments for this image @@ -69,8 +68,8 @@ RUN apk add --no-cache \ libjpeg libwebp zlib \ # Weasyprint requirements : https://doc.courtbouillon.org/weasyprint/stable/first_steps.html#alpine-3-12 py3-pip py3-pillow py3-cffi py3-brotli pango poppler-utils openldap \ - # Postgres client - postgresql13-client \ + # Postgres client (note: backwards compatible with postgres server <= 17) + postgresql17-client \ # MySQL / MariaDB client mariadb-client mariadb-connector-c \ && \ diff --git a/contrib/container/docker-compose.yml b/contrib/container/docker-compose.yml index 9a7a00927f..e8b07cff35 100644 --- a/contrib/container/docker-compose.yml +++ b/contrib/container/docker-compose.yml @@ -36,7 +36,7 @@ services: # Database service # Use PostgreSQL as the database backend inventree-db: - image: postgres:13 + image: postgres:17 container_name: inventree-db expose: - ${INVENTREE_DB_PORT:-5432}/tcp @@ -52,7 +52,7 @@ services: # redis acts as database cache manager inventree-cache: - image: redis:7.0 + image: redis:7-alpine container_name: inventree-cache env_file: - .env diff --git a/contrib/container/install_build_packages.sh b/contrib/container/install_build_packages.sh index bbeae43a54..bef3702226 100644 --- a/contrib/container/install_build_packages.sh +++ b/contrib/container/install_build_packages.sh @@ -8,5 +8,5 @@ apk add gcc g++ musl-dev openssl-dev libffi-dev cargo python3-dev openldap-dev \ jpeg-dev openjpeg-dev libwebp-dev zlib-dev \ sqlite sqlite-dev \ mariadb-connector-c-dev mariadb-client mariadb-dev \ - postgresql13-dev postgresql-libs \ + postgresql17-dev postgresql-libs \ $@ diff --git a/docs/docs/develop/devcontainer.md b/docs/docs/develop/devcontainer.md index 90d6adb18f..8a26ab0224 100644 --- a/docs/docs/develop/devcontainer.md +++ b/docs/docs/develop/devcontainer.md @@ -111,7 +111,7 @@ Make sure you have `gnupg` and `pinentry-mac` installed and set up correctly. Re #### Where are the database, media files, ... stored? -Backups, media/static files, venv, plugin.txt, secret_key.txt, ... are stored in the `dev` folder. If you want to start with a clean setup, you can remove that folder, but be aware that this will delete everything you already setup in InvenTree. +Database data, backups, media/static files, venv, plugin.txt, secret_key.txt, ... are stored in the `dev` folder. If you want to start with a clean setup, you can remove that folder, but be aware that this will delete everything you already setup in InvenTree. ### Performance Improvements @@ -123,6 +123,4 @@ You can also refer to the [Improve disk performance guide](https://code.visualst ### Redis Caching -The devcontainer setup provides a [redis](https://redis.io/) container which can be used for managing global cache. By default this is disabled, but it can be easily enabled for testing or developing with the [redis cache](../start/config.md#caching) enabled. - -To enable the cache, locate the InvenTree configuration file (`./dev/config.yaml`) and set the `cache.enabled` setting to `True`. +The devcontainer setup provides a [redis](https://redis.io/) container which can be used for managing global cache. By default this is enabled, but it can be easily disabled by adjusting the environment variabiles in the [docker compose file]({{ sourcefile('.devcontainer/docker-compose.yml') }}). diff --git a/docs/docs/start/docker.md b/docs/docs/start/docker.md index 27863f8173..eb900e9125 100644 --- a/docs/docs/start/docker.md +++ b/docs/docs/start/docker.md @@ -46,7 +46,7 @@ InvenTree run-time configuration options described in the [configuration documen As docker containers are ephemeral, any *persistent* data must be stored in an external [volume](https://docs.docker.com/storage/volumes/). To simplify installation / implementation, all external data are stored in a single volume, arranged as follows: -#### Media FIles +#### Media Files Uploaded media files are stored in the `media/` subdirectory of the external data volume. @@ -112,6 +112,19 @@ InvenTree stores any persistent data (e.g. uploaded media files, database data, !!! info "Data Directory" Make sure you change the path to the local directory where you want persistent data to be stored. +### Database Connection + +The `inventree-db` container is configured to use the `postgres:{{ config.extra.docker_postgres_version }}` docker image. + +Connecting to a different database container is entirely possible, but requires modification of the `docker-compose.yml` file. This is outside the scope of this documentation. + +#### Postgres Version + +The `inventree-server` and `inventree-worker` containers support connection to a postgres database up to (and including) version {{ config.extra.docker_postgres_version }}. + +!!! warning "Newer Postgres Versions" + The InvenTree docker image supports connection to a postgres database up to version {{ config.extra.docker_postgres_version }}. Connecting to a database using a newer version of postgres is not possible. + ## Common Issues ### Volume Mapping diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 848dc90e76..6f97a54a10 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -285,6 +285,7 @@ extra: min_python_version: 3.9 min_invoke_version: 2.0.0 django_version: 4.2 + docker_postgres_version: 17 version: default: stable diff --git a/tasks.py b/tasks.py index 575e10e24a..ee8a86a5c6 100644 --- a/tasks.py +++ b/tasks.py @@ -1403,7 +1403,7 @@ def frontend_server(c): """ info('Starting frontend development server') yarn(c, 'yarn run compile') - yarn(c, 'yarn run dev') + yarn(c, 'yarn run dev --host') @task(