From 914db9e913af7b019aaf9b948bffa136107e032f Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Mon, 10 May 2021 14:23:17 +1000 Subject: [PATCH 1/5] Development docker image - Uses multistage build - Adds a docker compose file for dev --- .github/workflows/docker_build.yaml | 1 + .github/workflows/docker_publish.yaml | 1 + docker/Dockerfile | 9 +++- docker/docker-compose.dev.yml | 70 +++++++++++++++++++++++++++ 4 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 docker/docker-compose.dev.yml diff --git a/.github/workflows/docker_build.yaml b/.github/workflows/docker_build.yaml index 26fc69a0f5..ec8bdf7306 100644 --- a/.github/workflows/docker_build.yaml +++ b/.github/workflows/docker_build.yaml @@ -30,6 +30,7 @@ jobs: context: ./docker platforms: linux/amd64,linux/arm64,linux/arm/v7 push: true + target: production repository: inventree/inventree tags: inventree/inventree:latest - name: Image Digest diff --git a/.github/workflows/docker_publish.yaml b/.github/workflows/docker_publish.yaml index c25696d6dd..4a8cef0952 100644 --- a/.github/workflows/docker_publish.yaml +++ b/.github/workflows/docker_publish.yaml @@ -28,4 +28,5 @@ jobs: repository: inventree/inventree tag_with_ref: true dockerfile: ./Dockerfile + target: production platforms: linux/amd64,linux/arm64,linux/arm/v7 diff --git a/docker/Dockerfile b/docker/Dockerfile index c95c2867df..504236c131 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM python:alpine as production +FROM python:alpine as base # GitHub source ARG repository="https://github.com/inventree/InvenTree.git" @@ -73,6 +73,7 @@ RUN pip install --no-cache-dir -U invoke RUN pip install --no-cache-dir -U psycopg2 mysqlclient pgcli mariadb RUN pip install --no-cache-dir -U gunicorn +FROM base as production # Clone source code RUN echo "Downloading InvenTree from ${INVENTREE_REPO}" RUN git clone --branch ${INVENTREE_BRANCH} --depth 1 ${INVENTREE_REPO} ${INVENTREE_SRC_DIR} @@ -97,3 +98,9 @@ WORKDIR ${INVENTREE_SRC_DIR} # Let us begin CMD ["bash", "./start_prod_server.sh"] + +FROM base as dev +# The development image requires the source code to be mounted to /home/inventree/src/ +# So from here, we don't actually "do" anything + +WORKDIR ${INVENTREE_SRC_DIR} diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml new file mode 100644 index 0000000000..b2e64f026a --- /dev/null +++ b/docker/docker-compose.dev.yml @@ -0,0 +1,70 @@ +version: "3.8" + +# Docker compose recipe for InvenTree development server +# - Runs sqlite3 as the database backend +# - Uses built-in django webserver + +# IMPORANT NOTE: +# The InvenTree docker image does not clone source code from git. +# Instead, you must specify *where* the source code is located, +# (on your local machine). +# The django server will auto-detect any code changes and reload the server. + +services: + # InvenTree web server services + # Uses gunicorn as the web server + inventree-server: + container_name: inventree-server + image: inventree/inventree:latest + entrypoint: ./start_dev_server.sh + ports: + - 8000 + depends_on: + - inventree-db + volumes: + # Ensure you specify the location of the 'src' directory at the end of this file + - src:/home/inventree/src + - data:/home/inventree/data + - static:/home/inventree/static + environment: + # Configure a simple sqlite server for development + # Note: You can always change to a different database backend if required! + - INVENTREE_DB_ENGINE=sqlite + - INVENTREE_DB_NAME=inventree + restart: unless-stopped + + # Background worker process handles long-running or periodic tasks + inventree-worker: + container_name: inventree-worker + image: inventree/inventree:latest + entrypoint: ./start_worker.sh + depends_on: + - inventree-db + - inventree-server + volumes: + # Ensure you specify the location of the 'src' directory at the end of this file + - data:/home/inventree/data + - static:/home/inventree/static + environment: + # Configure a simple sqlite server for development + # Note: You can always change to a different database backend if required! + - INVENTREE_DB_ENGINE=sqlite + - INVENTREE_DB_NAME=inventree + restart: unless-stopped + +volumes: + # NOTE: Change /path/to/src to a directory on your local machine, where the InvenTree source code is located + # This directory must conatin the file *manage.py* + # Persistent data, stored external to the container(s) + src: + driver: local + driver_opts: + type: none + o: bind + # This directory specified where InvenTree source code is stored "outside" the docker containers + # Note: This directory must conatin the file *manage.py* + device: /path/to/src + # Uploaded data / media files, shared between containers + data: + # Static files, shared between containers + static: \ No newline at end of file From 3381c5e25753eee420b58b888065ef2d6a5df668 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Wed, 12 May 2021 20:53:50 +1000 Subject: [PATCH 2/5] Improvements to development docker-compose script - Python packages are installed in a virtual environment within the src dir - This prevents a LONG installation process each time the docker image is rebuilt --- InvenTree/InvenTree/settings.py | 10 ++++++--- docker/Dockerfile | 8 +++++-- docker/docker-compose.dev.yml | 40 +++++++++++++++------------------ docker/start_dev_server.sh | 8 +++++++ 4 files changed, 39 insertions(+), 27 deletions(-) diff --git a/InvenTree/InvenTree/settings.py b/InvenTree/InvenTree/settings.py index cc61748372..5591678e4e 100644 --- a/InvenTree/InvenTree/settings.py +++ b/InvenTree/InvenTree/settings.py @@ -430,11 +430,15 @@ It can be specified in config.yaml (or envvar) as either (for example): - django.db.backends.postgresql """ -db_engine = db_config['ENGINE'] +db_engine = db_config['ENGINE'].lower() -if db_engine.lower() in ['sqlite3', 'postgresql', 'mysql']: +# Correct common misspelling +if db_engine == 'sqlite': + db_engine = 'sqlite3' + +if db_engine in ['sqlite3', 'postgresql', 'mysql']: # Prepend the required python module string - db_engine = f'django.db.backends.{db_engine.lower()}' + db_engine = f'django.db.backends.{db_engine}' db_config['ENGINE'] = db_engine db_name = db_config['NAME'] diff --git a/docker/Dockerfile b/docker/Dockerfile index 504236c131..61defc1607 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -86,11 +86,9 @@ COPY gunicorn.conf.py ${INVENTREE_HOME}/gunicorn.conf.py # Copy startup scripts COPY start_prod_server.sh ${INVENTREE_SRC_DIR}/start_prod_server.sh -COPY start_dev_server.sh ${INVENTREE_SRC_DIR}/start_dev_server.sh COPY start_worker.sh ${INVENTREE_SRC_DIR}/start_worker.sh RUN chmod 755 ${INVENTREE_SRC_DIR}/start_prod_server.sh -RUN chmod 755 ${INVENTREE_SRC_DIR}/start_dev_server.sh RUN chmod 755 ${INVENTREE_SRC_DIR}/start_worker.sh # exec commands should be executed from the "src" directory @@ -100,7 +98,13 @@ WORKDIR ${INVENTREE_SRC_DIR} CMD ["bash", "./start_prod_server.sh"] FROM base as dev + # The development image requires the source code to be mounted to /home/inventree/src/ # So from here, we don't actually "do" anything WORKDIR ${INVENTREE_SRC_DIR} + +COPY start_dev_server.sh ${INVENTREE_HOME}/start_dev_server.sh +RUN chmod 755 ${INVENTREE_HOME}/start_dev_server.sh + +CMD ["bash", "/home/inventree/start_dev_server.sh"] diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index b2e64f026a..bde7130c70 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -15,46 +15,46 @@ services: # Uses gunicorn as the web server inventree-server: container_name: inventree-server - image: inventree/inventree:latest - entrypoint: ./start_dev_server.sh + build: + context: . + target: dev ports: - 8000 - depends_on: - - inventree-db volumes: # Ensure you specify the location of the 'src' directory at the end of this file - src:/home/inventree/src - - data:/home/inventree/data - - static:/home/inventree/static environment: # Configure a simple sqlite server for development # Note: You can always change to a different database backend if required! - - INVENTREE_DB_ENGINE=sqlite - - INVENTREE_DB_NAME=inventree + - INVENTREE_DB_ENGINE=sqlite3 + - INVENTREE_DB_NAME=/home/inventree/data/inventree_database.sqlite3 + - INVENTREE_MEDIA_ROOT=/home/inventree/src/inventree_media + - INVENTREE_STATIC_ROOT=/home/inventree/src/inventree_static + - INVENTREE_CONFIG_FILE=/home/inventree/src/config.yaml restart: unless-stopped # Background worker process handles long-running or periodic tasks inventree-worker: container_name: inventree-worker - image: inventree/inventree:latest - entrypoint: ./start_worker.sh - depends_on: - - inventree-db + build: + context: . + target: dev + entrypoint: invoke worker - inventree-server volumes: # Ensure you specify the location of the 'src' directory at the end of this file - - data:/home/inventree/data - - static:/home/inventree/static + - src:/home/inventree/src environment: # Configure a simple sqlite server for development # Note: You can always change to a different database backend if required! - - INVENTREE_DB_ENGINE=sqlite - - INVENTREE_DB_NAME=inventree + - INVENTREE_DB_ENGINE=sqlite3 + - INVENTREE_DB_NAME=/home/inventree/data/inventree_database.sqlite3 + - INVENTREE_MEDIA_ROOT=/home/inventree/src/inventree_media + - INVENTREE_STATIC_ROOT=/home/inventree/src/inventree_static restart: unless-stopped volumes: # NOTE: Change /path/to/src to a directory on your local machine, where the InvenTree source code is located - # This directory must conatin the file *manage.py* # Persistent data, stored external to the container(s) src: driver: local @@ -63,8 +63,4 @@ volumes: o: bind # This directory specified where InvenTree source code is stored "outside" the docker containers # Note: This directory must conatin the file *manage.py* - device: /path/to/src - # Uploaded data / media files, shared between containers - data: - # Static files, shared between containers - static: \ No newline at end of file + device: /path/to/inventree/src diff --git a/docker/start_dev_server.sh b/docker/start_dev_server.sh index c22805b90b..47eb25f9ef 100644 --- a/docker/start_dev_server.sh +++ b/docker/start_dev_server.sh @@ -19,6 +19,14 @@ else cp $INVENTREE_SRC_DIR/InvenTree/config_template.yaml $INVENTREE_CONFIG_FILE fi +# Setup a virtual environment +python3 -m venv inventree-docker-dev + +source inventree-docker-dev/bin/activate + +echo "Installing required packages..." +pip install --no-cache-dir -U -r ${INVENTREE_SRC_DIR}/requirements.txt + echo "Starting InvenTree server..." # Wait for the database to be ready From 00c5a883838b5d72aff0e9ff3636a5a4099b99ee Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Wed, 12 May 2021 21:15:53 +1000 Subject: [PATCH 3/5] Split environment variables out into a .env file - No need to collect "static" file when we are running in DEBUG mode --- .gitignore | 1 + docker/dev-config.env | 7 +++++++ docker/docker-compose.dev.yml | 22 +++++++--------------- docker/start_dev_server.sh | 6 ++---- 4 files changed, 17 insertions(+), 19 deletions(-) create mode 100644 docker/dev-config.env diff --git a/.gitignore b/.gitignore index 54ad8f07b6..7c360a8231 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,7 @@ var/ *.log local_settings.py *.sqlite3 +*.sqlite3-journal *.backup *.old diff --git a/docker/dev-config.env b/docker/dev-config.env new file mode 100644 index 0000000000..200c3db479 --- /dev/null +++ b/docker/dev-config.env @@ -0,0 +1,7 @@ +INVENTREE_DB_ENGINE=sqlite3 +INVENTREE_DB_NAME=/home/inventree/src/inventree_docker_dev.sqlite3 +INVENTREE_MEDIA_ROOT=/home/inventree/src/inventree_media +INVENTREE_STATIC_ROOT=/home/inventree/src/inventree_static +INVENTREE_CONFIG_FILE=/home/inventree/src/config.yaml +INVENTREE_SECRET_KEY_FILE=/home/inventree/src/secret_key.txt +INVENTREE_DEBUG=true \ No newline at end of file diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index bde7130c70..f03ce45571 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -23,14 +23,10 @@ services: volumes: # Ensure you specify the location of the 'src' directory at the end of this file - src:/home/inventree/src - environment: - # Configure a simple sqlite server for development - # Note: You can always change to a different database backend if required! - - INVENTREE_DB_ENGINE=sqlite3 - - INVENTREE_DB_NAME=/home/inventree/data/inventree_database.sqlite3 - - INVENTREE_MEDIA_ROOT=/home/inventree/src/inventree_media - - INVENTREE_STATIC_ROOT=/home/inventree/src/inventree_static - - INVENTREE_CONFIG_FILE=/home/inventree/src/config.yaml + env_file: + # Environment variables required for the dev server are configured in dev-config.env + - dev-config.env + restart: unless-stopped # Background worker process handles long-running or periodic tasks @@ -44,13 +40,9 @@ services: volumes: # Ensure you specify the location of the 'src' directory at the end of this file - src:/home/inventree/src - environment: - # Configure a simple sqlite server for development - # Note: You can always change to a different database backend if required! - - INVENTREE_DB_ENGINE=sqlite3 - - INVENTREE_DB_NAME=/home/inventree/data/inventree_database.sqlite3 - - INVENTREE_MEDIA_ROOT=/home/inventree/src/inventree_media - - INVENTREE_STATIC_ROOT=/home/inventree/src/inventree_static + env_file: + # Environment variables required for the dev server are configured in dev-config.env + - dev-config.env restart: unless-stopped volumes: diff --git a/docker/start_dev_server.sh b/docker/start_dev_server.sh index 47eb25f9ef..8cf7d10468 100644 --- a/docker/start_dev_server.sh +++ b/docker/start_dev_server.sh @@ -35,16 +35,14 @@ python manage.py wait_for_db sleep 10 -echo "Running InvenTree database migrations and collecting static files..." +echo "Running InvenTree database migrations..." # We assume at this stage that the database is up and running # Ensure that the database schema are up to date python manage.py check || exit 1 python manage.py migrate --noinput || exit 1 python manage.py migrate --run-syncdb || exit 1 -python manage.py prerender || exit 1 -python manage.py collectstatic --noinput || exit 1 python manage.py clearsessions || exit 1 # Launch a development server -python manage.py runserver -a 0.0.0.0:$INVENTREE_WEB_PORT \ No newline at end of file +python manage.py runserver 0.0.0.0:$INVENTREE_WEB_PORT \ No newline at end of file From 530f28ba958336cdaa2e2d1b554de843126e621f Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Wed, 12 May 2021 21:35:54 +1000 Subject: [PATCH 4/5] Fix for running webserver --- docker/docker-compose.dev.yml | 2 +- docker/start_dev_server.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index f03ce45571..d6e6afb665 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -19,7 +19,7 @@ services: context: . target: dev ports: - - 8000 + - 8000:8000 volumes: # Ensure you specify the location of the 'src' directory at the end of this file - src:/home/inventree/src diff --git a/docker/start_dev_server.sh b/docker/start_dev_server.sh index 8cf7d10468..0c1564076a 100644 --- a/docker/start_dev_server.sh +++ b/docker/start_dev_server.sh @@ -45,4 +45,4 @@ python manage.py migrate --run-syncdb || exit 1 python manage.py clearsessions || exit 1 # Launch a development server -python manage.py runserver 0.0.0.0:$INVENTREE_WEB_PORT \ No newline at end of file +python manage.py runserver 0.0.0.0:$INVENTREE_WEB_PORT From 719b1775f2619e2c3e69b752cf969b4a51fb4451 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Wed, 12 May 2021 22:12:26 +1000 Subject: [PATCH 5/5] Add entrypoint for running development worker --- docker/Dockerfile | 2 ++ docker/docker-compose.dev.yml | 3 ++- docker/start_dev_worker.sh | 19 +++++++++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 docker/start_dev_worker.sh diff --git a/docker/Dockerfile b/docker/Dockerfile index 61defc1607..3e0a7e1230 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -105,6 +105,8 @@ FROM base as dev WORKDIR ${INVENTREE_SRC_DIR} COPY start_dev_server.sh ${INVENTREE_HOME}/start_dev_server.sh +COPY start_dev_worker.sh ${INVENTREE_HOME}/start_dev_worker.sh RUN chmod 755 ${INVENTREE_HOME}/start_dev_server.sh +RUN chmod 755 ${INVENTREE_HOME}/start_dev_worker.sh CMD ["bash", "/home/inventree/start_dev_server.sh"] diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index d6e6afb665..ddf50135c9 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -35,7 +35,8 @@ services: build: context: . target: dev - entrypoint: invoke worker + entrypoint: /home/inventree/start_dev_worker.sh + depends_on: - inventree-server volumes: # Ensure you specify the location of the 'src' directory at the end of this file diff --git a/docker/start_dev_worker.sh b/docker/start_dev_worker.sh new file mode 100644 index 0000000000..099f447a9c --- /dev/null +++ b/docker/start_dev_worker.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +echo "Starting InvenTree worker..." + +cd $INVENTREE_SRC_DIR + +# Activate virtual environment +source inventree-docker-dev/bin/activate + +sleep 5 + +# Wait for the database to be ready +cd $INVENTREE_MNG_DIR +python manage.py wait_for_db + +sleep 10 + +# Now we can launch the background worker process +python manage.py qcluster