From 8b66babd492b37bf26f5bf494d64a5fada8a0212 Mon Sep 17 00:00:00 2001 From: Oliver Date: Tue, 17 Aug 2021 22:58:44 +1000 Subject: [PATCH 01/17] Refactor dockerfile - Ref: https://github.com/inventree/InvenTree/pull/1949 - Squash all apk commands into single line - Drop to inventree user rather than running as root - Separate entrypoint and cmd for each target - Set the INVENTREE_PY_ENV variable in development mode --- docker/Dockerfile | 69 +++++++++++++++++++++++++---------------------- 1 file changed, 37 insertions(+), 32 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 1266a282e4..3d86d73eca 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -59,28 +59,27 @@ RUN apk add --no-cache git make bash \ gcc libgcc g++ libstdc++ \ libjpeg-turbo libjpeg-turbo-dev jpeg jpeg-dev \ libffi libffi-dev \ - zlib zlib-dev - -# Cairo deps for WeasyPrint (these will be deprecated once WeasyPrint drops cairo requirement) -RUN apk add --no-cache cairo cairo-dev pango pango-dev -RUN apk add --no-cache fontconfig ttf-droid ttf-liberation ttf-dejavu ttf-opensans ttf-ubuntu-font-family font-croscore font-noto - -# Python -RUN apk add --no-cache python3 python3-dev py3-pip - -# SQLite support -RUN apk add --no-cache sqlite - -# PostgreSQL support -RUN apk add --no-cache postgresql postgresql-contrib postgresql-dev libpq - -# MySQL support -RUN apk add --no-cache mariadb-connector-c mariadb-dev mariadb-client + zlib zlib-dev \ + # Cairo deps for WeasyPrint (these will be deprecated once WeasyPrint drops cairo requirement) + cairo cairo-dev pango pango-dev \ + # Fonts + fontconfig ttf-droid ttf-liberation ttf-dejavu ttf-opensans ttf-ubuntu-font-family font-croscore font-noto \ + # Core python + python3 python3-dev py3-pip \ + # SQLite support + sqlite \ + # PostgreSQL support + postgresql postgresql-contrib postgresql-dev libpq \ + # MySQL/MariaDB support + mariadb-connector-c mariadb-dev mariadb-client # Install required python packages RUN pip install --no-cache-dir -U psycopg2 mysqlclient pgcli mariadb +ENTRYPOINT ["/sbin/tini", "--"] + FROM base as production + # Clone source code RUN echo "Downloading InvenTree from ${INVENTREE_GIT_REPO}" @@ -89,23 +88,22 @@ RUN git clone --branch ${INVENTREE_GIT_BRANCH} --depth 1 ${INVENTREE_GIT_REPO} $ # Checkout against a particular git tag RUN if [ -n "${INVENTREE_GIT_TAG}" ] ; then cd ${INVENTREE_HOME} && git fetch --all --tags && git checkout tags/${INVENTREE_GIT_TAG} -b v${INVENTREE_GIT_TAG}-branch ; fi -# Install InvenTree packages -RUN pip install --no-cache-dir -U -r ${INVENTREE_HOME}/requirements.txt - -# Copy gunicorn config file -COPY gunicorn.conf.py ${INVENTREE_HOME}/gunicorn.conf.py - -# Copy startup scripts -COPY start_prod_server.sh ${INVENTREE_HOME}/start_prod_server.sh -COPY start_prod_worker.sh ${INVENTREE_HOME}/start_prod_worker.sh - -RUN chmod 755 ${INVENTREE_HOME}/start_prod_server.sh -RUN chmod 755 ${INVENTREE_HOME}/start_prod_worker.sh +RUN chown -R inventree:inventreegroup ${INVENTREE_HOME}/* WORKDIR ${INVENTREE_HOME} -# Let us begin -CMD ["bash", "./start_prod_server.sh"] +# Drop to the inventree user +USER inventree + +# Install InvenTree packages +RUN pip3 install --no-cache-dir --disable-pip-version-check pip==21.2.3 setuptools==57.4.0 wheel>=0.37.0 \ + && pip3 install --no-cache-dir --disable-pip-version-check -r ${INVENTREE_HOME}/requirements.txt + +# Server init entrypoint +ENTRYPOINT ./docker/init-server.sh + +# Launch the production server +CMD ["gunicorn -c ./docker/gunicorn.conf.py -b ${INVENTREE_WEB_ADDR}:${INVENTREE_WEB_PORT} InvenTree.wsgi"] FROM base as dev @@ -114,6 +112,10 @@ FROM base as dev ENV INVENTREE_DEV_DIR="${INVENTREE_HOME}/dev" +# Location for python virtual environment +# If the INVENTREE_PY_ENV variable is set, the entrypoint script will use it! +ENV INVENTREE_PY_ENV="${INVENTREE_HOME}/dev/env" + # Override default path settings ENV INVENTREE_STATIC_ROOT="${INVENTREE_DEV_DIR}/static" ENV INVENTREE_MEDIA_ROOT="${INVENTREE_DEV_DIR}/media" @@ -122,5 +124,8 @@ ENV INVENTREE_SECRET_KEY_FILE="${INVENTREE_DEV_DIR}/secret_key.txt" WORKDIR ${INVENTREE_HOME} +# Entrypoint +ENTRYPOINT ./docker/init-server.sh + # Launch the development server -CMD ["bash", "/home/inventree/docker/start_dev_server.sh"] +CMD ["python3 manage.py runserver ${INVENTREE_WEB_ADDR}:${INVENTREE_WEB_PORT}"] From 187c9b09716d9bd7f50bb1437322874fb66968f7 Mon Sep 17 00:00:00 2001 From: Oliver Date: Tue, 17 Aug 2021 23:10:57 +1000 Subject: [PATCH 02/17] Add server init script - Taken (mostly) from https://github.com/inventree/InvenTree/pull/1949 --- docker/Dockerfile | 2 +- docker/init-server.sh | 69 +++++++++++++++++++++++++++++++++++++++++++ tasks.py | 2 +- 3 files changed, 71 insertions(+), 2 deletions(-) create mode 100644 docker/init-server.sh diff --git a/docker/Dockerfile b/docker/Dockerfile index 3d86d73eca..26d57e4e47 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -114,7 +114,7 @@ ENV INVENTREE_DEV_DIR="${INVENTREE_HOME}/dev" # Location for python virtual environment # If the INVENTREE_PY_ENV variable is set, the entrypoint script will use it! -ENV INVENTREE_PY_ENV="${INVENTREE_HOME}/dev/env" +ENV INVENTREE_PY_ENV="${INVENTREE_DEV}/env" # Override default path settings ENV INVENTREE_STATIC_ROOT="${INVENTREE_DEV_DIR}/static" diff --git a/docker/init-server.sh b/docker/init-server.sh new file mode 100644 index 0000000000..48603815fc --- /dev/null +++ b/docker/init-server.sh @@ -0,0 +1,69 @@ +#!/bin/sh +# exit when any command fails +set -e + +# Create required directory structure (if it does not already exist) +if [[ ! -d "$INVENTREE_STATIC_ROOT" ]]; then + echo "Creating directory $INVENTREE_STATIC_ROOT" + mkdir -p $INVENTREE_STATIC_ROOT +fi + +if [[ ! -d "$INVENTREE_MEDIA_ROOT" ]]; then + echo "Creating directory $INVENTREE_MEDIA_ROOT" + mkdir -p $INVENTREE_MEDIA_ROOT +fi + +# Check if "config.yaml" has been copied into the correct location +if test -f "$INVENTREE_CONFIG_FILE"; then + echo "$INVENTREE_CONFIG_FILE exists - skipping" +else + echo "Copying config file to $INVENTREE_CONFIG_FILE" + cp $INVENTREE_HOME/InvenTree/config_template.yaml $INVENTREE_CONFIG_FILE +fi + +# Setup a python virtual environment +# This should be done on the *mounted* filesystem, +# so that the installed modules persist! +if [[ -n "$INVENTREE_PY_ENV" ]]; then + echo "Using Python virtual environment: ${INVENTREE_PY_ENV}" + # Setup a virtual environment (within the "dev" directory) + python3 -m venv ${INVENTREE_PY_ENV} + + # Activate the virtual environment + source ${INVENTREE_PY_ENV}/bin/activate + + echo "Installing required packages..." + pip install --no-cache-dir -U -r ${INVENTREE_HOME}/requirements.txt +fi + +# Wait for the InvenTree database to be ready +cd ${INVENTREE_MNG_DIR} +echo "InvenTree: Waiting for database connection" +python3 manage.py wait_for_db && echo "InvenTree: db found, sleeping 10" || { echo "InvenTree: Failed to connect to db due to errors, aborting"; exit 1; } +sleep 10 + +# Check database migrations +cd ${INVENTREE_HOME} + +# We assume at this stage that the database is up and running +# Ensure that the database schema are up to date +echo "InvenTree: Checking database..." +invoke check || exit 1 +echo "InvenTree: Check successful" +echo "InvenTree: Database Migrations..." +invoke migrate || exit 1 +echo "InvenTree: Migrations successful" +echo "InvenTree: Collecting static files..." +# Note: "translate" calls "static" also +invoke translate || exit 1 +echo "InvenTree: static successful" + +# Can be run as a cron job or directly to clean out expired sessions. +cd ${INVENTREE_MNG_DIR} +python3 manage.py clearsessions || exit 1 +echo "InvenTree: migrations complete" + +#Launch the CMD +#echo "init-server launching $@" +#exec "$@" +#echo "init-server exiting" diff --git a/tasks.py b/tasks.py index b3aaab2f92..837207f1ec 100644 --- a/tasks.py +++ b/tasks.py @@ -156,7 +156,7 @@ def migrate(c): print("========================================") manage(c, "makemigrations") - manage(c, "migrate") + manage(c, "migrate --noinput") manage(c, "migrate --run-syncdb") manage(c, "check") From b48db6f8fe84d783f70c1ffaf118819c83245d9b Mon Sep 17 00:00:00 2001 From: Oliver Date: Tue, 17 Aug 2021 23:15:05 +1000 Subject: [PATCH 03/17] Dockerfile fixes --- docker/Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 26d57e4e47..5fb7329498 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -96,8 +96,7 @@ WORKDIR ${INVENTREE_HOME} USER inventree # Install InvenTree packages -RUN pip3 install --no-cache-dir --disable-pip-version-check pip==21.2.3 setuptools==57.4.0 wheel>=0.37.0 \ - && pip3 install --no-cache-dir --disable-pip-version-check -r ${INVENTREE_HOME}/requirements.txt +RUN pip3 install --no-cache-dir --disable-pip-version-check -r ${INVENTREE_HOME}/requirements.txt # Server init entrypoint ENTRYPOINT ./docker/init-server.sh @@ -114,7 +113,7 @@ ENV INVENTREE_DEV_DIR="${INVENTREE_HOME}/dev" # Location for python virtual environment # If the INVENTREE_PY_ENV variable is set, the entrypoint script will use it! -ENV INVENTREE_PY_ENV="${INVENTREE_DEV}/env" +ENV INVENTREE_PY_ENV="${INVENTREE_DEV_DIR}/env" # Override default path settings ENV INVENTREE_STATIC_ROOT="${INVENTREE_DEV_DIR}/static" From da834d8bcc37e4f67bd039748935b2f52e48ce0a Mon Sep 17 00:00:00 2001 From: Oliver Date: Wed, 18 Aug 2021 00:04:38 +1000 Subject: [PATCH 04/17] Reduce cruft in logs --- InvenTree/InvenTree/settings.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/InvenTree/InvenTree/settings.py b/InvenTree/InvenTree/settings.py index a0a9ca510e..f3c166df88 100644 --- a/InvenTree/InvenTree/settings.py +++ b/InvenTree/InvenTree/settings.py @@ -235,8 +235,8 @@ MEDIA_URL = '/media/' if DEBUG: logger.info("InvenTree running in DEBUG mode") -logger.info(f"MEDIA_ROOT: '{MEDIA_ROOT}'") -logger.info(f"STATIC_ROOT: '{STATIC_ROOT}'") +logger.debug(f"MEDIA_ROOT: '{MEDIA_ROOT}'") +logger.debug(f"STATIC_ROOT: '{STATIC_ROOT}'") # Application definition @@ -420,7 +420,7 @@ Configure the database backend based on the user-specified values. - The following code lets the user "mix and match" database configuration """ -logger.info("Configuring database backend:") +logger.debug("Configuring database backend:") # Extract database configuration from the config.yaml file db_config = CONFIG.get('database', {}) @@ -474,11 +474,9 @@ if db_engine in ['sqlite3', 'postgresql', 'mysql']: db_name = db_config['NAME'] db_host = db_config.get('HOST', "''") -print("InvenTree Database Configuration") -print("================================") -print(f"ENGINE: {db_engine}") -print(f"NAME: {db_name}") -print(f"HOST: {db_host}") +logger.info(f"DB_ENGINE: {db_engine}") +logger.info(f"DB_NAME: {db_name}") +logger.info(f"DB_HOST: {db_host}") DATABASES['default'] = db_config From 3b8ee485813c67550b63d658bce164f748ed6abd Mon Sep 17 00:00:00 2001 From: Oliver Date: Wed, 18 Aug 2021 09:34:09 +1000 Subject: [PATCH 05/17] Fix env defines in dockerfile --- docker/Dockerfile | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 5fb7329498..d68236b91f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -36,18 +36,15 @@ ENV INVENTREE_GUNICORN_WORKERS="4" ENV INVENTREE_BACKGROUND_WORKERS="4" # Default web server address:port -ENV INVENTREE_WEB_ADDR="0.0.0.0" -ENV INVENTREE_WEB_PORT="8000" +ENV INVENTREE_WEB_ADDR=0.0.0.0 +ENV INVENTREE_WEB_PORT=8000 LABEL org.label-schema.schema-version="1.0" \ org.label-schema.build-date=${DATE} \ org.label-schema.vendor="inventree" \ org.label-schema.name="inventree/inventree" \ org.label-schema.url="https://hub.docker.com/r/inventree/inventree" \ - org.label-schema.version=${INVENTREE_VERSION} \ - org.label-schema.vcs-url=${INVENTREE_REPO} \ - org.label-schema.vcs-branch=${BRANCH} \ - org.label-schema.vcs-ref=${COMMIT} + org.label-schema.vcs-url=${INVENTREE_REPO} # Create user account RUN addgroup -S inventreegroup && adduser -S inventree -G inventreegroup From 7bfddd6d51f1b086a0eef8520ed0fd32fd60cdff Mon Sep 17 00:00:00 2001 From: Oliver Date: Wed, 18 Aug 2021 09:52:27 +1000 Subject: [PATCH 06/17] Simplify init scripts Single script init.sh which performs the following tasks: - Creates required directory structure - Activates python venv (if required) - Waits for database connection - Runs command --- docker/Dockerfile | 11 +++---- docker/docker-compose.dev.yml | 2 +- docker/{init-server.sh => init.sh} | 37 +++++++--------------- docker/start_dev_server.sh | 51 ------------------------------ docker/start_dev_worker.sh | 19 ----------- docker/start_prod_server.sh | 42 ------------------------ docker/start_prod_worker.sh | 14 -------- tasks.py | 34 ++++++++++---------- 8 files changed, 34 insertions(+), 176 deletions(-) rename docker/{init-server.sh => init.sh} (57%) delete mode 100644 docker/start_dev_server.sh delete mode 100644 docker/start_dev_worker.sh delete mode 100644 docker/start_prod_server.sh delete mode 100644 docker/start_prod_worker.sh diff --git a/docker/Dockerfile b/docker/Dockerfile index d68236b91f..5a9059ec7f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -73,8 +73,6 @@ RUN apk add --no-cache git make bash \ # Install required python packages RUN pip install --no-cache-dir -U psycopg2 mysqlclient pgcli mariadb -ENTRYPOINT ["/sbin/tini", "--"] - FROM base as production # Clone source code @@ -96,7 +94,7 @@ USER inventree RUN pip3 install --no-cache-dir --disable-pip-version-check -r ${INVENTREE_HOME}/requirements.txt # Server init entrypoint -ENTRYPOINT ./docker/init-server.sh +ENTRYPOINT ["/bin/bash", "./docker/init.sh"] # Launch the production server CMD ["gunicorn -c ./docker/gunicorn.conf.py -b ${INVENTREE_WEB_ADDR}:${INVENTREE_WEB_PORT} InvenTree.wsgi"] @@ -120,8 +118,9 @@ ENV INVENTREE_SECRET_KEY_FILE="${INVENTREE_DEV_DIR}/secret_key.txt" WORKDIR ${INVENTREE_HOME} -# Entrypoint -ENTRYPOINT ./docker/init-server.sh +# Entrypoint ensures that we are running in the python virtual environment +ENTRYPOINT ["/bin/bash", "./docker/init.sh"] # Launch the development server -CMD ["python3 manage.py runserver ${INVENTREE_WEB_ADDR}:${INVENTREE_WEB_PORT}"] +CMD ["invoke", "server", "-a", "${INVENTREE_WEB_ADDR}:${INVENTREE_WEB_PORT}"] + diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index ba8ed774ee..c4be092189 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -35,7 +35,7 @@ services: build: context: . target: dev - entrypoint: /home/inventree/docker/start_dev_worker.sh + command: invoke worker depends_on: - inventree-dev-server volumes: diff --git a/docker/init-server.sh b/docker/init.sh similarity index 57% rename from docker/init-server.sh rename to docker/init.sh index 48603815fc..9be8ffe44b 100644 --- a/docker/init-server.sh +++ b/docker/init.sh @@ -32,38 +32,25 @@ if [[ -n "$INVENTREE_PY_ENV" ]]; then # Activate the virtual environment source ${INVENTREE_PY_ENV}/bin/activate - echo "Installing required packages..." - pip install --no-cache-dir -U -r ${INVENTREE_HOME}/requirements.txt + # Note: Python packages will have to be installed on first run + # e.g docker-compose -f docker-compose.dev.yml run inventree-dev-server invoke install fi # Wait for the InvenTree database to be ready -cd ${INVENTREE_MNG_DIR} -echo "InvenTree: Waiting for database connection" -python3 manage.py wait_for_db && echo "InvenTree: db found, sleeping 10" || { echo "InvenTree: Failed to connect to db due to errors, aborting"; exit 1; } -sleep 10 +# cd ${INVENTREE_MNG_DIR} +# echo "InvenTree: Waiting for database connection" +# invoke wait && echo "InvenTree: Database connection successful" || { echo "InvenTree: Failed to connect to db due to errors, aborting"; exit 1; } +# sleep 5 -# Check database migrations cd ${INVENTREE_HOME} # We assume at this stage that the database is up and running -# Ensure that the database schema are up to date -echo "InvenTree: Checking database..." -invoke check || exit 1 -echo "InvenTree: Check successful" -echo "InvenTree: Database Migrations..." -invoke migrate || exit 1 -echo "InvenTree: Migrations successful" -echo "InvenTree: Collecting static files..." -# Note: "translate" calls "static" also -invoke translate || exit 1 -echo "InvenTree: static successful" +# echo "InvenTree: Checking database..." +# invoke check || exit 1 # Can be run as a cron job or directly to clean out expired sessions. -cd ${INVENTREE_MNG_DIR} -python3 manage.py clearsessions || exit 1 -echo "InvenTree: migrations complete" +# cd ${INVENTREE_MNG_DIR} +# python3 manage.py clearsessions || exit 1 -#Launch the CMD -#echo "init-server launching $@" -#exec "$@" -#echo "init-server exiting" +# Launch the CMD *after* the ENTRYPOINT completes +exec "$@" diff --git a/docker/start_dev_server.sh b/docker/start_dev_server.sh deleted file mode 100644 index a12a958a9a..0000000000 --- a/docker/start_dev_server.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/sh - -# Create required directory structure (if it does not already exist) -if [[ ! -d "$INVENTREE_STATIC_ROOT" ]]; then - echo "Creating directory $INVENTREE_STATIC_ROOT" - mkdir -p $INVENTREE_STATIC_ROOT -fi - -if [[ ! -d "$INVENTREE_MEDIA_ROOT" ]]; then - echo "Creating directory $INVENTREE_MEDIA_ROOT" - mkdir -p $INVENTREE_MEDIA_ROOT -fi - -# Check if "config.yaml" has been copied into the correct location -if test -f "$INVENTREE_CONFIG_FILE"; then - echo "$INVENTREE_CONFIG_FILE exists - skipping" -else - echo "Copying config file to $INVENTREE_CONFIG_FILE" - cp $INVENTREE_HOME/InvenTree/config_template.yaml $INVENTREE_CONFIG_FILE -fi - -# Setup a virtual environment (within the "dev" directory) -python3 -m venv ./dev/env - -# Activate the virtual environment -source ./dev/env/bin/activate - -echo "Installing required packages..." -pip install --no-cache-dir -U -r ${INVENTREE_HOME}/requirements.txt - -echo "Starting InvenTree server..." - -# Wait for the database to be ready -cd ${INVENTREE_HOME}/InvenTree -python3 manage.py wait_for_db - -sleep 10 - -echo "Running InvenTree database migrations..." - -# We assume at this stage that the database is up and running -# Ensure that the database schema are up to date -python3 manage.py check || exit 1 -python3 manage.py migrate --noinput || exit 1 -python3 manage.py migrate --run-syncdb || exit 1 -python3 manage.py clearsessions || exit 1 - -invoke static - -# Launch a development server -python3 manage.py runserver ${INVENTREE_WEB_ADDR}:${INVENTREE_WEB_PORT} diff --git a/docker/start_dev_worker.sh b/docker/start_dev_worker.sh deleted file mode 100644 index 7ee59ff28f..0000000000 --- a/docker/start_dev_worker.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/sh - -echo "Starting InvenTree worker..." - -cd $INVENTREE_HOME - -# Activate virtual environment -source ./dev/env/bin/activate - -sleep 5 - -# Wait for the database to be ready -cd InvenTree -python3 manage.py wait_for_db - -sleep 10 - -# Now we can launch the background worker process -python3 manage.py qcluster diff --git a/docker/start_prod_server.sh b/docker/start_prod_server.sh deleted file mode 100644 index 1660a64e60..0000000000 --- a/docker/start_prod_server.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/sh - -# Create required directory structure (if it does not already exist) -if [[ ! -d "$INVENTREE_STATIC_ROOT" ]]; then - echo "Creating directory $INVENTREE_STATIC_ROOT" - mkdir -p $INVENTREE_STATIC_ROOT -fi - -if [[ ! -d "$INVENTREE_MEDIA_ROOT" ]]; then - echo "Creating directory $INVENTREE_MEDIA_ROOT" - mkdir -p $INVENTREE_MEDIA_ROOT -fi - -# Check if "config.yaml" has been copied into the correct location -if test -f "$INVENTREE_CONFIG_FILE"; then - echo "$INVENTREE_CONFIG_FILE exists - skipping" -else - echo "Copying config file to $INVENTREE_CONFIG_FILE" - cp $INVENTREE_HOME/InvenTree/config_template.yaml $INVENTREE_CONFIG_FILE -fi - -echo "Starting InvenTree server..." - -# Wait for the database to be ready -cd $INVENTREE_MNG_DIR -python3 manage.py wait_for_db - -sleep 10 - -echo "Running InvenTree database migrations and collecting static files..." - -# We assume at this stage that the database is up and running -# Ensure that the database schema are up to date -python3 manage.py check || exit 1 -python3 manage.py migrate --noinput || exit 1 -python3 manage.py migrate --run-syncdb || exit 1 -python3 manage.py prerender || exit 1 -python3 manage.py collectstatic --noinput || exit 1 -python3 manage.py clearsessions || exit 1 - -# Now we can launch the server -gunicorn -c $INVENTREE_HOME/gunicorn.conf.py InvenTree.wsgi -b 0.0.0.0:$INVENTREE_WEB_PORT diff --git a/docker/start_prod_worker.sh b/docker/start_prod_worker.sh deleted file mode 100644 index d0762b430e..0000000000 --- a/docker/start_prod_worker.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh - -echo "Starting InvenTree worker..." - -sleep 5 - -# Wait for the database to be ready -cd $INVENTREE_MNG_DIR -python3 manage.py wait_for_db - -sleep 10 - -# Now we can launch the background worker process -python3 manage.py qcluster diff --git a/tasks.py b/tasks.py index 837207f1ec..9088efb12f 100644 --- a/tasks.py +++ b/tasks.py @@ -65,7 +65,7 @@ def manage(c, cmd, pty=False): cmd - django command to run """ - c.run('cd "{path}" && python3 manage.py {cmd}'.format( + result = c.run('cd "{path}" && python3 manage.py {cmd}'.format( path=managePyDir(), cmd=cmd ), pty=pty) @@ -80,14 +80,6 @@ def install(c): # Install required Python packages with PIP c.run('pip3 install -U -r requirements.txt') - # If a config.yaml file does not exist, copy from the template! - CONFIG_FILE = os.path.join(localDir(), 'InvenTree', 'config.yaml') - CONFIG_TEMPLATE_FILE = os.path.join(localDir(), 'InvenTree', 'config_template.yaml') - - if not os.path.exists(CONFIG_FILE): - print("Config file 'config.yaml' does not exist - copying from template.") - copyfile(CONFIG_TEMPLATE_FILE, CONFIG_FILE) - @task def shell(c): @@ -97,13 +89,6 @@ def shell(c): manage(c, 'shell', pty=True) -@task -def worker(c): - """ - Run the InvenTree background worker process - """ - - manage(c, 'qcluster', pty=True) @task def superuser(c): @@ -113,6 +98,7 @@ def superuser(c): manage(c, 'createsuperuser', pty=True) + @task def check(c): """ @@ -121,13 +107,24 @@ def check(c): manage(c, "check") + @task def wait(c): """ Wait until the database connection is ready """ - manage(c, "wait_for_db") + return manage(c, "wait_for_db") + + +@task(pre=[wait]) +def worker(c): + """ + Run the InvenTree background worker process + """ + + manage(c, 'qcluster', pty=True) + @task def rebuild(c): @@ -137,6 +134,7 @@ def rebuild(c): manage(c, "rebuild_models") + @task def clean_settings(c): """ @@ -145,7 +143,7 @@ def clean_settings(c): manage(c, "clean_settings") -@task +@task(post=[rebuild]) def migrate(c): """ Performs database migrations. From 8fea9bc645cef828443c707daf034909863af1c0 Mon Sep 17 00:00:00 2001 From: Oliver Date: Wed, 18 Aug 2021 11:25:19 +1000 Subject: [PATCH 07/17] Re-add docker file git version info --- docker/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 5a9059ec7f..b6d59de206 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -44,7 +44,9 @@ LABEL org.label-schema.schema-version="1.0" \ org.label-schema.vendor="inventree" \ org.label-schema.name="inventree/inventree" \ org.label-schema.url="https://hub.docker.com/r/inventree/inventree" \ - org.label-schema.vcs-url=${INVENTREE_REPO} + org.label-schema.vcs-url=${INVENTREE_GIT_REPO} \ + org.label-schema.vcs-branch=${INVENTREE_GIT_BRANCH} \ + org.label-schema.vss-ref=${INVENTREE_GIT_TAG} # Create user account RUN addgroup -S inventreegroup && adduser -S inventree -G inventreegroup From c2af4018546ce7eb5167c993bd7c7d27c262a265 Mon Sep 17 00:00:00 2001 From: Oliver Date: Wed, 18 Aug 2021 12:03:24 +1000 Subject: [PATCH 08/17] Pin base python package requirements - Require invoke to be installed before we can run "invoke update" --- docker/Dockerfile | 6 +++--- docker/requirements.txt | 13 +++++++++++++ requirements.txt | 8 +------- 3 files changed, 17 insertions(+), 10 deletions(-) create mode 100644 docker/requirements.txt diff --git a/docker/Dockerfile b/docker/Dockerfile index b6d59de206..9cfe88b2b5 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -72,9 +72,9 @@ RUN apk add --no-cache git make bash \ # MySQL/MariaDB support mariadb-connector-c mariadb-dev mariadb-client -# Install required python packages -RUN pip install --no-cache-dir -U psycopg2 mysqlclient pgcli mariadb - +# Install required base-level python packages +COPY requirements.txt requirements.txt +RUN pip install --no-cache-dir -U -r requirements.txt FROM base as production # Clone source code diff --git a/docker/requirements.txt b/docker/requirements.txt new file mode 100644 index 0000000000..b15d7c538d --- /dev/null +++ b/docker/requirements.txt @@ -0,0 +1,13 @@ +# Base python requirements for docker containers + +# Basic package requirements +setuptools>=57.4.0 +wheel>=0.37.0 +invoke>=1.4.0 # Invoke build tool +gunicorn>=20.1.0 # Gunicorn web server + +# Database links +psycopg2>=2.9.1 +mysqlclient>=2.0.3 +pgcli>=3.1.0 +mariadb>=1.0.7 diff --git a/requirements.txt b/requirements.txt index 637dbda99a..049bedcbeb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,5 @@ -# Basic package requirements -setuptools>=57.4.0 -wheel>=0.37.0 -invoke>=1.4.0 # Invoke build tool -gunicorn>=20.1.0 # Gunicorn web server - # Django framework -Django==3.2.4 # Django package +Django==3.2.4 # Django package pillow==8.2.0 # Image manipulation djangorestframework==3.12.4 # DRF framework From c1ea6dbb9b753603f80df568a5c5f11261fa0c6b Mon Sep 17 00:00:00 2001 From: Oliver Date: Wed, 18 Aug 2021 12:28:09 +1000 Subject: [PATCH 09/17] Remove commented out functionality from the entrypoint command --- docker/init.sh | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/docker/init.sh b/docker/init.sh index 9be8ffe44b..b598a3ee79 100644 --- a/docker/init.sh +++ b/docker/init.sh @@ -36,21 +36,7 @@ if [[ -n "$INVENTREE_PY_ENV" ]]; then # e.g docker-compose -f docker-compose.dev.yml run inventree-dev-server invoke install fi -# Wait for the InvenTree database to be ready -# cd ${INVENTREE_MNG_DIR} -# echo "InvenTree: Waiting for database connection" -# invoke wait && echo "InvenTree: Database connection successful" || { echo "InvenTree: Failed to connect to db due to errors, aborting"; exit 1; } -# sleep 5 - cd ${INVENTREE_HOME} -# We assume at this stage that the database is up and running -# echo "InvenTree: Checking database..." -# invoke check || exit 1 - -# Can be run as a cron job or directly to clean out expired sessions. -# cd ${INVENTREE_MNG_DIR} -# python3 manage.py clearsessions || exit 1 - # Launch the CMD *after* the ENTRYPOINT completes exec "$@" From d756579a067dbd6e7098cd43e438c3b61b43da17 Mon Sep 17 00:00:00 2001 From: Oliver Date: Wed, 18 Aug 2021 13:02:36 +1000 Subject: [PATCH 10/17] Split production environment variables out into a .env file --- docker/docker-compose.yml | 37 ++++++++++++++----------------------- docker/prod-config.env | 16 ++++++++++++++++ 2 files changed, 30 insertions(+), 23 deletions(-) create mode 100644 docker/prod-config.env diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index dcd35af148..3f8443065a 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -21,12 +21,13 @@ services: # just make sure that you change the INVENTREE_DB_xxx vars below inventree-db: container_name: inventree-db - image: postgres + image: postgres:13 ports: - 5432/tcp environment: - PGDATA=/var/lib/postgresql/data/pgdb # The pguser and pgpassword values must be the same in the other containers + # Ensure that these are correctly configured in your prod-config.env file - POSTGRES_USER=pguser - POSTGRES_PASSWORD=pgpassword volumes: @@ -38,6 +39,8 @@ services: # Uses gunicorn as the web server inventree-server: container_name: inventree-server + # If you wish to specify a particular InvenTree version, do so here + # e.g. image: inventree/inventree:0.5.2 image: inventree/inventree:latest expose: - 8000 @@ -46,39 +49,27 @@ services: volumes: # Data volume must map to /home/inventree/data - data:/home/inventree/data - environment: - # Default environment variables are configured to match the 'db' container - # Note: If you change the database image, these will need to be adjusted - # Note: INVENTREE_DB_HOST should match the container name of the database - - INVENTREE_DB_USER=pguser - - INVENTREE_DB_PASSWORD=pgpassword - - INVENTREE_DB_ENGINE=postgresql - - INVENTREE_DB_NAME=inventree - - INVENTREE_DB_HOST=inventree-db - - INVENTREE_DB_PORT=5432 + env_file: + # Environment variables required for the production server are configured in prod-config.env + - prod-config.env restart: unless-stopped # Background worker process handles long-running or periodic tasks inventree-worker: container_name: inventree-worker + # If you wish to specify a particular InvenTree version, do so here + # e.g. image: inventree/inventree:0.5.2 image: inventree/inventree:latest - entrypoint: ./start_prod_worker.sh + command: invoke worker depends_on: - inventree-db - inventree-server volumes: # Data volume must map to /home/inventree/data - data:/home/inventree/data - environment: - # Default environment variables are configured to match the 'db' container - # Note: If you change the database image, these will need to be adjusted - # Note: INVENTREE_DB_HOST should match the container name of the database - - INVENTREE_DB_USER=pguser - - INVENTREE_DB_PASSWORD=pgpassword - - INVENTREE_DB_ENGINE=postgresql - - INVENTREE_DB_NAME=inventree - - INVENTREE_DB_HOST=inventree-db - - INVENTREE_DB_PORT=5432 + env_file: + # Environment variables required for the production server are configured in prod-config.env + - prod-config.env restart: unless-stopped # nginx acts as a reverse proxy @@ -88,7 +79,7 @@ services: # NOTE: You will need to provide a working nginx.conf file! inventree-proxy: container_name: inventree-proxy - image: nginx + image: nginx:stable depends_on: - inventree-server ports: diff --git a/docker/prod-config.env b/docker/prod-config.env new file mode 100644 index 0000000000..93e3d123d6 --- /dev/null +++ b/docker/prod-config.env @@ -0,0 +1,16 @@ +# InvenTree environment variables for a production setup + +# Note: If your production setup varies from the example, you may want to change these values + +# Ensure debug is false for a production setup +INVENTREE_DEBUG=False +INVENTREE_DEBUG_LEVEL="WARNING" + +# Database configuration +# Note: The example setup is for a PostgreSQL database (change as required) +INVENTREE_DB_ENGINE=postgresql +INVENTREE_DB_NAME=inventree +INVENTREE_DB_HOST=inventree-db +INVENTREE_DB_PORT=5432 +INVENTREE_DB_USER=pguser +INVENTREE_DB_PASSWEORD=pgpassword From db477bceab768e7fcb2d7b90d94e91ccfe81dc19 Mon Sep 17 00:00:00 2001 From: Oliver Date: Wed, 18 Aug 2021 14:47:34 +1000 Subject: [PATCH 11/17] typo fix --- docker/Dockerfile | 4 ++-- docker/prod-config.env | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 9cfe88b2b5..2150b51558 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -51,8 +51,6 @@ LABEL org.label-schema.schema-version="1.0" \ # Create user account RUN addgroup -S inventreegroup && adduser -S inventree -G inventreegroup -WORKDIR ${INVENTREE_HOME} - # Install required system packages RUN apk add --no-cache git make bash \ gcc libgcc g++ libstdc++ \ @@ -75,6 +73,8 @@ RUN apk add --no-cache git make bash \ # Install required base-level python packages COPY requirements.txt requirements.txt RUN pip install --no-cache-dir -U -r requirements.txt + +# Production code (pulled from tagged github release) FROM base as production # Clone source code diff --git a/docker/prod-config.env b/docker/prod-config.env index 93e3d123d6..bb922f4b32 100644 --- a/docker/prod-config.env +++ b/docker/prod-config.env @@ -13,4 +13,4 @@ INVENTREE_DB_NAME=inventree INVENTREE_DB_HOST=inventree-db INVENTREE_DB_PORT=5432 INVENTREE_DB_USER=pguser -INVENTREE_DB_PASSWEORD=pgpassword +INVENTREE_DB_PASSWORD=pgpassword From 41db0ff60d19fbfee6f85883f647bb766e5a9fb5 Mon Sep 17 00:00:00 2001 From: Oliver Date: Wed, 18 Aug 2021 14:58:16 +1000 Subject: [PATCH 12/17] Need to specify python3 --- tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks.py b/tasks.py index 9088efb12f..7ebdd17480 100644 --- a/tasks.py +++ b/tasks.py @@ -188,7 +188,7 @@ def translate(c): path = os.path.join('InvenTree', 'script', 'translation_stats.py') - c.run(f'python {path}') + c.run(f'python3 {path}') @task(pre=[install, migrate, translate, clean_settings]) From 79d7a9f922862bf9b55d520bd2a3d6777846d2e0 Mon Sep 17 00:00:00 2001 From: Oliver Date: Wed, 18 Aug 2021 15:16:22 +1000 Subject: [PATCH 13/17] fix typo in dockerfile --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 2150b51558..1fadaa77cd 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -46,7 +46,7 @@ LABEL org.label-schema.schema-version="1.0" \ org.label-schema.url="https://hub.docker.com/r/inventree/inventree" \ org.label-schema.vcs-url=${INVENTREE_GIT_REPO} \ org.label-schema.vcs-branch=${INVENTREE_GIT_BRANCH} \ - org.label-schema.vss-ref=${INVENTREE_GIT_TAG} + org.label-schema.vcs-ref=${INVENTREE_GIT_TAG} # Create user account RUN addgroup -S inventreegroup && adduser -S inventree -G inventreegroup From 2095d666778c42ff3418564c5fbb8c30be48fe5e Mon Sep 17 00:00:00 2001 From: Oliver Date: Wed, 18 Aug 2021 16:29:54 +1000 Subject: [PATCH 14/17] Fix entrypoint / cmd for production server --- InvenTree/InvenTree/tasks.py | 2 +- docker/Dockerfile | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/InvenTree/InvenTree/tasks.py b/InvenTree/InvenTree/tasks.py index 24631dc9e5..deb834c322 100644 --- a/InvenTree/InvenTree/tasks.py +++ b/InvenTree/InvenTree/tasks.py @@ -36,7 +36,7 @@ def schedule_task(taskname, **kwargs): # If this task is already scheduled, don't schedule it again # Instead, update the scheduling parameters if Schedule.objects.filter(func=taskname).exists(): - logger.info(f"Scheduled task '{taskname}' already exists - updating!") + logger.debug(f"Scheduled task '{taskname}' already exists - updating!") Schedule.objects.filter(func=taskname).update(**kwargs) else: diff --git a/docker/Dockerfile b/docker/Dockerfile index 1fadaa77cd..f117055a78 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -87,19 +87,20 @@ RUN if [ -n "${INVENTREE_GIT_TAG}" ] ; then cd ${INVENTREE_HOME} && git fetch -- RUN chown -R inventree:inventreegroup ${INVENTREE_HOME}/* -WORKDIR ${INVENTREE_HOME} - # Drop to the inventree user USER inventree # Install InvenTree packages RUN pip3 install --no-cache-dir --disable-pip-version-check -r ${INVENTREE_HOME}/requirements.txt +# Need to be running from within this directory +WORKDIR ${INVENTREE_MNG_DIR} + # Server init entrypoint -ENTRYPOINT ["/bin/bash", "./docker/init.sh"] +ENTRYPOINT ["/bin/bash", "../docker/init.sh"] # Launch the production server -CMD ["gunicorn -c ./docker/gunicorn.conf.py -b ${INVENTREE_WEB_ADDR}:${INVENTREE_WEB_PORT} InvenTree.wsgi"] +CMD ["gunicorn", "-c", "../docker/gunicorn.conf.py", "InvenTree.wsgi", "-b", "${INVENTREE_WEB_ADDR}:${INVENTREE_WEB_PORT}", "--pythonpath", "${INVENTREE_MNG_DIR}"] FROM base as dev From eeac561b9b568f95f31e260f1d2293cfb5b4a98f Mon Sep 17 00:00:00 2001 From: Oliver Date: Wed, 18 Aug 2021 17:07:23 +1000 Subject: [PATCH 15/17] typo fix --- docker/prod-config.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/prod-config.env b/docker/prod-config.env index bb922f4b32..50cf7a867b 100644 --- a/docker/prod-config.env +++ b/docker/prod-config.env @@ -4,7 +4,7 @@ # Ensure debug is false for a production setup INVENTREE_DEBUG=False -INVENTREE_DEBUG_LEVEL="WARNING" +INVENTREE_LOG_LEVEL="WARNING" # Database configuration # Note: The example setup is for a PostgreSQL database (change as required) From 52bdfe5465a9c60afbf723ab8d656a4100404fe5 Mon Sep 17 00:00:00 2001 From: Oliver Date: Wed, 18 Aug 2021 20:52:14 +1000 Subject: [PATCH 16/17] Env interpolation doesn't seem to work in the CMD --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index f117055a78..2e6edd273d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -100,7 +100,7 @@ WORKDIR ${INVENTREE_MNG_DIR} ENTRYPOINT ["/bin/bash", "../docker/init.sh"] # Launch the production server -CMD ["gunicorn", "-c", "../docker/gunicorn.conf.py", "InvenTree.wsgi", "-b", "${INVENTREE_WEB_ADDR}:${INVENTREE_WEB_PORT}", "--pythonpath", "${INVENTREE_MNG_DIR}"] +CMD gunicorn -c ./docker/gunicorn.conf.py InvenTree.wsgi -b 0.0.0.0:8000 --chdir ./InvenTree FROM base as dev From 9ed2025021f5a04ff2fb041fddb77afbd0e8bef9 Mon Sep 17 00:00:00 2001 From: Oliver Date: Thu, 19 Aug 2021 11:14:13 +1000 Subject: [PATCH 17/17] Add a TODO for future reference --- docker/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index 2e6edd273d..e4ebbc1b4b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -100,6 +100,8 @@ WORKDIR ${INVENTREE_MNG_DIR} ENTRYPOINT ["/bin/bash", "../docker/init.sh"] # Launch the production server +# TODO: Work out why environment variables cannot be interpolated in this command +# TODO: e.g. -b ${INVENTREE_WEB_ADDR}:${INVENTREE_WEB_PORT} fails here CMD gunicorn -c ./docker/gunicorn.conf.py InvenTree.wsgi -b 0.0.0.0:8000 --chdir ./InvenTree FROM base as dev