inventree/InvenTree
2
0
Fork 0
mirror of https://github.com/inventree/InvenTree.git synced 2025-10-31 13:15:43 +00:00
Code Activity

feat(backend): add storages to make usage of s3/sftp easier (#10140)

Browse Source 
* feat(backend): add storages to make usage of S3 easy

* add S3/SFTP settings

* add changelog entry

* also configure static

* get it running on hetzner / exo

* doc additional settings

* fix style

* adress various review comments

* move setting files

* use enum for backends

* revert change

* split up storage settings

* fix comparison
This commit is contained in:
Matthias Mair
2025-10-29 21:57:22 +01:00
committed by GitHub
parent c1bbef1a4d
commit 6581af7165
13 changed files with 299 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/backend/InvenTree/InvenTree/helpers.py
View File

@@ -31,6 +31,7 @@ from PIL import Image
from common.currency import currency_code_default
from .setting.storages import StorageBackends
from .settings import MEDIA_URL, STATIC_URL
logger = structlog.get_logger('inventree')
@@ -176,6 +177,8 @@ def constructPathString(path: list[str], max_chars: int = 250) -> str:
def getMediaUrl(filename):
"""Return the qualified access path for the given file, under the media directory."""
if settings.STORAGE_TARGET == StorageBackends.S3:
return str(filename)
return os.path.join(MEDIA_URL, str(filename))

6
src/backend/InvenTree/InvenTree/serializers.py
View File

@@ -29,6 +29,8 @@ from common.currency import currency_code_default, currency_code_mappings
from InvenTree.fields import InvenTreeRestURLField, InvenTreeURLField
from InvenTree.helpers import str2bool
from .setting.storages import StorageBackends
# region path filtering
class FilterableSerializerField:
@@ -613,6 +615,8 @@ class InvenTreeAttachmentSerializerField(serializers.FileField):
if not value:
return None
if settings.STORAGE_TARGET == StorageBackends.S3:
return str(value.url)
return os.path.join(str(settings.MEDIA_URL), str(value))
@@ -627,6 +631,8 @@ class InvenTreeImageSerializerField(serializers.ImageField):
if not value:
return None
if settings.STORAGE_TARGET == StorageBackends.S3:
return str(value.url)
return os.path.join(str(settings.MEDIA_URL), str(value))

1
src/backend/InvenTree/InvenTree/setting/__init__.py Normal file
View File

@@ -0,0 +1 @@
"""Sub-setting files."""

0
src/backend/InvenTree/InvenTree/locales.py → src/backend/InvenTree/InvenTree/setting/locales.py
View File

111
src/backend/InvenTree/InvenTree/setting/storages.py Normal file
View File

@@ -0,0 +1,111 @@
"""Settings for storage backends."""
from enum import Enum
from typing import Optional
from InvenTree.config import get_boolean_setting, get_setting
class StorageBackends(str, Enum):
"""Enumeration of available storage backends."""
LOCAL = 'local'
S3 = 's3'
SFTP = 'sftp'
STORAGE_BACKEND_MAPPING = {
StorageBackends.LOCAL: 'django.core.files.storage.FileSystemStorage',
StorageBackends.S3: 'storages.backends.s3.S3Storage',
StorageBackends.SFTP: 'storages.backends.sftpstorage.SFTPStorage',
}
def init_storages() -> tuple[str, dict, Optional[str]]:
"""Initialize storage backend settings."""
target = get_setting(
'INVENTREE_STORAGE_TARGET',
'storage.target',
StorageBackends.LOCAL,
typecast=str,
)
# Check that the target is valid
if target not in STORAGE_BACKEND_MAPPING:
raise ValueError(f"Invalid storage target: '{target}'")
options = {}
media_url: Optional[str] = None
if target == 's3':
s3_bucket = get_setting(
'INVENTREE_S3_BUCKET_NAME', 'storage.s3.bucket_name', None, typecast=str
)
s3_acl = get_setting(
'INVENTREE_S3_DEFAULT_ACL', 'storage.s3.default_acl', None, typecast=str
)
s3_endpoint = get_setting(
'INVENTREE_S3_ENDPOINT_URL', 'storage.s3.endpoint_url', None, typecast=str
)
s3_location = get_setting(
'INVENTREE_S3_LOCATION',
'storage.s3.location',
'inventree-server',
typecast=str,
)
media_url = f'{s3_endpoint}/{s3_bucket}/{s3_location}/'
options = {
'access_key': get_setting(
'INVENTREE_S3_ACCESS_KEY', 'storage.s3.access_key', None, typecast=str
),
'secret_key': get_setting(
'INVENTREE_S3_SECRET_KEY', 'storage.s3.secret_key', None, typecast=str
),
'bucket_name': s3_bucket,
'default_acl': s3_acl,
'region_name': get_setting(
'INVENTREE_S3_REGION_NAME', 'storage.s3.region_name', None, typecast=str
),
'endpoint_url': s3_endpoint,
'verify': get_boolean_setting(
'INVENTREE_S3_VERIFY_SSL', 'storage.s3.verify_ssl', True
),
'location': s3_location,
'file_overwrite': get_boolean_setting(
'INVENTREE_S3_OVERWRITE', 'storage.s3.overwrite', True
),
'addressing_style': 'virtual'
if get_boolean_setting('INVENTREE_S3_VIRTUAL', 'storage.s3.virtual', False)
else 'path',
'object_parameters': {'CacheControl': 'public,max-age=86400'},
}
elif target == 'sftp':
options = {
'host': get_setting('INVENTREE_SFTP_HOST', 'sftp.host', None, typecast=str),
'uid': get_setting('INVENTREE_SFTP_UID', 'sftp.uid', None, typecast=int),
'gid': get_setting('INVENTREE_SFTP_GID', 'sftp.gid', None, typecast=int),
'location': get_setting(
'INVENTREE_SFTP_LOCATION',
'sftp.location',
'inventree-server',
typecast=str,
),
'params': get_setting(
'INVENTREE_SFTP_PARAMS', 'sftp.params', {}, typecast=dict
),
}
return (
target,
{
'default': {
'BACKEND': STORAGE_BACKEND_MAPPING.get(
target, STORAGE_BACKEND_MAPPING[StorageBackends.LOCAL]
),
'OPTIONS': options,
},
'staticfiles': {
'BACKEND': 'whitenoise.storage.CompressedStaticFilesStorage'
},
},
media_url,
)

10
src/backend/InvenTree/InvenTree/settings.py
View File

@@ -41,7 +41,8 @@ from InvenTree.version import (
)
from users.oauth2_scopes import oauth2_scopes
from . import config, locales
from . import config
from .setting import locales, storages
try:
import django_stubs_ext
@@ -343,6 +344,7 @@ INSTALLED_APPS = [
'django_ical', # For exporting calendars
'django_mailbox', # For email import
'anymail', # For email sending/receiving via ESPs
'storages',
]
MIDDLEWARE = CONFIG.get(
@@ -1562,3 +1564,9 @@ OAUTH2_CHECK_EXCLUDED = [ # This setting mutes schema checks for these rule/met
if SITE_URL and not TESTING: # pragma: no cover
SPECTACULAR_SETTINGS['SERVERS'] = [{'url': SITE_URL}]
# Storage backends
STORAGE_TARGET, STORAGES, _media = storages.init_storages()
if _media:
MEDIA_URL = _media
PRESIGNED_URL_EXPIRATION = 600

27
src/backend/InvenTree/config_template.yaml
View File

@@ -229,3 +229,30 @@ ldap:
#global_settings:
# INVENTREE_DEFAULT_CURRENCY: 'CNY'
# INVENTREE_RESTRICT_ABOUT: true
# Storage configuration
# Ref: https://docs.inventree.org/en/stable/start/config/#storage-backends
storage:
target: local # s3, sftp
# s3:
# access_key: 'abc123-key'
# secret_key: 'abc123-secret'
# bucket_name: 'my-bucket'
# region_name: 'fsn1'
# endpoint_url: 'https://fsn1.your-objectstorage.com'
# location: 'inventree-server_subdir'
# default_acl: private
# verify_ssl: true
# overwrite: true
# virtual: true
# sftp:
# host: 'sftp://ftp-target.example.org:22'
# uid: 1000
# gid: 1000
# location: 'inventree-server_subdir'
# params:
# # See https://docs.paramiko.org/en/latest/api/client.html#paramiko.client.SSHClient.connect
# port: 22
# username: 'user'
# password: 'pwd'
# compress: False

2
src/backend/InvenTree/part/serializers.py
View File

@@ -260,7 +260,7 @@ class PartThumbSerializer(serializers.Serializer):
Used to serve and display existing Part images.
"""
image = serializers.URLField(read_only=True)
image = InvenTree.serializers.InvenTreeImageSerializerField(read_only=True)
count = serializers.IntegerField(read_only=True)

1
src/backend/requirements.in
View File

@@ -26,6 +26,7 @@ django-sql-utils # Advanced query annotation / aggregatio
django-sslserver # Secure HTTP development server
django-structlog # Structured logging
django-stdimage # Advanced ImageField management
django-storages[s3,sftp] # Storage backends for Django
django-taggit # Tagging support
django-otp==1.3.0 # Two-factor authentication (legacy to ensure migrations) https://github.com/inventree/InvenTree/pull/6293
django-oauth-toolkit # OAuth2 provider

101
src/backend/requirements.txt
View File

@@ -22,6 +22,59 @@ babel==2.17.0 \
--hash=sha256:0c54cffb19f690cdcc52a3b50bcbf71e07a808d1c80d549f2459b9d2cf0afb9d \
--hash=sha256:4d0b53093fdfb4b21c92b5213dba5a1b23885afa8383709427046b21c366e5f2
# via py-moneyed
bcrypt==4.3.0 \
--hash=sha256:0042b2e342e9ae3d2ed22727c1262f76cc4f345683b5c1715f0250cf4277294f \
--hash=sha256:0142b2cb84a009f8452c8c5a33ace5e3dfec4159e7735f5afe9a4d50a8ea722d \
--hash=sha256:08bacc884fd302b611226c01014eca277d48f0a05187666bca23aac0dad6fe24 \
--hash=sha256:0d3efb1157edebfd9128e4e46e2ac1a64e0c1fe46fb023158a407c7892b0f8c3 \
--hash=sha256:0e30e5e67aed0187a1764911af023043b4542e70a7461ad20e837e94d23e1d6c \
--hash=sha256:107d53b5c67e0bbc3f03ebf5b030e0403d24dda980f8e244795335ba7b4a027d \
--hash=sha256:12fa6ce40cde3f0b899729dbd7d5e8811cb892d31b6f7d0334a1f37748b789fd \
--hash=sha256:17a854d9a7a476a89dcef6c8bd119ad23e0f82557afbd2c442777a16408e614f \
--hash=sha256:191354ebfe305e84f344c5964c7cd5f924a3bfc5d405c75ad07f232b6dffb49f \
--hash=sha256:2ef6630e0ec01376f59a006dc72918b1bf436c3b571b80fa1968d775fa02fe7d \
--hash=sha256:3004df1b323d10021fda07a813fd33e0fd57bef0e9a480bb143877f6cba996fe \
--hash=sha256:335a420cfd63fc5bc27308e929bee231c15c85cc4c496610ffb17923abf7f231 \
--hash=sha256:33752b1ba962ee793fa2b6321404bf20011fe45b9afd2a842139de3011898fef \
--hash=sha256:3a3fd2204178b6d2adcf09cb4f6426ffef54762577a7c9b54c159008cb288c18 \
--hash=sha256:3b8d62290ebefd49ee0b3ce7500f5dbdcf13b81402c05f6dafab9a1e1b27212f \
--hash=sha256:3e36506d001e93bffe59754397572f21bb5dc7c83f54454c990c74a468cd589e \
--hash=sha256:41261d64150858eeb5ff43c753c4b216991e0ae16614a308a15d909503617732 \
--hash=sha256:50e6e80a4bfd23a25f5c05b90167c19030cf9f87930f7cb2eacb99f45d1c3304 \
--hash=sha256:531457e5c839d8caea9b589a1bcfe3756b0547d7814e9ce3d437f17da75c32b0 \
--hash=sha256:55a935b8e9a1d2def0626c4269db3fcd26728cbff1e84f0341465c31c4ee56d8 \
--hash=sha256:57967b7a28d855313a963aaea51bf6df89f833db4320da458e5b3c5ab6d4c938 \
--hash=sha256:584027857bc2843772114717a7490a37f68da563b3620f78a849bcb54dc11e62 \
--hash=sha256:59e1aa0e2cd871b08ca146ed08445038f42ff75968c7ae50d2fdd7860ade2180 \
--hash=sha256:5bd3cca1f2aa5dbcf39e2aa13dd094ea181f48959e1071265de49cc2b82525af \
--hash=sha256:5c1949bf259a388863ced887c7861da1df681cb2388645766c89fdfd9004c669 \
--hash=sha256:62f26585e8b219cdc909b6a0069efc5e4267e25d4a3770a364ac58024f62a761 \
--hash=sha256:67a561c4d9fb9465ec866177e7aebcad08fe23aaf6fbd692a6fab69088abfc51 \
--hash=sha256:6fb1fd3ab08c0cbc6826a2e0447610c6f09e983a281b919ed721ad32236b8b23 \
--hash=sha256:74a8d21a09f5e025a9a23e7c0fd2c7fe8e7503e4d356c0a2c1486ba010619f09 \
--hash=sha256:79e70b8342a33b52b55d93b3a59223a844962bef479f6a0ea318ebbcadf71505 \
--hash=sha256:7a4be4cbf241afee43f1c3969b9103a41b40bcb3a3f467ab19f891d9bc4642e4 \
--hash=sha256:7c03296b85cb87db865d91da79bf63d5609284fc0cab9472fdd8367bbd830753 \
--hash=sha256:842d08d75d9fe9fb94b18b071090220697f9f184d4547179b60734846461ed59 \
--hash=sha256:864f8f19adbe13b7de11ba15d85d4a428c7e2f344bac110f667676a0ff84924b \
--hash=sha256:97eea7408db3a5bcce4a55d13245ab3fa566e23b4c67cd227062bb49e26c585d \
--hash=sha256:a839320bf27d474e52ef8cb16449bb2ce0ba03ca9f44daba6d93fa1d8828e48a \
--hash=sha256:afe327968aaf13fc143a56a3360cb27d4ad0345e34da12c7290f1b00b8fe9a8b \
--hash=sha256:b4d4e57f0a63fd0b358eb765063ff661328f69a04494427265950c71b992a39a \
--hash=sha256:b6354d3760fcd31994a14c89659dee887f1351a06e5dac3c1142307172a79f90 \
--hash=sha256:b693dbb82b3c27a1604a3dff5bfc5418a7e6a781bb795288141e5f80cf3a3492 \
--hash=sha256:bdc6a24e754a555d7316fa4774e64c6c3997d27ed2d1964d55920c7c227bc4ce \
--hash=sha256:beeefe437218a65322fbd0069eb437e7c98137e08f22c4660ac2dc795c31f8bb \
--hash=sha256:c5eeac541cefd0bb887a371ef73c62c3cd78535e4887b310626036a7c0a817bb \
--hash=sha256:c950d682f0952bafcceaf709761da0a32a942272fad381081b51096ffa46cea1 \
--hash=sha256:d9af79d322e735b1fc33404b5765108ae0ff232d4b54666d46730f8ac1a43676 \
--hash=sha256:e53e074b120f2877a35cc6c736b8eb161377caae8925c17688bd46ba56daaa5b \
--hash=sha256:e965a9c1e9a393b8005031ff52583cedc15b7884fce7deb8b0346388837d6cfe \
--hash=sha256:f01e060f14b6b57bbb72fc5b4a83ac21c443c9a2ee708e04a10e9192f90a6281 \
--hash=sha256:f1e3ffa1365e8702dc48c8b360fef8d7afeca482809c5e45e653af82ccd088c1 \
--hash=sha256:f6746e6fec103fcd509b96bacdfdaa2fbde9a553245dbada284435173a6f1aef \
--hash=sha256:f81b0ed2639568bf14749112298f9e4e2b28853dab50a8b357e31798686a036d
# via paramiko
bleach[css]==6.2.0 \
--hash=sha256:117d9c6097a7c3d22fd578fcd8d35ff1e125df6736f554da4e432fdd63f31e5e \
--hash=sha256:123e894118b8a599fd80d3ec1a6d4cc7ce4e5882b1317a7e1ba69b56e95f991f
@@ -33,7 +86,9 @@ blessed==1.22.0 \
boto3==1.40.55 \
--hash=sha256:27e35b4fa9edd414ce06c1a748bf57cacd8203271847d93fc1053e4a4ec6e1a9 \
--hash=sha256:2e30f5a0d49e107b8a5c0c487891afd300bfa410e1d918bf187ae45ac3839332
# via django-anymail
# via
# django-anymail
# django-storages
botocore==1.40.55 \
--hash=sha256:79b6472e2de92b3519d44fc1eec8c5feced7f99a0d10fdea6dc93133426057c1 \
--hash=sha256:cdc38f7a4ddb30a2cd1cdd4fabde2a5a16e41b5a642292e1c30de5c4e46f5d44
@@ -260,6 +315,7 @@ cffi==2.0.0 \
--hash=sha256:fe562eb1a64e67dd297ccc4f5addea2501664954f2692b69a76449ec7913ecbf
# via
# cryptography
# pynacl
# weasyprint
charset-normalizer==3.4.4 \
--hash=sha256:027f6de494925c0ab2a55eab46ae5129951638a49a34d87f4c3eda90f696b4ad \
@@ -420,6 +476,7 @@ cryptography==44.0.3 \
# djangorestframework-simplejwt
# fido2
# jwcrypto
# paramiko
# pyjwt
cssselect2==0.8.0 \
--hash=sha256:46fc70ebc41ced7a32cd42d58b1884d72ade23d21e5a4eaaf022401c13f0e76e \
@@ -455,6 +512,7 @@ django==4.2.25 \
# django-sql-utils
# django-sslserver
# django-stdimage
# django-storages
# django-structlog
# django-taggit
# django-xforwardedfor-middleware
@@ -565,6 +623,10 @@ django-stdimage==6.0.2 \
--hash=sha256:880ab14828be56b53f711c3afae83c219ddd5d9af00850626736feb48382bf7f \
--hash=sha256:9a73f7da48c48074580e2b032d5bdb7164935dbe4b9dc4fb88a7e112f3d521c8
# via -r src/backend/requirements.in
django-storages[s3, sftp]==1.14.6 \
--hash=sha256:11b7b6200e1cb5ffcd9962bd3673a39c7d6a6109e8096f0e03d46fab3d3aabd9 \
--hash=sha256:7a25ce8f4214f69ac9c7ce87e2603887f7ae99326c316bc8d2d75375e09341c9
# via -r src/backend/requirements.in
django-structlog==9.1.1 \
--hash=sha256:14342c6c824581f1e063c88a8bc52314cd67995a3bd4a4fc8c27ea37ccd78947 \
--hash=sha256:5b6ac3abdf6549e94ccb35160b1f10266f1627c3ac77844571235a08a1ddae66
@@ -798,6 +860,10 @@ inflection==0.5.1 \
--hash=sha256:1a29730d366e996aaacffb2f1f1cb9593dc38e2ddd30c91250c6dde09ea9b417 \
--hash=sha256:f38b2b640938a4f35ade69ac3d053042959b62a0f1076a5bbaa1b9526605a8a2
# via drf-spectacular
invoke==2.2.0 \
--hash=sha256:6ea924cc53d4f78e3d98bc436b08069a03077e6f85ad1ddaa8a116d7dad15820 \
--hash=sha256:ee6cbb101af1a859c7fe84f2a264c059020b0cb7fe3535f9424300ab568f6bd5
# via paramiko
isodate==0.7.2 \
--hash=sha256:28009937d8031054830160fce6d409ed342816b543597cece116d966c6d99e15 \
--hash=sha256:4cd1aa0f43ca76f4a6c6c0292a85f40b35ec2e43e315b59f06e6d32171a953e6
@@ -1202,6 +1268,10 @@ packaging==25.0 \
# via
# gunicorn
# opentelemetry-instrumentation
paramiko==4.0.0 \
--hash=sha256:0e20e00ac666503bf0b4eda3b6d833465a2b7aff2e2b3d79a8bba5ef144ee3b9 \
--hash=sha256:6a25f07b380cc9c9a88d2b920ad37167ac4667f8d9886ccebd8f90f654b5d69f
# via django-storages
pdf2image==1.17.0 \
--hash=sha256:eaa959bc116b420dd7ec415fcae49b98100dda3dd18cd2fdfa86d09f112f6d57 \
--hash=sha256:ecdd58d7afb810dffe21ef2b1bbc057ef434dabbac6c33778a38a3f7744a27e2
@@ -1383,6 +1453,35 @@ pyjwt[crypto]==2.10.1 \
# via
# django-allauth
# djangorestframework-simplejwt
pynacl==1.6.0 \
--hash=sha256:04f20784083014e265ad58c1b2dd562c3e35864b5394a14ab54f5d150ee9e53e \
--hash=sha256:10d755cf2a455d8c0f8c767a43d68f24d163b8fe93ccfaabfa7bafd26be58d73 \
--hash=sha256:140373378e34a1f6977e573033d1dd1de88d2a5d90ec6958c9485b2fd9f3eb90 \
--hash=sha256:16c60daceee88d04f8d41d0a4004a7ed8d9a5126b997efd2933e08e93a3bd850 \
--hash=sha256:16dd347cdc8ae0b0f6187a2608c0af1c8b7ecbbe6b4a06bff8253c192f696990 \
--hash=sha256:25720bad35dfac34a2bcdd61d9e08d6bfc6041bebc7751d9c9f2446cf1e77d64 \
--hash=sha256:2d6cd56ce4998cb66a6c112fda7b1fdce5266c9f05044fa72972613bef376d15 \
--hash=sha256:347dcddce0b4d83ed3f32fd00379c83c425abee5a9d2cd0a2c84871334eaff64 \
--hash=sha256:4853c154dc16ea12f8f3ee4b7e763331876316cc3a9f06aeedf39bcdca8f9995 \
--hash=sha256:49c336dd80ea54780bcff6a03ee1a476be1612423010472e60af83452aa0f442 \
--hash=sha256:4a25cfede801f01e54179b8ff9514bd7b5944da560b7040939732d1804d25419 \
--hash=sha256:51fed9fe1bec9e7ff9af31cd0abba179d0e984a2960c77e8e5292c7e9b7f7b5d \
--hash=sha256:536703b8f90e911294831a7fbcd0c062b837f3ccaa923d92a6254e11178aaf42 \
--hash=sha256:5789f016e08e5606803161ba24de01b5a345d24590a80323379fc4408832d290 \
--hash=sha256:6b08eab48c9669d515a344fb0ef27e2cbde847721e34bba94a343baa0f33f1f4 \
--hash=sha256:6b393bc5e5a0eb86bb85b533deb2d2c815666665f840a09e0aa3362bb6088736 \
--hash=sha256:84709cea8f888e618c21ed9a0efdb1a59cc63141c403db8bf56c469b71ad56f2 \
--hash=sha256:8bfaa0a28a1ab718bad6239979a5a57a8d1506d0caf2fba17e524dbb409441cf \
--hash=sha256:bbcc4452a1eb10cd5217318c822fde4be279c9de8567f78bad24c773c21254f8 \
--hash=sha256:cb36deafe6e2bce3b286e5d1f3e1c246e0ccdb8808ddb4550bb2792f2df298f2 \
--hash=sha256:cf831615cc16ba324240de79d925eacae8265b7691412ac6b24221db157f6bd1 \
--hash=sha256:dcdeb41c22ff3c66eef5e63049abf7639e0db4edee57ba70531fc1b6b133185d \
--hash=sha256:dea103a1afcbc333bc0e992e64233d360d393d1e63d0bc88554f572365664348 \
--hash=sha256:ef214b90556bb46a485b7da8258e59204c244b1b5b576fb71848819b468c44a7 \
--hash=sha256:f3482abf0f9815e7246d461fab597aa179b7524628a4bc36f86a7dc418d2608d \
--hash=sha256:f46386c24a65383a9081d68e9c2de909b1834ec74ff3013271f1bca9c2d233eb \
--hash=sha256:f4b3824920e206b4f52abd7de621ea7a44fd3cb5c8daceb7c3612345dfc54f2e
# via paramiko
pypdf==6.1.3 \
--hash=sha256:8d420d1e79dc1743f31a57707cabb6dcd5b17e8b9a302af64b30202c5700ab9d \
--hash=sha256:eb049195e46f014fc155f566fa20e09d70d4646a9891164ac25fa0cbcfcdbcb5