diff --git a/InvenTree/InvenTree/apps.py b/InvenTree/InvenTree/apps.py index faef1a6cdb..b32abf4a8e 100644 --- a/InvenTree/InvenTree/apps.py +++ b/InvenTree/InvenTree/apps.py @@ -100,7 +100,7 @@ class InvenTreeConfig(AppConfig): try: from djmoney.contrib.exchange.models import ExchangeBackend - from datetime import datetime, timedelta + from InvenTree.tasks import update_exchange_rates from common.settings import currency_code_default except AppRegistryNotReady: @@ -115,23 +115,18 @@ class InvenTreeConfig(AppConfig): last_update = backend.last_update - if last_update is not None: - delta = datetime.now().date() - last_update.date() - if delta > timedelta(days=1): - print(f"Last update was {last_update}") - update = True - else: + if last_update is None: # Never been updated - print("Exchange backend has never been updated") + logger.info("Exchange backend has never been updated") update = True # Backend currency has changed? if not base_currency == backend.base_currency: - print(f"Base currency changed from {backend.base_currency} to {base_currency}") + logger.info(f"Base currency changed from {backend.base_currency} to {base_currency}") update = True except (ExchangeBackend.DoesNotExist): - print("Exchange backend not found - updating") + logger.info("Exchange backend not found - updating") update = True except: @@ -139,4 +134,7 @@ class InvenTreeConfig(AppConfig): return if update: - update_exchange_rates() + try: + update_exchange_rates() + except Exception as e: + logger.error(f"Error updating exchange rates: {e}") diff --git a/InvenTree/InvenTree/exchange.py b/InvenTree/InvenTree/exchange.py index 4b99953382..a79239568d 100644 --- a/InvenTree/InvenTree/exchange.py +++ b/InvenTree/InvenTree/exchange.py @@ -1,3 +1,7 @@ +import certifi +import ssl +from urllib.request import urlopen + from common.settings import currency_code_default, currency_codes from urllib.error import URLError @@ -24,6 +28,22 @@ class InvenTreeExchange(SimpleExchangeBackend): return { } + def get_response(self, **kwargs): + """ + Custom code to get response from server. + Note: Adds a 5-second timeout + """ + + url = self.get_url(**kwargs) + + try: + context = ssl.create_default_context(cafile=certifi.where()) + response = urlopen(url, timeout=5, context=context) + return response.read() + except: + # Returning None here will raise an error upstream + return None + def update_rates(self, base_currency=currency_code_default()): symbols = ','.join(currency_codes()) diff --git a/InvenTree/InvenTree/ready.py b/InvenTree/InvenTree/ready.py index 7d63861f4b..9f5ad0ea49 100644 --- a/InvenTree/InvenTree/ready.py +++ b/InvenTree/InvenTree/ready.py @@ -6,10 +6,16 @@ def isInTestMode(): Returns True if the database is in testing mode """ - if 'test' in sys.argv: - return True + return 'test' in sys.argv - return False + +def isImportingData(): + """ + Returns True if the database is currently importing data, + e.g. 'loaddata' command is performed + """ + + return 'loaddata' in sys.argv def canAppAccessDatabase(allow_test=False): diff --git a/InvenTree/InvenTree/settings.py b/InvenTree/InvenTree/settings.py index f2654a49cc..171426bce5 100644 --- a/InvenTree/InvenTree/settings.py +++ b/InvenTree/InvenTree/settings.py @@ -172,12 +172,6 @@ if MEDIA_ROOT is None: print("ERROR: INVENTREE_MEDIA_ROOT directory is not defined") sys.exit(1) -# Options for django-maintenance-mode : https://pypi.org/project/django-maintenance-mode/ -MAINTENANCE_MODE_STATE_FILE_PATH = os.path.join( - config_dir, - 'maintenance_mode_state.txt', -) - # List of allowed hosts (default = allow all) ALLOWED_HOSTS = CONFIG.get('allowed_hosts', ['*']) @@ -870,6 +864,7 @@ MARKDOWNIFY_BLEACH = False # Maintenance mode MAINTENANCE_MODE_RETRY_AFTER = 60 +MAINTENANCE_MODE_STATE_BACKEND = 'maintenance_mode.backends.DefaultStorageBackend' # Are plugins enabled? PLUGINS_ENABLED = _is_true(get_setting( diff --git a/InvenTree/InvenTree/tasks.py b/InvenTree/InvenTree/tasks.py index 0a098e5f8c..a76f766120 100644 --- a/InvenTree/InvenTree/tasks.py +++ b/InvenTree/InvenTree/tasks.py @@ -269,10 +269,13 @@ def update_exchange_rates(): logger.info(f"Using base currency '{base}'") - backend.update_rates(base_currency=base) + try: + backend.update_rates(base_currency=base) - # Remove any exchange rates which are not in the provided currencies - Rate.objects.filter(backend="InvenTreeExchange").exclude(currency__in=currency_codes()).delete() + # Remove any exchange rates which are not in the provided currencies + Rate.objects.filter(backend="InvenTreeExchange").exclude(currency__in=currency_codes()).delete() + except Exception as e: + logger.error(f"Error updating exchange rates: {e}") def send_email(subject, body, recipients, from_email=None, html_message=None): diff --git a/InvenTree/InvenTree/validators.py b/InvenTree/InvenTree/validators.py index 76d485cef9..6bb2c1b350 100644 --- a/InvenTree/InvenTree/validators.py +++ b/InvenTree/InvenTree/validators.py @@ -2,6 +2,8 @@ Custom field validators for InvenTree """ +from decimal import Decimal, InvalidOperation + from django.conf import settings from django.core.exceptions import ValidationError from django.utils.translation import gettext_lazy as _ @@ -115,26 +117,28 @@ def validate_tree_name(value): def validate_overage(value): - """ Validate that a BOM overage string is properly formatted. + """ + Validate that a BOM overage string is properly formatted. An overage string can look like: - An integer number ('1' / 3 / 4) + - A decimal number ('0.123') - A percentage ('5%' / '10 %') """ value = str(value).lower().strip() - # First look for a simple integer value + # First look for a simple numerical value try: - i = int(value) + i = Decimal(value) if i < 0: raise ValidationError(_("Overage value must not be negative")) - # Looks like an integer! + # Looks like a number return True - except ValueError: + except (ValueError, InvalidOperation): pass # Now look for a percentage value @@ -155,7 +159,7 @@ def validate_overage(value): pass raise ValidationError( - _("Overage must be an integer value or a percentage") + _("Invalid value for overage") ) diff --git a/InvenTree/build/tasks.py b/InvenTree/build/tasks.py index 6fe4be5119..6752bc5501 100644 --- a/InvenTree/build/tasks.py +++ b/InvenTree/build/tasks.py @@ -12,6 +12,8 @@ from allauth.account.models import EmailAddress import build.models import InvenTree.helpers import InvenTree.tasks +from InvenTree.ready import isImportingData + import part.models as part_models @@ -24,6 +26,10 @@ def check_build_stock(build: build.models.Build): and send an email out to any subscribed users if stock is low. """ + # Do not notify if we are importing data + if isImportingData(): + return + # Iterate through each of the parts required for this build lines = [] diff --git a/InvenTree/build/templates/build/detail.html b/InvenTree/build/templates/build/detail.html index 8479c2819f..b548632b56 100644 --- a/InvenTree/build/templates/build/detail.html +++ b/InvenTree/build/templates/build/detail.html @@ -252,6 +252,7 @@ + {% include "filter_list.html" with id='incompletebuilditems' %} {% endif %} diff --git a/InvenTree/build/views.py b/InvenTree/build/views.py index 1a933af835..21cf5dda99 100644 --- a/InvenTree/build/views.py +++ b/InvenTree/build/views.py @@ -95,17 +95,24 @@ class BuildOutputCreate(AjaxUpdateView): quantity = form.cleaned_data.get('output_quantity', None) serials = form.cleaned_data.get('serial_numbers', None) - if quantity: + if quantity is not None: build = self.get_object() # Check that requested output don't exceed build remaining quantity maximum_output = int(build.remaining - build.incomplete_count) + if quantity > maximum_output: form.add_error( 'output_quantity', _('Maximum output quantity is ') + str(maximum_output), ) + elif quantity <= 0: + form.add_error( + 'output_quantity', + _('Output quantity must be greater than zero'), + ) + # Check that the serial numbers are valid if serials: try: diff --git a/InvenTree/common/models.py b/InvenTree/common/models.py index ba475e75b8..9ef5a4d0c3 100644 --- a/InvenTree/common/models.py +++ b/InvenTree/common/models.py @@ -776,6 +776,18 @@ class InvenTreeSetting(BaseInvenTreeSetting): 'validator': bool, }, + # 2022-02-03 + # This setting exists as an interim solution for extremely slow part page load times when the part has a complex BOM + # In an upcoming release, pricing history (and BOM pricing) will be cached, + # rather than having to be re-calculated every time the page is loaded! + # For now, we will simply hide part pricing by default + 'PART_SHOW_PRICE_HISTORY': { + 'name': _('Show Price History'), + 'description': _('Display historical pricing for Part'), + 'default': False, + 'validator': bool, + }, + 'PART_SHOW_RELATED': { 'name': _('Show related parts'), 'description': _('Display related parts for a part'), diff --git a/InvenTree/order/templates/order/order_base.html b/InvenTree/order/templates/order/order_base.html index 195f2273a3..c675574d30 100644 --- a/InvenTree/order/templates/order/order_base.html +++ b/InvenTree/order/templates/order/order_base.html @@ -48,7 +48,7 @@ {% endif %} -{% if order.status == PurchaseOrderStatus.PENDING and order.lines.count > 0 %} +{% if order.status == PurchaseOrderStatus.PENDING %} diff --git a/InvenTree/part/api.py b/InvenTree/part/api.py index d146c3d7b3..4c52b87520 100644 --- a/InvenTree/part/api.py +++ b/InvenTree/part/api.py @@ -1533,6 +1533,40 @@ class BomList(generics.ListCreateAPIView): ] +class BomExtract(generics.CreateAPIView): + """ + API endpoint for extracting BOM data from a BOM file. + """ + + queryset = Part.objects.none() + serializer_class = part_serializers.BomExtractSerializer + + def create(self, request, *args, **kwargs): + """ + Custom create function to return the extracted data + """ + + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + self.perform_create(serializer) + headers = self.get_success_headers(serializer.data) + + data = serializer.extract_data() + + return Response(data, status=status.HTTP_201_CREATED, headers=headers) + + +class BomUpload(generics.CreateAPIView): + """ + API endpoint for uploading a complete Bill of Materials. + + It is assumed that the BOM has been extracted from a file using the BomExtract endpoint. + """ + + queryset = Part.objects.all() + serializer_class = part_serializers.BomUploadSerializer + + class BomDetail(generics.RetrieveUpdateDestroyAPIView): """ API endpoint for detail view of a single BomItem object """ @@ -1685,6 +1719,10 @@ bom_api_urls = [ url(r'^.*$', BomDetail.as_view(), name='api-bom-item-detail'), ])), + url(r'^extract/', BomExtract.as_view(), name='api-bom-extract'), + + url(r'^upload/', BomUpload.as_view(), name='api-bom-upload'), + # Catch-all url(r'^.*$', BomList.as_view(), name='api-bom-list'), ] diff --git a/InvenTree/part/serializers.py b/InvenTree/part/serializers.py index 16ecc8da21..351348c6bc 100644 --- a/InvenTree/part/serializers.py +++ b/InvenTree/part/serializers.py @@ -4,9 +4,11 @@ JSON serializers for Part app import imghdr from decimal import Decimal +import os +import tablib from django.urls import reverse_lazy -from django.db import models +from django.db import models, transaction from django.db.models import Q from django.db.models.functions import Coalesce from django.utils.translation import ugettext_lazy as _ @@ -462,7 +464,13 @@ class BomItemSerializer(InvenTreeModelSerializer): price_range = serializers.CharField(read_only=True) - quantity = InvenTreeDecimalField() + quantity = InvenTreeDecimalField(required=True) + + def validate_quantity(self, quantity): + if quantity <= 0: + raise serializers.ValidationError(_("Quantity must be greater than zero")) + + return quantity part = serializers.PrimaryKeyRelatedField(queryset=Part.objects.filter(assembly=True)) @@ -699,3 +707,289 @@ class PartCopyBOMSerializer(serializers.Serializer): skip_invalid=data.get('skip_invalid', False), include_inherited=data.get('include_inherited', False), ) + + +class BomExtractSerializer(serializers.Serializer): + """ + Serializer for uploading a file and extracting data from it. + + Note: 2022-02-04 - This needs a *serious* refactor in future, probably + + When parsing the file, the following things happen: + + a) Check file format and validity + b) Look for "required" fields + c) Look for "part" fields - used to "infer" part + + Once the file itself has been validated, we iterate through each data row: + + - If the "level" column is provided, ignore anything below level 1 + - Try to "guess" the part based on part_id / part_name / part_ipn + - Extract other fields as required + + """ + + class Meta: + fields = [ + 'bom_file', + 'part', + 'clear_existing', + ] + + # These columns must be present + REQUIRED_COLUMNS = [ + 'quantity', + ] + + # We need at least one column to specify a "part" + PART_COLUMNS = [ + 'part', + 'part_id', + 'part_name', + 'part_ipn', + ] + + # These columns are "optional" + OPTIONAL_COLUMNS = [ + 'allow_variants', + 'inherited', + 'optional', + 'overage', + 'note', + 'reference', + ] + + def find_matching_column(self, col_name, columns): + + # Direct match + if col_name in columns: + return col_name + + col_name = col_name.lower().strip() + + for col in columns: + if col.lower().strip() == col_name: + return col + + # No match + return None + + def find_matching_data(self, row, col_name, columns): + """ + Extract data from the row, based on the "expected" column name + """ + + col_name = self.find_matching_column(col_name, columns) + + return row.get(col_name, None) + + bom_file = serializers.FileField( + label=_("BOM File"), + help_text=_("Select Bill of Materials file"), + required=True, + allow_empty_file=False, + ) + + def validate_bom_file(self, bom_file): + """ + Perform validation checks on the uploaded BOM file + """ + + self.filename = bom_file.name + + name, ext = os.path.splitext(bom_file.name) + + # Remove the leading . from the extension + ext = ext[1:] + + accepted_file_types = [ + 'xls', 'xlsx', + 'csv', 'tsv', + 'xml', + ] + + if ext not in accepted_file_types: + raise serializers.ValidationError(_("Unsupported file type")) + + # Impose a 50MB limit on uploaded BOM files + max_upload_file_size = 50 * 1024 * 1024 + + if bom_file.size > max_upload_file_size: + raise serializers.ValidationError(_("File is too large")) + + # Read file data into memory (bytes object) + data = bom_file.read() + + if ext in ['csv', 'tsv', 'xml']: + data = data.decode() + + # Convert to a tablib dataset (we expect headers) + self.dataset = tablib.Dataset().load(data, ext, headers=True) + + for header in self.REQUIRED_COLUMNS: + + match = self.find_matching_column(header, self.dataset.headers) + + if match is None: + raise serializers.ValidationError(_("Missing required column") + f": '{header}'") + + part_column_matches = {} + + part_match = False + + for col in self.PART_COLUMNS: + col_match = self.find_matching_column(col, self.dataset.headers) + + part_column_matches[col] = col_match + + if col_match is not None: + part_match = True + + if not part_match: + raise serializers.ValidationError(_("No part column found")) + + return bom_file + + def extract_data(self): + """ + Read individual rows out of the BOM file + """ + + rows = [] + + headers = self.dataset.headers + + level_column = self.find_matching_column('level', headers) + + for row in self.dataset.dict: + + """ + If the "level" column is specified, and this is not a top-level BOM item, ignore the row! + """ + if level_column is not None: + level = row.get('level', None) + + if level is not None: + try: + level = int(level) + if level != 1: + continue + except: + pass + + """ + Next, we try to "guess" the part, based on the provided data. + + A) If the part_id is supplied, use that! + B) If the part name and/or part_ipn are supplied, maybe we can use those? + """ + part_id = self.find_matching_data(row, 'part_id', headers) + part_name = self.find_matching_data(row, 'part_name', headers) + part_ipn = self.find_matching_data(row, 'part_ipn', headers) + + part = None + + if part_id is not None: + try: + part = Part.objects.get(pk=part_id) + except (ValueError, Part.DoesNotExist): + pass + + # Optionally, specify using field "part" + if part is None: + pk = self.find_matching_data(row, 'part', headers) + + if pk is not None: + try: + part = Part.objects.get(pk=pk) + except (ValueError, Part.DoesNotExist): + pass + + if part is None: + + if part_name is not None or part_ipn is not None: + queryset = Part.objects.all() + + if part_name is not None: + queryset = queryset.filter(name=part_name) + + if part_ipn is not None: + queryset = queryset.filter(IPN=part_ipn) + + # Only if we have a single direct match + if queryset.exists() and queryset.count() == 1: + part = queryset.first() + + row['part'] = part.pk if part is not None else None + + rows.append(row) + + return { + 'rows': rows, + 'headers': headers, + 'filename': self.filename, + } + + part = serializers.PrimaryKeyRelatedField(queryset=Part.objects.filter(assembly=True), required=True) + + clear_existing = serializers.BooleanField( + label=_("Clear Existing BOM"), + help_text=_("Delete existing BOM data first"), + ) + + def save(self): + + data = self.validated_data + + master_part = data['part'] + clear_existing = data['clear_existing'] + + if clear_existing: + + # Remove all existing BOM items + master_part.bom_items.all().delete() + + +class BomUploadSerializer(serializers.Serializer): + """ + Serializer for uploading a BOM against a specified part. + + A "BOM" is a set of BomItem objects which are to be validated together as a set + """ + + items = BomItemSerializer(many=True, required=True) + + def validate(self, data): + + items = data['items'] + + if len(items) == 0: + raise serializers.ValidationError(_("At least one BOM item is required")) + + data = super().validate(data) + + return data + + def save(self): + + data = self.validated_data + + items = data['items'] + + try: + with transaction.atomic(): + + for item in items: + + part = item['part'] + sub_part = item['sub_part'] + + # Ignore duplicate BOM items + if BomItem.objects.filter(part=part, sub_part=sub_part).exists(): + continue + + # Create a new BomItem object + BomItem.objects.create(**item) + + except Exception as e: + raise serializers.ValidationError(detail=serializers.as_serializer_error(e)) diff --git a/InvenTree/part/tasks.py b/InvenTree/part/tasks.py index a7c24b385b..a004b8a8cf 100644 --- a/InvenTree/part/tasks.py +++ b/InvenTree/part/tasks.py @@ -13,6 +13,7 @@ from common.models import NotificationEntry import InvenTree.helpers import InvenTree.tasks +from InvenTree.ready import isImportingData import part.models @@ -24,6 +25,10 @@ def notify_low_stock(part: part.models.Part): Notify users who have starred a part when its stock quantity falls below the minimum threshold """ + # Do not notify if we are importing data + if isImportingData(): + return + # Check if we have notified recently... delta = timedelta(days=1) diff --git a/InvenTree/part/templates/part/bom_upload/match_fields.html b/InvenTree/part/templates/part/bom_upload/match_fields.html deleted file mode 100644 index b09260cf46..0000000000 --- a/InvenTree/part/templates/part/bom_upload/match_fields.html +++ /dev/null @@ -1,99 +0,0 @@ -{% extends "part/bom_upload/upload_file.html" %} -{% load inventree_extras %} -{% load i18n %} -{% load static %} - -{% block form_alert %} -{% if missing_columns and missing_columns|length > 0 %} -
{{ row.errors.part }}
- {% endif %} -{{ row.errors.reference }}
- {% endif %} -{{ row.errors.quantity }}
- {% endif %} -{% blocktrans with step=wizard.steps.step1 count=wizard.steps.count %}Step {{step}} of {{count}}{% endblocktrans %} - {% if description %}- {{ description }}{% endif %}
- - - {% endblock form_buttons_bottom %} -{% trans "Part" %} | +{% trans "Quantity" %} | +{% trans "Reference" %} | +{% trans "Overage" %} | +{% trans "Allow Variants" %} | +{% trans "Inherited" %} | +{% trans "Optional" %} | +{% trans "Note" %} | ++ |
---|