From 6d3978ea28f05930ebde996186ad9bdee7f9686f Mon Sep 17 00:00:00 2001 From: Oliver Date: Wed, 19 Jul 2023 06:24:16 +1000 Subject: [PATCH] Add database model for defining custom units (#5268) * Add database model for defining custom units - Database model - DRF serializer - API endpoints * Add validation hook * Custom check for the 'definition' field * Add settings page for custom units - Table of units - Create / edit / delete buttons * Allow "unit" field to be empty - Not actually required for custom unit definition * Load custom unit definitions into global registry * Docs: add core concepts page(s) * Add some back links * Update docs * Add unit test for custom unit conversion * More unit testing * remove print statements * Add missing table rule --- InvenTree/InvenTree/conversion.py | 38 +++++-- InvenTree/InvenTree/tests.py | 33 ++++++- InvenTree/common/api.py | 25 +++++ .../common/migrations/0020_customunit.py | 22 +++++ InvenTree/common/models.py | 89 +++++++++++++++++ InvenTree/common/serializers.py | 15 +++ InvenTree/common/tests.py | 98 +++++++++++++++++-- .../InvenTree/settings/physical_units.html | 21 ++++ .../InvenTree/settings/settings.html | 1 + .../InvenTree/settings/settings_staff_js.html | 76 ++++++++++++++ .../templates/InvenTree/settings/sidebar.html | 2 + InvenTree/users/models.py | 1 + docs/docs/{ => concepts}/terminology.md | 0 docs/docs/concepts/units.md | 41 ++++++++ docs/docs/part/parameter.md | 8 +- docs/docs/part/part.md | 3 + docs/mkdocs.yml | 10 +- 17 files changed, 458 insertions(+), 25 deletions(-) create mode 100644 InvenTree/common/migrations/0020_customunit.py create mode 100644 InvenTree/templates/InvenTree/settings/physical_units.html rename docs/docs/{ => concepts}/terminology.md (100%) create mode 100644 docs/docs/concepts/units.md diff --git a/InvenTree/InvenTree/conversion.py b/InvenTree/InvenTree/conversion.py index 67e18dc0e7..583fb98245 100644 --- a/InvenTree/InvenTree/conversion.py +++ b/InvenTree/InvenTree/conversion.py @@ -20,9 +20,9 @@ def get_unit_registry(): # Cache the unit registry for speedier access if _unit_registry is None: - reload_unit_registry() - - return _unit_registry + return reload_unit_registry() + else: + return _unit_registry def reload_unit_registry(): @@ -36,20 +36,38 @@ def reload_unit_registry(): global _unit_registry - _unit_registry = pint.UnitRegistry() + _unit_registry = None + + reg = pint.UnitRegistry() # Define some "standard" additional units - _unit_registry.define('piece = 1') - _unit_registry.define('each = 1 = ea') - _unit_registry.define('dozen = 12 = dz') - _unit_registry.define('hundred = 100') - _unit_registry.define('thousand = 1000') + reg.define('piece = 1') + reg.define('each = 1 = ea') + reg.define('dozen = 12 = dz') + reg.define('hundred = 100') + reg.define('thousand = 1000') - # TODO: Allow for custom units to be defined in the database + # Allow for custom units to be defined in the database + try: + from common.models import CustomUnit + + for cu in CustomUnit.objects.all(): + try: + reg.define(cu.fmt_string()) + except Exception as e: + logger.error(f'Failed to load custom unit: {cu.fmt_string()} - {e}') + + # Once custom units are loaded, save registry + _unit_registry = reg + + except Exception as e: + logger.error(f'Failed to load custom units: {e}') dt = time.time() - t_start logger.debug(f'Loaded unit registry in {dt:.3f}s') + return reg + def convert_physical_value(value: str, unit: str = None): """Validate that the provided value is a valid physical quantity. diff --git a/InvenTree/InvenTree/tests.py b/InvenTree/InvenTree/tests.py index 6b979fed51..1bc9ae5680 100644 --- a/InvenTree/InvenTree/tests.py +++ b/InvenTree/InvenTree/tests.py @@ -16,6 +16,7 @@ from django.core.exceptions import ValidationError from django.test import TestCase, override_settings from django.urls import reverse +import pint.errors from djmoney.contrib.exchange.exceptions import MissingRate from djmoney.contrib.exchange.models import Rate, convert_money from djmoney.money import Money @@ -26,7 +27,7 @@ import InvenTree.format import InvenTree.helpers import InvenTree.helpers_model import InvenTree.tasks -from common.models import InvenTreeSetting +from common.models import CustomUnit, InvenTreeSetting from common.settings import currency_codes from InvenTree.sanitizer import sanitize_svg from InvenTree.unit_test import InvenTreeTestCase @@ -76,6 +77,36 @@ class ConversionTest(TestCase): with self.assertRaises(ValidationError): InvenTree.conversion.convert_physical_value(val) + def test_custom_units(self): + """Tests for custom unit conversion""" + + # Start with an empty set of units + CustomUnit.objects.all().delete() + InvenTree.conversion.reload_unit_registry() + + # Ensure that the custom unit does *not* exist to start with + reg = InvenTree.conversion.get_unit_registry() + + with self.assertRaises(pint.errors.UndefinedUnitError): + reg['hpmm'] + + # Create a new custom unit + CustomUnit.objects.create( + name='fanciful_unit', + definition='henry / mm', + symbol='hpmm', + ) + + # Reload registry + reg = InvenTree.conversion.get_unit_registry() + + # Ensure that the custom unit is now available + reg['hpmm'] + + # Convert some values + q = InvenTree.conversion.convert_physical_value('1 hpmm', 'henry / km') + self.assertEqual(q.magnitude, 1000000) + class ValidatorTest(TestCase): """Simple tests for custom field validators.""" diff --git a/InvenTree/common/api.py b/InvenTree/common/api.py index abde58501f..3e17af11c9 100644 --- a/InvenTree/common/api.py +++ b/InvenTree/common/api.py @@ -486,6 +486,23 @@ class ProjectCodeDetail(RetrieveUpdateDestroyAPI): permission_classes = [permissions.IsAuthenticated, IsStaffOrReadOnly] +class CustomUnitList(ListCreateAPI): + """List view for custom units""" + + queryset = common.models.CustomUnit.objects.all() + serializer_class = common.serializers.CustomUnitSerializer + permission_classes = [permissions.IsAuthenticated, IsStaffOrReadOnly] + filter_backends = SEARCH_ORDER_FILTER + + +class CustomUnitDetail(RetrieveUpdateDestroyAPI): + """Detail view for a particular custom unit""" + + queryset = common.models.CustomUnit.objects.all() + serializer_class = common.serializers.CustomUnitSerializer + permission_classes = [permissions.IsAuthenticated, IsStaffOrReadOnly] + + class FlagList(ListAPI): """List view for feature flags.""" @@ -554,6 +571,14 @@ common_api_urls = [ re_path(r'^.*$', ProjectCodeList.as_view(), name='api-project-code-list'), ])), + # Custom physical units + re_path(r'^units/', include([ + path(r'/', include([ + re_path(r'^.*$', CustomUnitDetail.as_view(), name='api-custom-unit-detail'), + ])), + re_path(r'^.*$', CustomUnitList.as_view(), name='api-custom-unit-list'), + ])), + # Currencies re_path(r'^currency/', include([ re_path(r'^exchange/', CurrencyExchangeView.as_view(), name='api-currency-exchange'), diff --git a/InvenTree/common/migrations/0020_customunit.py b/InvenTree/common/migrations/0020_customunit.py new file mode 100644 index 0000000000..500d34c683 --- /dev/null +++ b/InvenTree/common/migrations/0020_customunit.py @@ -0,0 +1,22 @@ +# Generated by Django 3.2.20 on 2023-07-18 11:38 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('common', '0019_projectcode_metadata'), + ] + + operations = [ + migrations.CreateModel( + name='CustomUnit', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(help_text='Unit name', max_length=50, unique=True, verbose_name='Name')), + ('symbol', models.CharField(blank=True, help_text='Optional unit symbol', max_length=10, unique=True, verbose_name='Symbol')), + ('definition', models.CharField(help_text='Unit definition', max_length=50, verbose_name='Definition')), + ], + ), + ] diff --git a/InvenTree/common/models.py b/InvenTree/common/models.py index 2addc8c8cc..83933574e9 100644 --- a/InvenTree/common/models.py +++ b/InvenTree/common/models.py @@ -30,7 +30,9 @@ from django.core.exceptions import AppRegistryNotReady, ValidationError from django.core.validators import (MaxValueValidator, MinValueValidator, URLValidator) from django.db import models, transaction +from django.db.models.signals import post_delete, post_save from django.db.utils import IntegrityError, OperationalError +from django.dispatch.dispatcher import receiver from django.urls import reverse from django.utils.timezone import now from django.utils.translation import gettext_lazy as _ @@ -2794,3 +2796,90 @@ class NotesImage(models.Model): user = models.ForeignKey(User, on_delete=models.SET_NULL, null=True, blank=True) date = models.DateTimeField(auto_now_add=True) + + +class CustomUnit(models.Model): + """Model for storing custom physical unit definitions + + Model Attributes: + name: Name of the unit + definition: Definition of the unit + symbol: Symbol for the unit (e.g. 'm' for 'metre') (optional) + + Refer to the pint documentation for further information on unit definitions. + https://pint.readthedocs.io/en/stable/advanced/defining.html + """ + + def fmt_string(self): + """Construct a unit definition string e.g. 'dog_year = 52 * day = dy'""" + fmt = f'{self.name} = {self.definition}' + + if self.symbol: + fmt += f' = {self.symbol}' + + return fmt + + def clean(self): + """Validate that the provided custom unit is indeed valid""" + + super().clean() + + from InvenTree.conversion import get_unit_registry + + registry = get_unit_registry() + + # Check that the 'name' field is valid + self.name = self.name.strip() + + # Cannot be zero length + if not self.name.isidentifier(): + raise ValidationError({ + 'name': _('Unit name must be a valid identifier') + }) + + self.definition = self.definition.strip() + + # Check that the 'definition' is valid, by itself + try: + registry.Quantity(self.definition) + except Exception as exc: + raise ValidationError({ + 'definition': str(exc) + }) + + # Finally, test that the entire custom unit definition is valid + try: + registry.define(self.fmt_string()) + except Exception as exc: + raise ValidationError(str(exc)) + + name = models.CharField( + max_length=50, + verbose_name=_('Name'), + help_text=_('Unit name'), + unique=True, blank=False, + ) + + symbol = models.CharField( + max_length=10, + verbose_name=_('Symbol'), + help_text=_('Optional unit symbol'), + unique=True, blank=True, + ) + + definition = models.CharField( + max_length=50, + verbose_name=_('Definition'), + help_text=_('Unit definition'), + blank=False, + ) + + +@receiver(post_save, sender=CustomUnit, dispatch_uid='custom_unit_saved') +@receiver(post_delete, sender=CustomUnit, dispatch_uid='custom_unit_deleted') +def after_custom_unit_updated(sender, instance, **kwargs): + """Callback when a custom unit is updated or deleted""" + + # Force reload of the unit registry + from InvenTree.conversion import reload_unit_registry + reload_unit_registry() diff --git a/InvenTree/common/serializers.py b/InvenTree/common/serializers.py index 4cb6a5dd45..4c1d0b5a54 100644 --- a/InvenTree/common/serializers.py +++ b/InvenTree/common/serializers.py @@ -296,3 +296,18 @@ class FlagSerializer(serializers.Serializer): data['conditions'] = self.instance[instance] return data + + +class CustomUnitSerializer(InvenTreeModelSerializer): + """DRF serializer for CustomUnit model.""" + + class Meta: + """Meta options for CustomUnitSerializer.""" + + model = common_models.CustomUnit + fields = [ + 'pk', + 'name', + 'symbol', + 'definition', + ] diff --git a/InvenTree/common/tests.py b/InvenTree/common/tests.py index 740de2e419..d9de170d07 100644 --- a/InvenTree/common/tests.py +++ b/InvenTree/common/tests.py @@ -22,9 +22,10 @@ from plugin import registry from plugin.models import NotificationUserSetting from .api import WebhookView -from .models import (ColorTheme, InvenTreeSetting, InvenTreeUserSetting, - NotesImage, NotificationEntry, NotificationMessage, - ProjectCode, WebhookEndpoint, WebhookMessage) +from .models import (ColorTheme, CustomUnit, InvenTreeSetting, + InvenTreeUserSetting, NotesImage, NotificationEntry, + NotificationMessage, ProjectCode, WebhookEndpoint, + WebhookMessage) CONTENT_TYPE_JSON = 'application/json' @@ -1061,7 +1062,7 @@ class NotesImageTest(InvenTreeAPITestCase): image.save(output, format='PNG') contents = output.getvalue() - response = self.post( + self.post( reverse('api-notes-image-list'), data={ 'image': SimpleUploadedFile('test.png', contents, content_type='image/png'), @@ -1070,8 +1071,6 @@ class NotesImageTest(InvenTreeAPITestCase): expected_code=201 ) - print(response.data) - # Check that a new file has been created self.assertEqual(NotesImage.objects.count(), n + 1) @@ -1184,3 +1183,90 @@ class ProjectCodesTest(InvenTreeAPITestCase): }, expected_code=403 ) + + +class CustomUnitAPITest(InvenTreeAPITestCase): + """Unit tests for the CustomUnit API""" + + @property + def url(self): + """Return the API endpoint for the CustomUnit list""" + return reverse('api-custom-unit-list') + + @classmethod + def setUpTestData(cls): + """Construct some initial test fixture data""" + super().setUpTestData() + + units = [ + CustomUnit(name='metres_per_amp', definition='meter / ampere', symbol='m/A'), + CustomUnit(name='hectares_per_second', definition='hectares per second', symbol='ha/s'), + ] + + CustomUnit.objects.bulk_create(units) + + def test_list(self): + """Test API list functionality""" + + response = self.get(self.url, expected_code=200) + self.assertEqual(len(response.data), CustomUnit.objects.count()) + + def test_edit(self): + """Test edit permissions for CustomUnit model""" + + unit = CustomUnit.objects.first() + + # Try to edit without permission + self.user.is_staff = False + self.user.save() + + self.patch( + reverse('api-custom-unit-detail', kwargs={'pk': unit.pk}), + { + 'name': 'new_unit_name', + }, + expected_code=403 + ) + + # Ok, what if we have permission? + self.user.is_staff = True + self.user.save() + + self.patch( + reverse('api-custom-unit-detail', kwargs={'pk': unit.pk}), + { + 'name': 'new_unit_name', + }, + # expected_code=200 + ) + + unit.refresh_from_db() + self.assertEqual(unit.name, 'new_unit_name') + + def test_validation(self): + """Test that validation works as expected""" + + unit = CustomUnit.objects.first() + + self.user.is_staff = True + self.user.save() + + # Test invalid 'name' values (must be valid identifier) + invalid_name_values = [ + '1', + '1abc', + 'abc def', + 'abc-def', + 'abc.def', + ] + + url = reverse('api-custom-unit-detail', kwargs={'pk': unit.pk}) + + for name in invalid_name_values: + self.patch( + url, + { + 'name': name, + }, + expected_code=400 + ) diff --git a/InvenTree/templates/InvenTree/settings/physical_units.html b/InvenTree/templates/InvenTree/settings/physical_units.html new file mode 100644 index 0000000000..2be0f68dbc --- /dev/null +++ b/InvenTree/templates/InvenTree/settings/physical_units.html @@ -0,0 +1,21 @@ +{% extends "panel.html" %} + +{% load i18n %} +{% load inventree_extras %} + +{% block label %}units{% endblock label %} + +{% block heading %}{% trans "Physical Units" %}{% endblock heading %} +{% block actions %} + +{% endblock actions %} + +{% block content %} + + +
+ +{% endblock content %} diff --git a/InvenTree/templates/InvenTree/settings/settings.html b/InvenTree/templates/InvenTree/settings/settings.html index 6a90d5e8c5..b06d7d607c 100644 --- a/InvenTree/templates/InvenTree/settings/settings.html +++ b/InvenTree/templates/InvenTree/settings/settings.html @@ -32,6 +32,7 @@ {% include "InvenTree/settings/login.html" %} {% include "InvenTree/settings/barcode.html" %} {% include "InvenTree/settings/project_codes.html" %} +{% include "InvenTree/settings/physical_units.html" %} {% include "InvenTree/settings/notifications.html" %} {% include "InvenTree/settings/label.html" %} {% include "InvenTree/settings/report.html" %} diff --git a/InvenTree/templates/InvenTree/settings/settings_staff_js.html b/InvenTree/templates/InvenTree/settings/settings_staff_js.html index aa1b182151..7d8931d79c 100644 --- a/InvenTree/templates/InvenTree/settings/settings_staff_js.html +++ b/InvenTree/templates/InvenTree/settings/settings_staff_js.html @@ -52,6 +52,82 @@ onPanelLoad('pricing', function() { }); }); +// Javascript for units panel +onPanelLoad('units', function() { + + console.log("units panel"); + + // Construct the "units" table + $('#physical-units-table').bootstrapTable({ + url: '{% url "api-custom-unit-list" %}', + search: true, + columns: [ + { + field: 'name', + title: '{% trans "Name" %}', + }, + { + field: 'definition', + title: '{% trans "Definition" %}', + }, + { + field: 'symbol', + title: '{% trans "Symbol" %}', + formatter: function(value, row) { + let html = value; + let buttons = ''; + + buttons += makeEditButton('button-units-edit', row.pk, '{% trans "Edit" %}'); + buttons += makeDeleteButton('button-units-delete', row.pk, '{% trans "Delete" %}'); + + html += wrapButtons(buttons); + return html; + } + }, + ] + }); + + // Callback to edit a custom unit + $('#physical-units-table').on('click', '.button-units-edit', function() { + let pk = $(this).attr('pk'); + + constructForm(`{% url "api-custom-unit-list" %}${pk}/`, { + title: '{% trans "Edit Custom Unit" %}', + fields: { + name: {}, + definition: {}, + symbol: {}, + }, + refreshTable: '#physical-units-table', + }); + }); + + // Callback to delete a custom unit + $('#physical-units-table').on('click', '.button-units-delete', function() { + let pk = $(this).attr('pk'); + + constructForm(`{% url "api-custom-unit-list" %}${pk}/`, { + title: '{% trans "Delete Custom Unit" %}', + method: 'DELETE', + refreshTable: '#physical-units-table', + }); + }); + + // Callback to create a new custom unit + $('#custom-unit-create').click(function() { + constructForm('{% url "api-custom-unit-list" %}', { + fields: { + name: {}, + definition: {}, + symbol: {}, + }, + title: '{% trans "New Custom Unit" %}', + method: 'POST', + refreshTable: '#physical-units-table', + }); + }); +}); + // Javascript for project codes panel onPanelLoad('project-codes', function() { diff --git a/InvenTree/templates/InvenTree/settings/sidebar.html b/InvenTree/templates/InvenTree/settings/sidebar.html index 177a4e0a9b..5d29bdc9c8 100644 --- a/InvenTree/templates/InvenTree/settings/sidebar.html +++ b/InvenTree/templates/InvenTree/settings/sidebar.html @@ -32,6 +32,8 @@ {% include "sidebar_item.html" with label='barcodes' text=text icon="fa-qrcode" %} {% trans "Project Codes" as text %} {% include "sidebar_item.html" with label='project-codes' text=text icon="fa-list" %} +{% trans "Physical Units" as text %} +{% include "sidebar_item.html" with label='units' text=text icon="fa-balance-scale" %} {% trans "Notifications" as text %} {% include "sidebar_item.html" with label='global-notifications' text=text icon="fa-bell" %} {% trans "Pricing" as text %} diff --git a/InvenTree/users/models.py b/InvenTree/users/models.py index 5be59452ee..91e7cd40c0 100644 --- a/InvenTree/users/models.py +++ b/InvenTree/users/models.py @@ -188,6 +188,7 @@ class RuleSet(models.Model): # Models which currently do not require permissions 'common_colortheme', + 'common_customunit', 'common_inventreesetting', 'common_inventreeusersetting', 'common_notificationentry', diff --git a/docs/docs/terminology.md b/docs/docs/concepts/terminology.md similarity index 100% rename from docs/docs/terminology.md rename to docs/docs/concepts/terminology.md diff --git a/docs/docs/concepts/units.md b/docs/docs/concepts/units.md new file mode 100644 index 0000000000..fccaa7d7a8 --- /dev/null +++ b/docs/docs/concepts/units.md @@ -0,0 +1,41 @@ +--- +title: Physical Units +--- + +## Physical Units + +Support for real-world "physical" units of measure is implemented using the [pint](https://pint.readthedocs.io/en/stable/) Python library. This library provides the following core functions: + +- Ensures consistent use of real units for your inventory management +- Convert between compatible units of measure from suppliers +- Enforce use of compatible units when creating part parameters +- Enable custom units as required + +## Unit Support + +Physical units are supported by the following InvenTree subsystems: + +### Part + +The [unit of measure](../part/part.md#units-of-measure) field for the [Part](../part/part.md) model uses real-world units. + +### Supplier Part + +The [supplier part](../part/part/#supplier-parts) model uses real-world units to convert between supplier part quantities and internal stock quantities. Unit conversion rules ensure that only compatible unit types can be supplied + +### Part Parameter + +The [part parameter template](../part/parameter.md#parameter-templates) model can specify units of measure, and part parameters can be specified against these templates with compatible units + +## Custom Units + +Out of the box, the Pint library provides a wide range of units for use. However, it may not be sufficient for a given application. In such cases, custom units can be easily defined to meet custom requirements. + +Custom units can be defined to provide a new physical quantity, link existing units together, or simply provide an alias for an existing unit. + +!!! tip "More Info" + For further information, refer to the [pint documentation](https://pint.readthedocs.io/en/stable/advanced/defining.html) regarding custom unit definition + +### Create Custom Units + +To view, edit and create custom units, locate the *Physical Units* tab in the [settings panel](../settings/global.md). diff --git a/docs/docs/part/parameter.md b/docs/docs/part/parameter.md index a2047b2767..1e7028cc0b 100644 --- a/docs/docs/part/parameter.md +++ b/docs/docs/part/parameter.md @@ -77,13 +77,13 @@ The parametric parts table allows the returned parts to be sorted by particular ## Parameter Units -The *units* field (which is defined against a [parameter template](#parameter-templates)) defines the base unit of that template. Any parameters which are created against that unit *must* be specified in compatible units. Unit conversion is implemented using the [pint](https://pint.readthedocs.io/en/stable/) Python library. This conversion library is used to perform two main functions: - -- Enforce use of compatible units when creating part parameters -- Perform conversion to the base template unit +The *units* field (which is defined against a [parameter template](#parameter-templates)) defines the base unit of that template. Any parameters which are created against that unit *must* be specified in compatible units. The in-built conversion functionality means that parameter values can be input in different dimensions - *as long as the dimension is compatible with the base template units*. +!!! info "Read Mode" + Read more about how InvenTree supports [physical units of measure](../concepts/units.md) + ### Incompatible Units If a part parameter is created with a value which is incompatible with the units specified for the template, it will be rejected: diff --git a/docs/docs/part/part.md b/docs/docs/part/part.md index 6baa6f7a74..acb90571df 100644 --- a/docs/docs/part/part.md +++ b/docs/docs/part/part.md @@ -81,6 +81,9 @@ By default, all parts are *Active*. Marking a part as inactive means it is not a Each type of part can define a custom "unit of measure" which is a standardized unit which is used to track quantities for a particular part. By default, the "unit of measure" for each part is blank, which means that each part is tracked in dimensionless quantities of "pieces". +!!! info "Read More" + Read more on how InvenTree supports [physical units of measure](../concepts/units.md) + ### Physical Units It is possible to track parts using physical quantity values, such as *metres* or *litres*. For example, it would make sense to track a "wire" in units of "metres": diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 38aef90214..1b6a0525d0 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -70,15 +70,17 @@ nav: - InvenTree: - InvenTree: index.md - Features: features.md - - Release Notes: releases/release_notes.md - FAQ: faq.md - - Credits: credits.md - - Privacy: privacy.md - - Terminology: terminology.md + - Core Concepts: + - Terminology: concepts/terminology.md + - Physical Units: concepts/units.md - Development: - Getting started: develop/starting.md - Contributing: develop/contributing.md - Devcontainer: develop/devcontainer.md + - Credits: credits.md + - Privacy: privacy.md + - Release Notes: releases/release_notes.md - Install: - Introduction: start/intro.md - Configuration: start/config.md