inventree/InvenTree
2
0
Fork 0
mirror of https://github.com/inventree/InvenTree.git synced 2025-06-18 04:55:44 +00:00
Code Activity

[Refactor] Serial generation (#8246)

Browse Source 
* Refactor "update_serial_number" method

* Refactor serial number validation

- Query is much more efficient now
- Does not have to check each serial number individually
- Makes use of existing Part class method

* Refactor creation of multiple stock items

* Fix for singular item creation

* Refactor serializeStock method:

- Push "rebuild tree" to background worker
- Use bulk_create actions

* Refactor createion of serialized build outputs

* Prevent 1+N DB hits

* Cleanup

* Cleanup

* Reinstate serial number checks

* Add limit for serial number extraction

* Fix cache config

* Revert cache settings

* Fix for unit tests

* Playwright tests

* Bug fix

* Force False cookie mode in testing

* Revert aria-label for PanelGroup items

- No longer works as expected with playwright locators

* Fix playwright vtest

* Further updates

* Playwright test adjustments

* Remove duplicate locator
This commit is contained in:
Oliver
2024-10-12 10:08:57 +11:00
committed by GitHub
parent f77c8a5b5b
commit 7443d21854
21 changed files with 527 additions and 307 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/backend/InvenTree/InvenTree/helpers.py
View File

@ -487,6 +487,11 @@ def extract_serial_numbers(input_string, expected_quantity: int, starting_value=
except ValueError:
raise ValidationError([_('Invalid quantity provided')])
if expected_quantity > 1000:
raise ValidationError({
'quantity': [_('Cannot serialize more than 1000 items at once')]
})
input_string = str(input_string).strip() if input_string else ''
if len(input_string) == 0:

2
src/backend/InvenTree/InvenTree/settings.py
View File

@ -1101,7 +1101,7 @@ COOKIE_MODE = (
# Valid modes (as per the django settings documentation)
valid_cookie_modes = ['lax', 'strict', 'none']
if not DEBUG and COOKIE_MODE in valid_cookie_modes:
if not DEBUG and not TESTING and COOKIE_MODE in valid_cookie_modes:
# Set the cookie mode (in production mode only)
COOKIE_MODE = COOKIE_MODE.capitalize()
else:

10
src/backend/InvenTree/InvenTree/unit_test.py
View File

@ -263,7 +263,9 @@ class InvenTreeAPITestCase(ExchangeRateMixin, UserMixin, APITestCase):
MAX_QUERY_TIME = 7.5
@contextmanager
def assertNumQueriesLessThan(self, value, using='default', verbose=False, url=None):
def assertNumQueriesLessThan(
self, value, using='default', verbose=False, url=None, log_to_file=False
):
"""Context manager to check that the number of queries is less than a certain value.
Example:
@ -281,6 +283,12 @@ class InvenTreeAPITestCase(ExchangeRateMixin, UserMixin, APITestCase):
f'Query count exceeded at {url}: Expected < {value} queries, got {n}'
) # pragma: no cover
# Useful for debugging, disabled by default
if log_to_file:
with open('queries.txt', 'w', encoding='utf-8') as f:
for q in context.captured_queries:
f.write(str(q['sql']) + '\n')
if verbose and n >= value:
msg = f'\r\n{json.dumps(context.captured_queries, indent=4)}' # pragma: no cover
else:

137
src/backend/InvenTree/build/models.py
View File

@ -834,6 +834,16 @@ class Build(
location: Override location
auto_allocate: Automatically allocate stock with matching serial numbers
"""
trackable_parts = self.part.get_trackable_parts()
# Create (and cache) a map of valid parts for allocation
valid_parts = {}
for bom_item in trackable_parts:
parts = bom_item.get_valid_parts_for_allocation()
valid_parts[bom_item.pk] = list([part.pk for part in parts])
user = kwargs.get('user', None)
batch = kwargs.get('batch', self.batch)
location = kwargs.get('location', None)
@ -843,81 +853,51 @@ class Build(
if location is None:
location = self.destination or self.part.get_default_location()
"""
Determine if we can create a single output (with quantity > 0),
or multiple outputs (with quantity = 1)
"""
def _add_tracking_entry(output, user):
"""Helper function to add a tracking entry to the newly created output"""
deltas = {
'quantity': float(output.quantity),
'buildorder': self.pk,
}
if output.batch:
deltas['batch'] = output.batch
if output.serial:
deltas['serial'] = output.serial
if output.location:
deltas['location'] = output.location.pk
output.add_tracking_entry(StockHistoryCode.BUILD_OUTPUT_CREATED, user, deltas)
multiple = False
# Serial numbers are provided? We need to split!
if serials:
multiple = True
# BOM has trackable parts, so we must split!
if self.part.has_trackable_parts:
multiple = True
if multiple:
# We are generating multiple serialized outputs
if serials or self.part.has_trackable_parts:
"""Create multiple build outputs with a single quantity of 1."""
# Quantity *must* be an integer at this point!
quantity = int(quantity)
# Create tracking entries for each item
tracking = []
allocations = []
for ii in range(quantity):
outputs = stock.models.StockItem._create_serial_numbers(
serials,
part=self.part,
build=self,
batch=batch,
is_building=True
)
if serials:
serial = serials[ii]
else:
serial = None
for output in outputs:
# Generate a new historical tracking entry
if entry := output.add_tracking_entry(
StockHistoryCode.BUILD_OUTPUT_CREATED,
user,
deltas={
'quantity': 1,
'buildorder': self.pk,
'batch': output.batch,
'serial': output.serial,
'location': location.pk if location else None
},
commit=False
):
tracking.append(entry)
output = stock.models.StockItem.objects.create(
quantity=1,
location=location,
part=self.part,
build=self,
batch=batch,
serial=serial,
is_building=True,
)
# Auto-allocate stock based on serial number
if auto_allocate:
allocations = []
_add_tracking_entry(output, user)
if auto_allocate and serial is not None:
# Get a list of BomItem objects which point to "trackable" parts
for bom_item in self.part.get_trackable_parts():
parts = bom_item.get_valid_parts_for_allocation()
for bom_item in trackable_parts:
valid_part_ids = valid_parts.get(bom_item.pk, [])
items = stock.models.StockItem.objects.filter(
part__in=parts,
serial=str(serial),
part__pk__in=valid_part_ids,
serial=output.serial,
quantity=1,
).filter(stock.models.StockItem.IN_STOCK_FILTER)
"""
Test if there is a matching serial number!
"""
if items.exists() and items.count() == 1:
stock_item = items[0]
@ -929,15 +909,23 @@ class Build(
)
# Allocate the stock items against the BuildLine
BuildItem.objects.create(
build_line=build_line,
stock_item=stock_item,
quantity=1,
install_into=output,
allocations.append(
BuildItem(
build_line=build_line,
stock_item=stock_item,
quantity=1,
install_into=output,
)
)
except BuildLine.DoesNotExist:
pass
# Bulk create tracking entries
stock.models.StockItemTracking.objects.bulk_create(tracking)
# Generate stock allocations
BuildItem.objects.bulk_create(allocations)
else:
"""Create a single build output of the given quantity."""
@ -950,7 +938,16 @@ class Build(
is_building=True
)
_add_tracking_entry(output, user)
output.add_tracking_entry(
StockHistoryCode.BUILD_OUTPUT_CREATED,
user,
deltas={
'quantity': quantity,
'buildorder': self.pk,
'batch': batch,
'location': location.pk if location else None
}
)
if self.status == BuildStatus.PENDING:
self.status = BuildStatus.PRODUCTION.value

6
src/backend/InvenTree/build/serializers.py
View File

@ -404,11 +404,7 @@ class BuildOutputCreateSerializer(serializers.Serializer):
})
# Check for conflicting serial numbesr
existing = []
for serial in self.serials:
if not part.validate_serial_number(serial):
existing.append(serial)
existing = part.find_conflicting_serial_numbers(self.serials)
if len(existing) > 0:

30
src/backend/InvenTree/part/models.py
View File

@ -833,12 +833,38 @@ class Part(
# This serial number is perfectly valid
return True
def find_conflicting_serial_numbers(self, serials: list):
def find_conflicting_serial_numbers(self, serials: list) -> list:
"""For a provided list of serials, return a list of those which are conflicting."""
from part.models import Part
from stock.models import StockItem
conflicts = []
# First, check for raw conflicts based on efficient database queries
if get_global_setting('SERIAL_NUMBER_GLOBALLY_UNIQUE', False):
# Serial number must be unique across *all* parts
parts = Part.objects.all()
else:
# Serial number must only be unique across this part "tree"
parts = Part.objects.filter(tree_id=self.tree_id)
items = StockItem.objects.filter(part__in=parts, serial__in=serials)
items = items.order_by('serial_int', 'serial')
for item in items:
conflicts.append(item.serial)
for serial in serials:
if not self.validate_serial_number(serial, part=self):
if serial in conflicts:
# Already found a conflict, no need to check further
continue
try:
self.validate_serial_number(
serial, raise_error=True, check_duplicates=False
)
except ValidationError:
# Serial number is invalid (as determined by plugin)
conflicts.append(serial)
return conflicts

80
src/backend/InvenTree/stock/api.py
View File

@ -922,12 +922,12 @@ class StockList(DataExportViewMixin, ListCreateDestroyAPIView):
data.update(self.clean_data(request.data))
quantity = data.get('quantity', None)
location = data.get('location', None)
if quantity is None:
raise ValidationError({'quantity': _('Quantity is required')})
try:
Part.objects.prefetch_related(None)
part = Part.objects.get(pk=data.get('part', None))
except (ValueError, Part.DoesNotExist):
raise ValidationError({'part': _('Valid part must be supplied')})
@ -951,15 +951,13 @@ class StockList(DataExportViewMixin, ListCreateDestroyAPIView):
serials = None
# Check if a set of serial numbers was provided
serial_numbers = data.get('serial_numbers', '')
serial_numbers = data.pop('serial_numbers', '')
# Check if the supplier_part has a package size defined, which is not 1
if 'supplier_part' in data and data['supplier_part'] is not None:
if supplier_part_id := data.get('supplier_part', None):
try:
supplier_part = SupplierPart.objects.get(
pk=data.get('supplier_part', None)
)
except (ValueError, SupplierPart.DoesNotExist):
supplier_part = SupplierPart.objects.get(pk=supplier_part_id)
except Exception:
raise ValidationError({
'supplier_part': _('The given supplier part does not exist')
})
@ -988,8 +986,7 @@ class StockList(DataExportViewMixin, ListCreateDestroyAPIView):
# Now remove the flag from data, so that it doesn't interfere with saving
# Do this regardless of results above
if 'use_pack_size' in data:
data.pop('use_pack_size')
data.pop('use_pack_size', None)
# Assign serial numbers for a trackable part
if serial_numbers:
@ -1011,22 +1008,20 @@ class StockList(DataExportViewMixin, ListCreateDestroyAPIView):
invalid = []
errors = []
for serial in serials:
try:
part.validate_serial_number(serial, raise_error=True)
except DjangoValidationError as exc:
# Catch raised error to extract specific error information
invalid.append(serial)
try:
invalid = part.find_conflicting_serial_numbers(serials)
except DjangoValidationError as exc:
errors.append(exc.message)
if exc.message not in errors:
errors.append(exc.message)
if len(errors) > 0:
if len(invalid) > 0:
msg = _('The following serial numbers already exist or are invalid')
msg += ' : '
msg += ','.join([str(e) for e in invalid])
raise ValidationError({'serial_numbers': [*errors, msg]})
errors.append(msg)
if len(errors) > 0:
raise ValidationError({'serial_numbers': errors})
except DjangoValidationError as e:
raise ValidationError({
@ -1043,34 +1038,43 @@ class StockList(DataExportViewMixin, ListCreateDestroyAPIView):
serializer.is_valid(raise_exception=True)
with transaction.atomic():
# Create an initial StockItem object
item = serializer.save()
if serials:
# Assign the first serial number to the "master" item
item.serial = serials[0]
# Create multiple serialized StockItem objects
items = StockItem._create_serial_numbers(
serials, **serializer.validated_data
)
# Save the item (with user information)
item.save(user=user)
# Next, bulk-create stock tracking entries for the newly created items
tracking = []
if serials:
for serial in serials[1:]:
# Create a duplicate stock item with the next serial number
item.pk = None
item.serial = serial
for item in items:
if entry := item.add_tracking_entry(
StockHistoryCode.CREATED,
user,
deltas={'status': item.status},
location=location,
quantity=float(item.quantity),
commit=False,
):
tracking.append(entry)
item.save(user=user)
StockItemTracking.objects.bulk_create(tracking)
response_data = {'quantity': quantity, 'serial_numbers': serials}
else:
# Create a single StockItem object
# Note: This automatically creates a tracking entry
item = serializer.save()
item.save(user=user)
response_data = serializer.data
return Response(
response_data,
status=status.HTTP_201_CREATED,
headers=self.get_success_headers(serializer.data),
)
return Response(
response_data,
status=status.HTTP_201_CREATED,
headers=self.get_success_headers(serializer.data),
)
def get_queryset(self, *args, **kwargs):
"""Annotate queryset before returning."""

220
src/backend/InvenTree/stock/models.py
View File

@ -12,7 +12,7 @@ from django.contrib.auth.models import User
from django.core.exceptions import ValidationError
from django.core.validators import MinValueValidator
from django.db import models, transaction
from django.db.models import Q, Sum
from django.db.models import Q, QuerySet, Sum
from django.db.models.functions import Coalesce
from django.db.models.signals import post_delete, post_save, pre_delete
from django.db.utils import IntegrityError, OperationalError
@ -33,6 +33,7 @@ import InvenTree.ready
import InvenTree.tasks
import report.mixins
import report.models
import stock.tasks
from build import models as BuildModels
from common.icons import validate_icon
from common.settings import get_global_setting
@ -459,6 +460,97 @@ class StockItem(
& Q(expiry_date__lt=InvenTree.helpers.current_date())
)
@classmethod
def _create_serial_numbers(cls, serials: list, **kwargs) -> QuerySet:
"""Create multiple stock items with the provided serial numbers.
Arguments:
serials: List of serial numbers to create
**kwargs: Additional keyword arguments to pass to the StockItem creation function
Returns:
QuerySet: The created StockItem objects
raises:
ValidationError: If any of the provided serial numbers are invalid
This method uses bulk_create to create multiple StockItem objects in a single query,
which is much more efficient than creating them one-by-one.
However, it does not perform any validation checks on the provided serial numbers,
and also does not generate any "stock tracking entries".
Note: This is an 'internal' function and should not be used by external code / plugins.
"""
# Ensure the primary-key field is not provided
kwargs.pop('id', None)
kwargs.pop('pk', None)
part = kwargs.get('part')
if not part:
raise ValidationError({'part': _('Part must be specified')})
# Create a list of StockItem objects
items = []
# Provide some default field values
data = {**kwargs}
# Remove some extraneous keys which cause issues
for key in ['parent_id', 'part_id', 'build_id']:
data.pop(key, None)
data['parent'] = kwargs.pop('parent', None)
data['tree_id'] = kwargs.pop('tree_id', 0)
data['level'] = kwargs.pop('level', 0)
data['rght'] = kwargs.pop('rght', 0)
data['lft'] = kwargs.pop('lft', 0)
# Force single quantity for each item
data['quantity'] = 1
for serial in serials:
data['serial'] = serial
data['serial_int'] = StockItem.convert_serial_to_int(serial)
items.append(StockItem(**data))
# Create the StockItem objects in bulk
StockItem.objects.bulk_create(items)
# Return the newly created StockItem objects
return StockItem.objects.filter(part=part, serial__in=serials)
@staticmethod
def convert_serial_to_int(serial: str) -> int:
"""Convert the provided serial number to an integer value.
This function hooks into the plugin system to allow for custom serial number conversion.
"""
from plugin.registry import registry
# First, let any plugins convert this serial number to an integer value
# If a non-null value is returned (by any plugin) we will use that
for plugin in registry.with_mixin('validation'):
serial_int = plugin.convert_serial_to_int(serial)
# Save the first returned result
if serial_int is not None:
# Ensure that it is clipped within a range allowed in the database schema
clip = 0x7FFFFFFF
serial_int = abs(serial_int)
serial_int = min(serial_int, clip)
# Return the first non-null value
return serial_int
# None of the plugins provided a valid integer value
if serial not in [None, '']:
return InvenTree.helpers.extract_int(serial)
else:
return None
def update_serial_number(self):
"""Update the 'serial_int' field, to be an integer representation of the serial number.
@ -466,35 +558,15 @@ class StockItem(
"""
serial = str(getattr(self, 'serial', '')).strip()
from plugin.registry import registry
serial_int = self.convert_serial_to_int(serial)
# First, let any plugins convert this serial number to an integer value
# If a non-null value is returned (by any plugin) we will use that
try:
serial_int = int(serial_int)
serial_int = None
for plugin in registry.with_mixin('validation'):
serial_int = plugin.convert_serial_to_int(serial)
if serial_int is not None:
# Save the first returned result
# Ensure that it is clipped within a range allowed in the database schema
clip = 0x7FFFFFFF
serial_int = abs(serial_int)
serial_int = min(serial_int, clip)
self.serial_int = serial_int
return
# If we get to this point, none of the available plugins provided an integer value
# Default value if we cannot convert to an integer
serial_int = 0
if serial not in [None, '']:
serial_int = InvenTree.helpers.extract_int(serial)
if serial_int <= 0:
serial_int = 0
except (ValueError, TypeError):
serial_int = 0
self.serial_int = serial_int
@ -1452,6 +1524,7 @@ class StockItem(
user: User,
deltas: dict | None = None,
notes: str = '',
commit: bool = True,
**kwargs,
):
"""Add a history tracking entry for this StockItem.
@ -1461,6 +1534,9 @@ class StockItem(
user (User): The user performing this action
deltas (dict, optional): A map of the changes made to the model. Defaults to None.
notes (str, optional): URL associated with this tracking entry. Defaults to ''.
Returns:
StockItemTracking: The created tracking entry
"""
if deltas is None:
deltas = {}
@ -1471,7 +1547,7 @@ class StockItem(
and len(deltas) == 0
and not notes
):
return
return None
# Has a location been specified?
location = kwargs.get('location')
@ -1485,7 +1561,7 @@ class StockItem(
if quantity:
deltas['quantity'] = float(quantity)
entry = StockItemTracking.objects.create(
entry = StockItemTracking(
item=self,
tracking_type=entry_type.value,
user=user,
@ -1494,7 +1570,10 @@ class StockItem(
deltas=deltas,
)
entry.save()
if commit:
entry.save()
return entry
@transaction.atomic
def serializeStock(self, quantity, serials, user, notes='', location=None):
@ -1536,7 +1615,7 @@ class StockItem(
if type(serials) not in [list, tuple]:
raise ValidationError({
'serial_numbers': _('Serial numbers must be a list of integers')
'serial_numbers': _('Serial numbers must be provided as a list')
})
if quantity != len(serials):
@ -1552,45 +1631,54 @@ class StockItem(
msg = _('Serial numbers already exist') + f': {exists}'
raise ValidationError({'serial_numbers': msg})
# Create a new stock item for each unique serial number
for serial in serials:
# Create a copy of this StockItem
new_item = StockItem.objects.get(pk=self.pk)
new_item.quantity = 1
new_item.serial = serial
new_item.pk = None
new_item.parent = self
# Serialize this StockItem
data = dict(StockItem.objects.filter(pk=self.pk).values()[0])
if location:
new_item.location = location
if location:
data['location'] = location
# The item already has a transaction history, don't create a new note
new_item.save(user=user, notes=notes)
data['part'] = self.part
data['parent'] = self
data['tree_id'] = self.tree_id
# Copy entire transaction history
new_item.copyHistoryFrom(self)
# Generate a new serial number for each item
items = StockItem._create_serial_numbers(serials, **data)
# Copy test result history
new_item.copyTestResultsFrom(self)
# Create a new tracking entry for each item
history_items = []
# Create a new stock tracking item
new_item.add_tracking_entry(
for item in items:
if entry := item.add_tracking_entry(
StockHistoryCode.ASSIGNED_SERIAL,
user,
notes=notes,
deltas={'serial': serial},
deltas={'serial': item.serial},
location=location,
)
commit=False,
):
history_items.append(entry)
StockItemTracking.objects.bulk_create(history_items)
# Duplicate test results
test_results = []
for test_result in self.test_results.all():
for item in items:
test_result.pk = None
test_result.stock_item = item
test_results.append(test_result)
StockItemTestResult.objects.bulk_create(test_results)
# Remove the equivalent number of items
self.take_stock(quantity, user, notes=notes)
# Rebuild the stock tree
try:
StockItem.objects.partial_rebuild(tree_id=self.tree_id)
except Exception:
logger.warning('Failed to rebuild stock tree during serializeStock')
StockItem.objects.rebuild()
InvenTree.tasks.offload_task(
stock.tasks.rebuild_stock_item_tree, tree_id=self.tree_id
)
@transaction.atomic
def copyHistoryFrom(self, other):
@ -1822,12 +1910,10 @@ class StockItem(
self.save()
# Rebuild stock trees as required
try:
for tree_id in tree_ids:
StockItem.objects.partial_rebuild(tree_id=tree_id)
except Exception:
logger.warning('Rebuilding entire StockItem tree during merge_stock_items')
StockItem.objects.rebuild()
for tree_id in tree_ids:
InvenTree.tasks.offload_task(
stock.tasks.rebuild_stock_item_tree, tree_id=tree_id
)
@transaction.atomic
def splitStock(self, quantity, location=None, user=None, **kwargs):
@ -1922,11 +2008,9 @@ class StockItem(
)
# Rebuild the tree for this parent item
try:
StockItem.objects.partial_rebuild(tree_id=self.tree_id)
except Exception:
logger.warning('Rebuilding entire StockItem tree')
StockItem.objects.rebuild()
InvenTree.tasks.offload_task(
stock.tasks.rebuild_stock_item_tree, tree_id=self.tree_id
)
# Attempt to reload the new item from the database
try:

24
src/backend/InvenTree/stock/tasks.py Normal file
View File

@ -0,0 +1,24 @@
"""Background tasks for the stock app."""
import logging
logger = logging.getLogger('inventree')
def rebuild_stock_item_tree(tree_id=None):
"""Rebuild the stock tree structure.
The StockItem tree uses the MPTT library to manage the tree structure.
"""
from stock.models import StockItem
if tree_id:
try:
StockItem.objects.partial_rebuild(tree_id)
except Exception:
logger.warning('Failed to rebuild StockItem tree')
# If the partial rebuild fails, rebuild the entire tree
StockItem.objects.rebuild()
else:
# No tree_id provided, so rebuild the entire tree
StockItem.objects.rebuild()

2
src/backend/InvenTree/stock/test_api.py
View File

@ -2406,7 +2406,7 @@ class StockStatisticsTest(StockAPITestCase):
fixtures = [*StockAPITestCase.fixtures, 'build']
def test_test_statics(self):
def test_test_statistics(self):
"""Test the test statistics API endpoints."""
part = Part.objects.first()
response = self.get(