Migration bug fix (#3325)

* Add fix for stock migration

- Ensure the serial number is not too large when performing migration
- Add unit test for data migration

(cherry picked from commit 661fbf0e3dbdf6444d3d25b02d68ad229925d87c)
(cherry picked from commit 233446c2bb520e7ef08a1664f7ec6451b93cef29)

* Add similar fixes for PO and SO migrations

(cherry picked from commit bde23c130c879e7663091fba808bbd57c52ed8bf)
(cherry picked from commit 4261090e6d44231cbef741c989be25020a1cf89a)

* And similar fix for BuildOrder reference field

(cherry picked from commit ca0f4e00310aed0551f8fad5c57f90fae2177f04)
(cherry picked from commit 9fa4ee48d65b0af9a83be9e859a731da04186121)

* Fix for plugin unit testing

* Revert test database name

(cherry picked from commit 53333c29c38ae393b1e31e764e08a1239839a594)

* Override default URL behaviour for unit test

(cherry picked from commit 2c12a695294c2785e82b7f469f79a7d1a5412e71)

InvenTree/InvenTree/settings.py

@@ -522,7 +522,7 @@ if "mysql" in db_engine:  # pragma: no cover
     # https://docs.djangoproject.com/en/3.2/ref/databases/#mysql-isolation-level
     if "isolation_level" not in db_options:
         serializable = _is_true(
-            os.getenv("INVENTREE_DB_ISOLATION_SERIALIZABLE", "true")
+            os.getenv("INVENTREE_DB_ISOLATION_SERIALIZABLE", "false")
         )
         db_options["isolation_level"] = (
             "serializable" if serializable else "read committed"

InvenTree/build/migrations/0032_auto_20211014_0632.py

@@ -24,6 +24,10 @@ def build_refs(apps, schema_editor):
             except:  # pragma: no cover
                 ref = 0
 
+        # Clip integer value to ensure it does not overflow database field
+        if ref > 0x7fffffff:
+            ref = 0x7fffffff
+
         build.reference_int = ref
         build.save()
 

InvenTree/order/migrations/0052_auto_20211014_0631.py

@@ -23,6 +23,10 @@ def build_refs(apps, schema_editor):
             except:  # pragma: no cover
                 ref = 0
 
+        # Clip integer value to ensure it does not overflow database field
+        if ref > 0x7fffffff:
+            ref = 0x7fffffff
+
         order.reference_int = ref
         order.save()
 
@@ -40,6 +44,10 @@ def build_refs(apps, schema_editor):
             except:  # pragma: no cover
                 ref = 0
 
+        # Clip integer value to ensure it does not overflow database field
+        if ref > 0x7fffffff:
+            ref = 0x7fffffff
+
         order.reference_int = ref
         order.save()
 

InvenTree/order/test_migrations.py

@@ -56,6 +56,19 @@ class TestRefIntMigrations(MigratorTestCase):
             with self.assertRaises(AttributeError):
                 print(sales_order.reference_int)
 
+        # Create orders with very large reference values
+        self.po_pk = PurchaseOrder.objects.create(
+            supplier=supplier,
+            reference='999999999999999999999999999999999',
+            description='Big reference field',
+        ).pk
+
+        self.so_pk = SalesOrder.objects.create(
+            customer=supplier,
+            reference='999999999999999999999999999999999',
+            description='Big reference field',
+        ).pk
+
     def test_ref_field(self):
         """
         Test that the 'reference_int' field has been created and is filled out correctly
@@ -73,6 +86,15 @@ class TestRefIntMigrations(MigratorTestCase):
             self.assertEqual(po.reference_int, ii)
             self.assertEqual(so.reference_int, ii)
 
+        # Tests for orders with overly large reference values
+        po = PurchaseOrder.objects.get(pk=self.po_pk)
+        self.assertEqual(po.reference, '999999999999999999999999999999999')
+        self.assertEqual(po.reference_int, 0x7fffffff)
+
+        so = SalesOrder.objects.get(pk=self.so_pk)
+        self.assertEqual(so.reference, '999999999999999999999999999999999')
+        self.assertEqual(so.reference_int, 0x7fffffff)
+
 
 class TestShipmentMigration(MigratorTestCase):
     """

InvenTree/plugin/base/integration/mixins.py

@@ -467,6 +467,10 @@ class APICallMixin:
         if endpoint_is_url:
             url = endpoint
         else:
+
+            if endpoint.startswith('/'):
+                endpoint = endpoint[1:]
+
             url = f'{self.api_url}/{endpoint}'
 
         # build kwargs for call
@@ -474,6 +478,7 @@ class APICallMixin:
             'url': url,
             'headers': headers,
         }
+
         if data:
             kwargs['data'] = json.dumps(data)
 

InvenTree/plugin/base/integration/test_mixins.py

@@ -1,4 +1,4 @@
-""" Unit tests for base mixins for plugins """
+"""Unit tests for base mixins for plugins."""
 
 from django.conf import settings
 from django.test import TestCase
@@ -17,7 +17,10 @@ from plugin.urls import PLUGIN_BASE
 
 
 class BaseMixinDefinition:
+    """Mixin to test the meta functions of all mixins."""
+
     def test_mixin_name(self):
+        """Test that the mixin registers itseld correctly."""
         # mixin name
         self.assertIn(self.MIXIN_NAME, [item['key'] for item in self.mixin.registered_mixins])
         # human name
@@ -25,6 +28,8 @@ class BaseMixinDefinition:
 
 
 class SettingsMixinTest(BaseMixinDefinition, InvenTreeTestCase):
+    """Tests for SettingsMixin."""
+
     MIXIN_HUMAN_NAME = 'Settings'
     MIXIN_NAME = 'settings'
     MIXIN_ENABLE_CHECK = 'has_settings'
@@ -32,6 +37,7 @@ class SettingsMixinTest(BaseMixinDefinition, InvenTreeTestCase):
     TEST_SETTINGS = {'SETTING1': {'default': '123', }}
 
     def setUp(self):
+        """Setup for all tests."""
         class SettingsCls(SettingsMixin, InvenTreePlugin):
             SETTINGS = self.TEST_SETTINGS
         self.mixin = SettingsCls()
@@ -43,6 +49,7 @@ class SettingsMixinTest(BaseMixinDefinition, InvenTreeTestCase):
         super().setUp()
 
     def test_function(self):
+        """Test that the mixin functions."""
         # settings variable
         self.assertEqual(self.mixin.settings, self.TEST_SETTINGS)
 
@@ -60,11 +67,14 @@ class SettingsMixinTest(BaseMixinDefinition, InvenTreeTestCase):
 
 
 class UrlsMixinTest(BaseMixinDefinition, TestCase):
+    """Tests for UrlsMixin."""
+
     MIXIN_HUMAN_NAME = 'URLs'
     MIXIN_NAME = 'urls'
     MIXIN_ENABLE_CHECK = 'has_urls'
 
     def setUp(self):
+        """Setup for all tests."""
         class UrlsCls(UrlsMixin, InvenTreePlugin):
             def test():
                 return 'ccc'
@@ -76,6 +86,7 @@ class UrlsMixinTest(BaseMixinDefinition, TestCase):
         self.mixin_nothing = NoUrlsCls()
 
     def test_function(self):
+        """Test that the mixin functions."""
         plg_name = self.mixin.plugin_name()
 
         # base_url
@@ -99,26 +110,32 @@ class UrlsMixinTest(BaseMixinDefinition, TestCase):
 
 
 class AppMixinTest(BaseMixinDefinition, TestCase):
+    """Tests for AppMixin."""
+
     MIXIN_HUMAN_NAME = 'App registration'
     MIXIN_NAME = 'app'
     MIXIN_ENABLE_CHECK = 'has_app'
 
     def setUp(self):
+        """Setup for all tests."""
         class TestCls(AppMixin, InvenTreePlugin):
             pass
         self.mixin = TestCls()
 
     def test_function(self):
-        # test that this plugin is in settings
+        """Test that the sample plugin registers in settings."""
         self.assertIn('plugin.samples.integration', settings.INSTALLED_APPS)
 
 
 class NavigationMixinTest(BaseMixinDefinition, TestCase):
+    """Tests for NavigationMixin."""
+
     MIXIN_HUMAN_NAME = 'Navigation Links'
     MIXIN_NAME = 'navigation'
     MIXIN_ENABLE_CHECK = 'has_naviation'
 
     def setUp(self):
+        """Setup for all tests."""
         class NavigationCls(NavigationMixin, InvenTreePlugin):
             NAVIGATION = [
                 {'name': 'aa', 'link': 'plugin:test:test_view'},
@@ -131,6 +148,7 @@ class NavigationMixinTest(BaseMixinDefinition, TestCase):
         self.nothing_mixin = NothingNavigationCls()
 
     def test_function(self):
+        """Test that a correct configuration functions."""
         # check right configuration
         self.assertEqual(self.mixin.navigation, [{'name': 'aa', 'link': 'plugin:test:test_view'}, ])
 
@@ -139,7 +157,7 @@ class NavigationMixinTest(BaseMixinDefinition, TestCase):
         self.assertEqual(self.nothing_mixin.navigation_name, '')
 
     def test_fail(self):
-        # check wrong links fails
+        """Test that wrong links fail."""
         with self.assertRaises(NotImplementedError):
             class NavigationCls(NavigationMixin, InvenTreePlugin):
                 NAVIGATION = ['aa', 'aa']
@@ -147,11 +165,15 @@ class NavigationMixinTest(BaseMixinDefinition, TestCase):
 
 
 class APICallMixinTest(BaseMixinDefinition, TestCase):
+    """Tests for APICallMixin."""
+
     MIXIN_HUMAN_NAME = 'API calls'
     MIXIN_NAME = 'api_call'
     MIXIN_ENABLE_CHECK = 'has_api_call'
 
     def setUp(self):
+        """Setup for all tests."""
+
         class MixinCls(APICallMixin, SettingsMixin, InvenTreePlugin):
             NAME = "Sample API Caller"
 
@@ -163,40 +185,47 @@ class APICallMixinTest(BaseMixinDefinition, TestCase):
                 'API_URL': {
                     'name': 'External URL',
                     'description': 'Where is your API located?',
-                    'default': 'reqres.in',
+                    'default': 'https://api.github.com',
                 },
             }
+
             API_URL_SETTING = 'API_URL'
             API_TOKEN_SETTING = 'API_TOKEN'
 
+            @property
+            def api_url(self):
+                """Override API URL for this test"""
+                return "https://api.github.com"
+
             def get_external_url(self, simple: bool = True):
-                '''
+                """Returns data from the sample endpoint."""
-                returns data from the sample endpoint
+                return self.api_call('orgs/inventree', simple_response=simple)
-                '''
+
-                return self.api_call('api/users/2', simple_response=simple)
         self.mixin = MixinCls()
 
         class WrongCLS(APICallMixin, InvenTreePlugin):
             pass
+
         self.mixin_wrong = WrongCLS()
 
         class WrongCLS2(APICallMixin, InvenTreePlugin):
             API_URL_SETTING = 'test'
+
         self.mixin_wrong2 = WrongCLS2()
 
     def test_base_setup(self):
-        """Test that the base settings work"""
+        """Test that the base settings work."""
         # check init
         self.assertTrue(self.mixin.has_api_call)
         # api_url
-        self.assertEqual('https://reqres.in', self.mixin.api_url)
+        self.assertEqual('https://api.github.com', self.mixin.api_url)
 
         # api_headers
         headers = self.mixin.api_headers
         self.assertEqual(headers, {'Bearer': '', 'Content-Type': 'application/json'})
 
     def test_args(self):
-        """Test that building up args work"""
+        """Test that building up args work."""
         # api_build_url_args
         # 1 arg
         result = self.mixin.api_build_url_args({'a': 'b'})
@@ -209,11 +238,13 @@ class APICallMixinTest(BaseMixinDefinition, TestCase):
         self.assertEqual(result, '?a=b&c=d,e,f')
 
     def test_api_call(self):
-        """Test that api calls work"""
+        """Test that api calls work."""
         # api_call
         result = self.mixin.get_external_url()
         self.assertTrue(result)
-        self.assertIn('data', result,)
+
+        for key in ['login', 'email', 'name', 'twitter_username']:
+            self.assertIn(key, result)
 
         # api_call without json conversion
         result = self.mixin.get_external_url(False)
@@ -221,25 +252,25 @@ class APICallMixinTest(BaseMixinDefinition, TestCase):
         self.assertEqual(result.reason, 'OK')
 
         # api_call with full url
-        result = self.mixin.api_call('https://reqres.in/api/users/2', endpoint_is_url=True)
+        result = self.mixin.api_call('https://api.github.com/orgs/inventree', endpoint_is_url=True)
         self.assertTrue(result)
 
         # api_call with post and data
         result = self.mixin.api_call(
-            'api/users/',
+            'repos/inventree/InvenTree',
-            data={"name": "morpheus", "job": "leader"},
+            method='GET'
-            method='POST'
         )
+
         self.assertTrue(result)
-        self.assertEqual(result['name'], 'morpheus')
+        self.assertEqual(result['name'], 'InvenTree')
+        self.assertEqual(result['html_url'], 'https://github.com/inventree/InvenTree')
 
         # api_call with filter
-        result = self.mixin.api_call('api/users', url_args={'page': '2'})
+        result = self.mixin.api_call('repos/inventree/InvenTree/stargazers', url_args={'page': '2'})
         self.assertTrue(result)
-        self.assertEqual(result['page'], 2)
 
     def test_function_errors(self):
-        """Test function errors"""
+        """Test function errors."""
         # wrongly defined plugins should not load
         with self.assertRaises(MixinNotImplementedError):
             self.mixin_wrong.has_api_call()
@@ -250,7 +281,7 @@ class APICallMixinTest(BaseMixinDefinition, TestCase):
 
 
 class PanelMixinTests(InvenTreeTestCase):
-    """Test that the PanelMixin plugin operates correctly"""
+    """Test that the PanelMixin plugin operates correctly."""
 
     fixtures = [
         'category',
@@ -262,8 +293,7 @@ class PanelMixinTests(InvenTreeTestCase):
     roles = 'all'
 
     def test_installed(self):
-        """Test that the sample panel plugin is installed"""
+        """Test that the sample panel plugin is installed."""
-
         plugins = registry.with_mixin('panel')
 
         self.assertTrue(len(plugins) > 0)
@@ -275,8 +305,7 @@ class PanelMixinTests(InvenTreeTestCase):
         self.assertEqual(len(plugins), 0)
 
     def test_disabled(self):
-        """Test that the panels *do not load* if the plugin is not enabled"""
+        """Test that the panels *do not load* if the plugin is not enabled."""
-
         plugin = registry.get_plugin('samplepanel')
 
         plugin.set_setting('ENABLE_HELLO_WORLD', True)
@@ -305,10 +334,7 @@ class PanelMixinTests(InvenTreeTestCase):
             self.assertNotIn('Custom Part Panel', str(response.content))
 
     def test_enabled(self):
-        """
+        """Test that the panels *do* load if the plugin is enabled."""
-        Test that the panels *do* load if the plugin is enabled
-        """
-
         plugin = registry.get_plugin('samplepanel')
 
         self.assertEqual(len(registry.with_mixin('panel', active=True)), 0)
@@ -382,8 +408,7 @@ class PanelMixinTests(InvenTreeTestCase):
         self.assertEqual(Error.objects.count(), n_errors + len(urls))
 
     def test_mixin(self):
-        """Test that ImplementationError is raised"""
+        """Test that ImplementationError is raised."""
-
         with self.assertRaises(MixinNotImplementedError):
             class Wrong(PanelMixin, InvenTreePlugin):
                 pass

InvenTree/stock/migrations/0069_auto_20211109_2347.py

@@ -28,6 +28,9 @@ def update_serials(apps, schema_editor):
             except:
                 serial = 0
 
+        # Ensure the integer value is not too large for the database field
+        if serial > 0x7fffffff:
+            serial = 0x7fffffff
 
         item.serial_int = serial
         item.save()

InvenTree/stock/test_migrations.py

@@ -0,0 +1,69 @@
+"""Unit tests for data migrations in the 'stock' app"""
+
+from django_test_migrations.contrib.unittest_case import MigratorTestCase
+
+from InvenTree import helpers
+
+
+class TestSerialNumberMigration(MigratorTestCase):
+    """Test data migration which updates serial numbers"""
+
+    migrate_from = ('stock', '0067_alter_stockitem_part')
+    migrate_to = ('stock', helpers.getNewestMigrationFile('stock'))
+
+    def prepare(self):
+        """Create initial data for this migration"""
+
+        Part = self.old_state.apps.get_model('part', 'part')
+        StockItem = self.old_state.apps.get_model('stock', 'stockitem')
+
+        # Create a base part
+        my_part = Part.objects.create(
+            name='PART-123',
+            description='Some part',
+            active=True,
+            trackable=True,
+            level=0,
+            tree_id=0,
+            lft=0, rght=0
+        )
+
+        # Create some serialized stock items
+        for sn in range(10, 20):
+            StockItem.objects.create(
+                part=my_part,
+                quantity=1,
+                serial=sn,
+                level=0,
+                tree_id=0,
+                lft=0, rght=0
+            )
+
+        # Create a stock item with a very large serial number
+        item = StockItem.objects.create(
+            part=my_part,
+            quantity=1,
+            serial='9999999999999999999999999999999999999999999999999999999999999',
+            level=0,
+            tree_id=0,
+            lft=0, rght=0
+        )
+
+        self.big_ref_pk = item.pk
+
+    def test_migrations(self):
+        """Test that the migrations have been applied correctly"""
+
+        StockItem = self.new_state.apps.get_model('stock', 'stockitem')
+
+        # Check that the serial number integer conversion has been applied correctly
+        for sn in range(10, 20):
+            item = StockItem.objects.get(serial_int=sn)
+
+            self.assertEqual(item.serial, str(sn))
+
+        big_ref_item = StockItem.objects.get(pk=self.big_ref_pk)
+
+        # Check that the StockItem maximum serial number
+        self.assertEqual(big_ref_item.serial, '9999999999999999999999999999999999999999999999999999999999999')
+        self.assertEqual(big_ref_item.serial_int, 0x7fffffff)