From f798537c7379822d6ee01474f3c7b168ca3a4123 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Wed, 3 Feb 2021 09:52:59 +1100 Subject: [PATCH 1/5] Reverse migration company.0024 --- .../migrations/0024_unique_name_email_constraint.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/InvenTree/company/migrations/0024_unique_name_email_constraint.py b/InvenTree/company/migrations/0024_unique_name_email_constraint.py index 3a8781f98d..e6d1fd93dd 100644 --- a/InvenTree/company/migrations/0024_unique_name_email_constraint.py +++ b/InvenTree/company/migrations/0024_unique_name_email_constraint.py @@ -1,6 +1,14 @@ from django.db import migrations, models +def reverse_empty_email(apps, schema_editor): + Company = apps.get_model('company', 'Company') + for company in Company.objects.all(): + if company.email == None: + company.email = '' + company.save() + + def make_empty_email_field_null(apps, schema_editor): Company = apps.get_model('company', 'Company') for company in Company.objects.all(): @@ -23,7 +31,7 @@ class Migration(migrations.Migration): field=models.EmailField(blank=True, help_text='Contact email address', max_length=254, null=True, unique=False, verbose_name='Email'), ), # Convert empty email string to NULL - migrations.RunPython(make_empty_email_field_null), + migrations.RunPython(make_empty_email_field_null, reverse_code=reverse_empty_email), # Remove unique constraint on name field migrations.AlterField( model_name='company', From bc43d14ebfd5c455453aca55bc0183bc09c6dfda Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Wed, 3 Feb 2021 11:28:43 +1100 Subject: [PATCH 2/5] Change model functions to raw SQL --- .../migrations/0019_auto_20200413_0642.py | 91 ++++++++++++------- 1 file changed, 57 insertions(+), 34 deletions(-) diff --git a/InvenTree/company/migrations/0019_auto_20200413_0642.py b/InvenTree/company/migrations/0019_auto_20200413_0642.py index c3c2f58ea0..ced7ed04c3 100644 --- a/InvenTree/company/migrations/0019_auto_20200413_0642.py +++ b/InvenTree/company/migrations/0019_auto_20200413_0642.py @@ -112,50 +112,67 @@ def associate_manufacturers(apps, schema_editor): # Map company names to company objects companies = {} - for company in Company.objects.all(): - companies[company.name] = company + # Iterate through each company object + cursor = connection.cursor() + response = cursor.execute("select id, name from company_company;") + results = cursor.fetchall() - def link_part(part, name): + for index, row in enumerate(results): + pk, name = row + + companies[name] = pk + + def link_part(part_id, name): """ Attempt to link Part to an existing Company """ # Matches a company name directly if name in companies.keys(): - print(" - Part[{pk}]: '{n}' maps to existing manufacturer".format(pk=part.pk, n=name)) - part.manufacturer = companies[name] - part.save() + print(" - Part[{pk}]: '{n}' maps to existing manufacturer".format(pk=part_id, n=name)) + + manufacturer_id = companies[name] + + query = f"update part_supplierpart set manufacturer_id={manufacturer_id} where id={part_id};" + result = cursor.execute(query) + return True # Have we already mapped this if name in links.keys(): - print(" - Part[{pk}]: Mapped '{n}' - '{c}'".format(pk=part.pk, n=name, c=links[name].name)) - part.manufacturer = links[name] - part.save() + print(" - Part[{pk}]: Mapped '{n}' - '{c}'".format(pk=part_id, n=name, c=links[name].name)) + + query = f"update part_supplierpart set manufacturer_id={manufacturer_id} where id={part_id};" + result = query.execute() return True # Mapping not possible return False - def create_manufacturer(part, input_name, company_name): + def create_manufacturer(part_id, input_name, company_name): """ Create a new manufacturer """ - company = Company(name=company_name, description=company_name, is_manufacturer=True) + # Manually create a new database row + # Note: Have to fill out all empty string values! + new_manufacturer_query = f"insert into company_company ('name', 'description', 'is_customer', 'is_supplier', 'is_manufacturer', 'address', 'website', 'phone', 'email', 'contact', 'link', 'notes') values ('{company_name}', '{company_name}', false, false, true, '', '', '', '', '', '', '');" - company.is_manufacturer = True - - # Save the company BEFORE we associate the part, otherwise the PK does not exist - company.save() + cursor = connection.cursor() + + cursor.execute(new_manufacturer_query) + + # Extract the company back from the database + response = cursor.execute(f"select id from company_company where name='{company_name}';") + row = response.fetchone() + manufacturer_id = int(row[0]) # Map both names to the same company - links[input_name] = company - links[company_name] = company + links[input_name] = manufacturer_id + links[company_name] = manufacturer_id - companies[company_name] = company + companies[company_name] = manufacturer_id - print(" - Part[{pk}]: Created new manufacturer: '{name}'".format(pk=part.pk, name=company_name)) + print(" - Part[{pk}]: Created new manufacturer: '{name}'".format(pk=part_id, name=company_name)) - # Save the manufacturer reference link - part.manufacturer = company - part.save() + # Update SupplierPart object in the database + cursor.execute(f"update part_supplierpart set manufacturer_id={manufacturer_id} where id={part_id};") def find_matches(text, threshold=65): """ @@ -178,17 +195,17 @@ def associate_manufacturers(apps, schema_editor): return [] - def map_part_to_manufacturer(part, idx, total): + def map_part_to_manufacturer(part_id, idx, total): - name = get_manufacturer_name(part.id) + name = get_manufacturer_name(part_id) # Skip empty names if not name or len(name) == 0: - print(" - Part[{pk}]: No manufacturer_name provided, skipping".format(pk=part.pk)) + print(" - Part[{pk}]: No manufacturer_name provided, skipping".format(pk=part_id)) return # Can be linked to an existing manufacturer - if link_part(part, name): + if link_part(part_id, name): return # Find a list of potential matches @@ -198,7 +215,7 @@ def associate_manufacturers(apps, schema_editor): # Present a list of options print("----------------------------------") - print("Checking part [{pk}] ({idx} of {total})".format(pk=part.pk, idx=idx+1, total=total)) + print("Checking part [{pk}] ({idx} of {total})".format(pk=part_id, idx=idx+1, total=total)) print("Manufacturer name: '{n}'".format(n=name)) print("----------------------------------") print("Select an option from the list below:") @@ -222,7 +239,7 @@ def associate_manufacturers(apps, schema_editor): # Option 0) is to create a new manufacturer with the current name if n == 0: - create_manufacturer(part, name, name) + create_manufacturer(part_id, name, name) return # Options 1) - n) select an existing manufacturer @@ -270,7 +287,7 @@ def associate_manufacturers(apps, schema_editor): # No match, create a new manufacturer else: - create_manufacturer(part, name, response) + create_manufacturer(part_id, name, response) return clear() @@ -292,16 +309,22 @@ def associate_manufacturers(apps, schema_editor): clear() - part_count = SupplierPart.objects.count() + # Extract all SupplierPart objects from the database + cursor = connection.cursor() + response = cursor.execute("select id, MPN, SKU, manufacturer_id, manufacturer_name from part_supplierpart;") + results = response.fetchall() + part_count = len(results) + # Create a unique set of manufacturer names - for idx, part in enumerate(SupplierPart.objects.all()): + for index, row in enumerate(results): + pk, MPN, SKU, manufacturer_id, manufacturer_name = row - if part.manufacturer is not None: - print(" - Part '{p}' already has a manufacturer associated (skipping)".format(p=part)) + if manufacturer_id is not None: + print(f" - SupplierPart '{pk}' already has a manufacturer associated (skipping)") continue - map_part_to_manufacturer(part, idx, part_count) + map_part_to_manufacturer(pk, index, part_count) print("Done!") From 793e5b820e4334837d9c068debc32ec838c86447 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Wed, 3 Feb 2021 11:56:48 +1100 Subject: [PATCH 3/5] Remove all model references from migration file --- .../migrations/0019_auto_20200413_0642.py | 46 +++++++++++-------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/InvenTree/company/migrations/0019_auto_20200413_0642.py b/InvenTree/company/migrations/0019_auto_20200413_0642.py index ced7ed04c3..deda87a040 100644 --- a/InvenTree/company/migrations/0019_auto_20200413_0642.py +++ b/InvenTree/company/migrations/0019_auto_20200413_0642.py @@ -4,7 +4,6 @@ import os from rapidfuzz import fuzz from django.db import migrations, connection -from company.models import Company, SupplierPart from django.db.utils import OperationalError, ProgrammingError @@ -21,21 +20,25 @@ def reverse_association(apps, schema_editor): into the 'manufacturer_name' field. """ + cursor = connection.cursor() + + response = cursor.execute("select id, MPN from part_supplierpart;") + supplier_parts = response.fetchall() + # Exit if there are no SupplierPart objects # This crucial otherwise the unit test suite fails! - if SupplierPart.objects.count() == 0: + if len(supplier_parts) == 0: return print("Reversing migration for manufacturer association") - for part in SupplierPart.objects.all(): + for (index, row) in enumerate(supplier_parts): + supplier_part_id, MPN = row - print("Checking part [{pk}]:".format(pk=part.pk)) - - cursor = connection.cursor() + print(f"Checking SupplierPart [{supplier_part_id}]:") # Grab the manufacturer ID from the part - response = cursor.execute('SELECT manufacturer_id FROM part_supplierpart WHERE id={ID};'.format(ID=part.id)) + response = cursor.execute(f"SELECT manufacturer_id FROM part_supplierpart WHERE id={supplier_part_id};") manufacturer_id = None @@ -54,7 +57,7 @@ def reverse_association(apps, schema_editor): print(" - Manufacturer ID: [{id}]".format(id=manufacturer_id)) # Now extract the "name" for the manufacturer - response = cursor.execute('SELECT name from company_company where id={ID};'.format(ID=manufacturer_id)) + response = cursor.execute(f"SELECT name from company_company where id={manufacturer_id};") row = response.fetchone() @@ -62,7 +65,7 @@ def reverse_association(apps, schema_editor): print(" - Manufacturer name: '{name}'".format(name=name)) - response = cursor.execute("UPDATE part_supplierpart SET manufacturer_name='{name}' WHERE id={ID};".format(name=name, ID=part.id)) + response = cursor.execute("UPDATE part_supplierpart SET manufacturer_name='{name}' WHERE id={ID};".format(name=name, ID=supplier_part_id)) def associate_manufacturers(apps, schema_editor): """ @@ -100,10 +103,14 @@ def associate_manufacturers(apps, schema_editor): return row[0] return '' + cursor = connection.cursor() + + response = cursor.execute("select id, MPN from part_supplierpart;") + supplier_parts = response.fetchall() # Exit if there are no SupplierPart objects # This crucial otherwise the unit test suite fails! - if SupplierPart.objects.count() == 0: + if len(supplier_parts) == 0: return # Link a 'manufacturer_name' to a 'Company' @@ -113,7 +120,6 @@ def associate_manufacturers(apps, schema_editor): companies = {} # Iterate through each company object - cursor = connection.cursor() response = cursor.execute("select id, name from company_company;") results = cursor.fetchall() @@ -197,6 +203,8 @@ def associate_manufacturers(apps, schema_editor): def map_part_to_manufacturer(part_id, idx, total): + cursor = connection.cursor() + name = get_manufacturer_name(part_id) # Skip empty names @@ -249,21 +257,19 @@ def associate_manufacturers(apps, schema_editor): if n < len(matches): # Get the company which matches the selected options company_name = matches[n] - company = companies[company_name] + company_id = companies[company_name] # Ensure the company is designated as a manufacturer - company.is_manufacturer = True - company.save() + cursor.execute(f"update company_company set is_manufacturer=true where id={company_id};") # Link the company to the part - part.manufacturer = company - part.save() + cursor.execute(f"update part_supplierpart set manufacturer_id={company_id} where id={part_id};") # Link the name to the company - links[name] = company - links[company_name] = company + links[name] = company_id + links[company_name] = company_id - print(" - Part[{pk}]: Linked '{n}' to manufacturer '{m}'".format(pk=part.pk, n=name, m=company_name)) + print(" - Part[{pk}]: Linked '{n}' to manufacturer '{m}'".format(pk=part_id, n=name, m=company_name)) return else: @@ -321,7 +327,7 @@ def associate_manufacturers(apps, schema_editor): pk, MPN, SKU, manufacturer_id, manufacturer_name = row if manufacturer_id is not None: - print(f" - SupplierPart '{pk}' already has a manufacturer associated (skipping)") + print(f" - SupplierPart <{pk}> already has a manufacturer associated (skipping)") continue map_part_to_manufacturer(pk, index, part_count) From 0e246a7fdfb68335942deb8144d67c64677e2fc3 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Wed, 3 Feb 2021 13:02:28 +1100 Subject: [PATCH 4/5] Migration fix (response is different for postgresql) --- .../company/migrations/0019_auto_20200413_0642.py | 12 ++++++------ .../company/migrations/0026_auto_20201110_1011.py | 2 +- InvenTree/part/migrations/0056_auto_20201110_1125.py | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/InvenTree/company/migrations/0019_auto_20200413_0642.py b/InvenTree/company/migrations/0019_auto_20200413_0642.py index deda87a040..a27f56ce3a 100644 --- a/InvenTree/company/migrations/0019_auto_20200413_0642.py +++ b/InvenTree/company/migrations/0019_auto_20200413_0642.py @@ -23,7 +23,7 @@ def reverse_association(apps, schema_editor): cursor = connection.cursor() response = cursor.execute("select id, MPN from part_supplierpart;") - supplier_parts = response.fetchall() + supplier_parts = cursor.fetchall() # Exit if there are no SupplierPart objects # This crucial otherwise the unit test suite fails! @@ -42,7 +42,7 @@ def reverse_association(apps, schema_editor): manufacturer_id = None - row = response.fetchone() + row = cursor.fetchone() if len(row) > 0: try: @@ -59,7 +59,7 @@ def reverse_association(apps, schema_editor): # Now extract the "name" for the manufacturer response = cursor.execute(f"SELECT name from company_company where id={manufacturer_id};") - row = response.fetchone() + row = cursor.fetchone() name = row[0] @@ -106,7 +106,7 @@ def associate_manufacturers(apps, schema_editor): cursor = connection.cursor() response = cursor.execute("select id, MPN from part_supplierpart;") - supplier_parts = response.fetchall() + supplier_parts = cursor.fetchall() # Exit if there are no SupplierPart objects # This crucial otherwise the unit test suite fails! @@ -166,7 +166,7 @@ def associate_manufacturers(apps, schema_editor): # Extract the company back from the database response = cursor.execute(f"select id from company_company where name='{company_name}';") - row = response.fetchone() + row = cursor.fetchone() manufacturer_id = int(row[0]) # Map both names to the same company @@ -318,7 +318,7 @@ def associate_manufacturers(apps, schema_editor): # Extract all SupplierPart objects from the database cursor = connection.cursor() response = cursor.execute("select id, MPN, SKU, manufacturer_id, manufacturer_name from part_supplierpart;") - results = response.fetchall() + results = cursor.fetchall() part_count = len(results) diff --git a/InvenTree/company/migrations/0026_auto_20201110_1011.py b/InvenTree/company/migrations/0026_auto_20201110_1011.py index 1202fbe7f7..553eced0fc 100644 --- a/InvenTree/company/migrations/0026_auto_20201110_1011.py +++ b/InvenTree/company/migrations/0026_auto_20201110_1011.py @@ -106,7 +106,7 @@ def reverse_currencies(apps, schema_editor): # For each currency code in use, check if we have a matching Currency object for code in codes_in_use: response = cursor.execute(f"SELECT id, suffix from common_currency where suffix='{code}';") - row = response.fetchone() + row = cursor.fetchone() if row is not None: # A match exists! diff --git a/InvenTree/part/migrations/0056_auto_20201110_1125.py b/InvenTree/part/migrations/0056_auto_20201110_1125.py index 3e4572b518..13a512bdd0 100644 --- a/InvenTree/part/migrations/0056_auto_20201110_1125.py +++ b/InvenTree/part/migrations/0056_auto_20201110_1125.py @@ -106,7 +106,7 @@ def reverse_currencies(apps, schema_editor): # For each currency code in use, check if we have a matching Currency object for code in codes_in_use: response = cursor.execute(f"SELECT id, suffix from common_currency where suffix='{code}';") - row = response.fetchone() + row = cursor.fetchone() if row is not None: # A match exists! From 5e9097b5e09d088a01a6a15016802a56f7882d59 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Wed, 3 Feb 2021 13:16:32 +1100 Subject: [PATCH 5/5] PSQL: Upper-case column names *must* be qualified with double-quotes Ref: https://www.xspdf.com/resolution/53039249.html --- InvenTree/company/migrations/0019_auto_20200413_0642.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/InvenTree/company/migrations/0019_auto_20200413_0642.py b/InvenTree/company/migrations/0019_auto_20200413_0642.py index a27f56ce3a..509c45aaa2 100644 --- a/InvenTree/company/migrations/0019_auto_20200413_0642.py +++ b/InvenTree/company/migrations/0019_auto_20200413_0642.py @@ -22,7 +22,7 @@ def reverse_association(apps, schema_editor): cursor = connection.cursor() - response = cursor.execute("select id, MPN from part_supplierpart;") + response = cursor.execute('select id, "MPN" from part_supplierpart;') supplier_parts = cursor.fetchall() # Exit if there are no SupplierPart objects @@ -105,7 +105,7 @@ def associate_manufacturers(apps, schema_editor): cursor = connection.cursor() - response = cursor.execute("select id, MPN from part_supplierpart;") + response = cursor.execute(f'select id, "MPN" from part_supplierpart;') supplier_parts = cursor.fetchall() # Exit if there are no SupplierPart objects @@ -317,7 +317,7 @@ def associate_manufacturers(apps, schema_editor): # Extract all SupplierPart objects from the database cursor = connection.cursor() - response = cursor.execute("select id, MPN, SKU, manufacturer_id, manufacturer_name from part_supplierpart;") + response = cursor.execute('select id, "MPN", "SKU", manufacturer_id, manufacturer_name from part_supplierpart;') results = cursor.fetchall() part_count = len(results)