Migration fix (#12808)

* Fix for Note migration - Handles case where data migration is running on a DB which does not support RETURNING * Run migration tests against all supported backends * Fix typo * Adjust CI workflows * TZ migration fix * Tweak migration test * Adjust migration tests * Ensure consistent ordering * Bump API version * Fix tests to allow non psql database support
2026-09-09 22:30:17 +00:00 · 2026-09-08 18:50:13 +10:00
parent 0a5b04b16d
commit 7cd74eecd4
7 changed files with 320 additions and 147 deletions
@@ -0,0 +1,241 @@
+# Data migration unit tests, run against every supported database backend
+
+name: Migration Tests
+
+on:
+  push:
+    branches-ignore: ["l10*", "dependabot/**", "backport/**"]
+  pull_request:
+    branches-ignore: ["l10*"]
+
+env:
+  python_version: 3.12
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  INVENTREE_MEDIA_ROOT: /home/runner/work/InvenTree/test_inventree_media
+  INVENTREE_STATIC_ROOT: /home/runner/work/InvenTree/test_inventree_static
+  INVENTREE_BACKUP_DIR: /home/runner/work/InvenTree/test_inventree_backup
+  INVENTREE_SITE_URL: http://localhost:8000
+
+permissions:
+  contents: read
+
+jobs:
+  paths-filter:
+    name: Filter
+    runs-on: ubuntu-latest
+
+    outputs:
+      migrations: ${{ steps.filter.outputs.migrations }}
+      force: ${{ steps.force.outputs.force }}
+
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3
+        id: filter
+        with:
+          filters: |
+            migrations:
+              - '**/test_migrations.py'
+              - '**/*migrations.py'
+              - '**/migrations/**'
+              - '.github/workflows**'
+              - 'src/backend/requirements.txt'
+      - name: Is CI being forced?
+        run: echo "force=true" >> $GITHUB_OUTPUT
+        id: force
+        if: |
+          contains(github.event.pull_request.labels.*.name, 'dependency') ||
+          contains(github.event.pull_request.labels.*.name, 'full-run')
+
+  sqlite:
+    name: Tests - Migrations [SQLite]
+    runs-on: ubuntu-latest
+    needs: paths-filter
+    if: ${{ (needs.paths-filter.outputs.force == 'true') || (github.ref == 'refs/heads/master' && needs.paths-filter.outputs.migrations == 'true') }}
+
+    env:
+      INVENTREE_DB_ENGINE: sqlite3
+      INVENTREE_DB_NAME: /home/runner/work/InvenTree/db.sqlite3
+      INVENTREE_DEBUG: true
+      INVENTREE_LOG_LEVEL: WARNING
+      INVENTREE_PLUGINS_ENABLED: false
+
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - name: Environment Setup
+        uses: ./.github/actions/setup
+        with:
+          apt-dependency: gettext poppler-utils
+          dev-install: true
+          update: true
+      - name: Run Tests
+        run: invoke dev.test --check --migrations --report --coverage --translations
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
+        if: always()
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          slug: inventree/InvenTree
+          flags: migrations-sqlite
+
+  mysql:
+    name: Tests - Migrations [MySQL]
+    runs-on: ubuntu-24.04
+    needs: paths-filter
+    if: ${{ (needs.paths-filter.outputs.force == 'true') || (github.ref == 'refs/heads/master' && needs.paths-filter.outputs.migrations == 'true') }}
+
+    env:
+      INVENTREE_DB_ENGINE: django.db.backends.mysql
+      INVENTREE_DB_NAME: inventree
+      INVENTREE_DB_USER: root
+      INVENTREE_DB_PASSWORD: password
+      INVENTREE_DB_HOST: "127.0.0.1"
+      INVENTREE_DB_PORT: 3306
+      INVENTREE_DEBUG: true
+      INVENTREE_LOG_LEVEL: WARNING
+      INVENTREE_PLUGINS_ENABLED: false
+
+    services:
+      mysql:
+        image: mysql:9
+        env:
+          MYSQL_ALLOW_EMPTY_PASSWORD: yes
+          MYSQL_DATABASE: inventree
+          MYSQL_USER: inventree
+          MYSQL_PASSWORD: password
+          MYSQL_ROOT_PASSWORD: password
+        options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3
+        ports:
+          - 3306:3306
+
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - name: Environment Setup
+        uses: ./.github/actions/setup
+        with:
+          apt-dependency: gettext poppler-utils libmysqlclient-dev
+          pip-dependency: mysqlclient
+          dev-install: true
+          update: true
+      - name: Run Tests
+        run: invoke dev.test --check --migrations --report --coverage --translations
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
+        if: always()
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          slug: inventree/InvenTree
+          flags: migrations-mysql
+
+  postgresql:
+    name: Tests - Migrations [PostgreSQL]
+    runs-on: ubuntu-latest
+    needs: paths-filter
+    if: ${{ (needs.paths-filter.outputs.force == 'true') || (github.ref == 'refs/heads/master' && needs.paths-filter.outputs.migrations == 'true') }}
+
+    env:
+      INVENTREE_DB_ENGINE: django.db.backends.postgresql
+      INVENTREE_DB_NAME: inventree
+      INVENTREE_DB_USER: inventree
+      INVENTREE_DB_PASSWORD: password
+      INVENTREE_DB_HOST: "127.0.0.1"
+      INVENTREE_DB_PORT: 5432
+      INVENTREE_DEBUG: false
+      INVENTREE_LOG_LEVEL: WARNING
+      INVENTREE_PLUGINS_ENABLED: false
+
+    services:
+      postgres:
+        image: postgres:17
+        env:
+          POSTGRES_USER: inventree
+          POSTGRES_PASSWORD: password
+        ports:
+          - 5432:5432
+
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - name: Environment Setup
+        uses: ./.github/actions/setup
+        with:
+          apt-dependency: gettext poppler-utils libpq-dev
+          pip-dependency: psycopg
+          dev-install: true
+          update: true
+      - name: Run Tests
+        run: invoke dev.test --check --migrations --report --coverage --translations
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
+        if: always()
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          slug: inventree/InvenTree
+          flags: migrations-postgresql
+
+  migrations-checks:
+    name: Tests - Database Migrations
+    runs-on: ubuntu-latest
+    needs: paths-filter
+    if: ${{ (needs.paths-filter.outputs.force == 'true') || (github.ref == 'refs/heads/master' && needs.paths-filter.outputs.migrations == 'true') }}
+
+    env:
+      INVENTREE_DB_ENGINE: sqlite3
+      INVENTREE_DB_NAME: /home/runner/work/InvenTree/db.sqlite3
+      INVENTREE_DEBUG: true
+      INVENTREE_LOG_LEVEL: WARNING
+      INVENTREE_PLUGINS_ENABLED: false
+
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+        name: Checkout Code
+      - name: Environment Setup
+        uses: ./.github/actions/setup
+        with:
+          install: true
+      - name: Fetch Database
+        run: git clone --depth 1 https://github.com/inventree/test-db ./test-db
+
+      - name: 0.10.0 Database
+        run: |
+          rm /home/runner/work/InvenTree/db.sqlite3
+          cp test-db/stable_0.10.0.sqlite3 /home/runner/work/InvenTree/db.sqlite3
+          chmod +rw /home/runner/work/InvenTree/db.sqlite3
+          invoke migrate
+
+      - name: 0.11.0 Database
+        run: |
+          rm /home/runner/work/InvenTree/db.sqlite3
+          cp test-db/stable_0.11.0.sqlite3 /home/runner/work/InvenTree/db.sqlite3
+          chmod +rw /home/runner/work/InvenTree/db.sqlite3
+          invoke migrate
+
+      - name: 0.13.5 Database
+        run: |
+          rm /home/runner/work/InvenTree/db.sqlite3
+          cp test-db/stable_0.13.5.sqlite3 /home/runner/work/InvenTree/db.sqlite3
+          chmod +rw /home/runner/work/InvenTree/db.sqlite3
+          invoke migrate
+
+      - name: 0.16.0 Database
+        run: |
+          rm /home/runner/work/InvenTree/db.sqlite3
+          cp test-db/stable_0.16.0.sqlite3 /home/runner/work/InvenTree/db.sqlite3
+          chmod +rw /home/runner/work/InvenTree/db.sqlite3
+          invoke migrate
+
+      - name: 0.17.0 Database
+        run: |
+          rm /home/runner/work/InvenTree/db.sqlite3
+          cp test-db/stable_0.17.0.sqlite3 /home/runner/work/InvenTree/db.sqlite3
+          chmod +rw /home/runner/work/InvenTree/db.sqlite3
+          invoke migrate
@@ -33,7 +33,6 @@ jobs:

    outputs:
      server: ${{ steps.filter.outputs.server }}
-      migrations: ${{ steps.filter.outputs.migrations }}
      frontend: ${{ steps.filter.outputs.frontend }}
      api: ${{ steps.filter.outputs.api }}
      force: ${{ steps.force.outputs.force }}
@@ -55,11 +54,6 @@ jobs:
              - 'src/backend/InvenTree/**'
              - 'src/backend/requirements.txt'
              - 'src/backend/requirements-dev.txt'
-            migrations:
-              - '**/test_migrations.py'
-              - '**/migrations/**'
-              - '.github/workflows**'
-              - 'src/backend/requirements.txt'
            api:
              - 'src/backend/InvenTree/InvenTree/api_version.py'
            frontend:
@@ -559,113 +553,6 @@ jobs:
      - name: Data Export Test
        uses: ./.github/actions/migration

-  migration-tests:
-    name: Tests - Migrations [PostgreSQL]
-    runs-on: ubuntu-latest
-    needs: paths-filter
-    if: ${{ (needs.paths-filter.outputs.force == 'true') || (github.ref == 'refs/heads/master' && needs.paths-filter.outputs.migrations == 'true') }}
-
-    env:
-      INVENTREE_DB_ENGINE: django.db.backends.postgresql
-      INVENTREE_DB_NAME: inventree
-      INVENTREE_DB_USER: inventree
-      INVENTREE_DB_PASSWORD: password
-      INVENTREE_DB_HOST: "127.0.0.1"
-      INVENTREE_DB_PORT: 5432
-      INVENTREE_DEBUG: False
-      INVENTREE_LOG_LEVEL: WARNING
-      INVENTREE_PLUGINS_ENABLED: false
-
-    services:
-      postgres:
-        image: postgres:17
-        env:
-          POSTGRES_USER: inventree
-          POSTGRES_PASSWORD: password
-        ports:
-          - 5432:5432
-
-    steps:
-      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
-        with:
-          persist-credentials: false
-      - name: Environment Setup
-        uses: ./.github/actions/setup
-        with:
-          apt-dependency: gettext poppler-utils libpq-dev
-          pip-dependency: psycopg
-          dev-install: true
-          update: true
-      - name: Run Tests
-        run: invoke dev.test --check --migrations --report --coverage --translations
-      - name: Upload coverage reports to Codecov
-        uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
-        if: always()
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          slug: inventree/InvenTree
-          flags: migrations
-
-  migrations-checks:
-    name: Tests - Full Migration [SQLite]
-    runs-on: ubuntu-latest
-    needs: paths-filter
-    if: ${{ (needs.paths-filter.outputs.force == 'true') || (github.ref == 'refs/heads/master' && needs.paths-filter.outputs.migrations == 'true') }}
-
-    env:
-      INVENTREE_DB_ENGINE: sqlite3
-      INVENTREE_DB_NAME: /home/runner/work/InvenTree/db.sqlite3
-      INVENTREE_DEBUG: true
-      INVENTREE_LOG_LEVEL: WARNING
-      INVENTREE_PLUGINS_ENABLED: false
-
-    steps:
-      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
-        with:
-          persist-credentials: false
-        name: Checkout Code
-      - name: Environment Setup
-        uses: ./.github/actions/setup
-        with:
-          install: true
-      - name: Fetch Database
-        run: git clone --depth 1 https://github.com/inventree/test-db ./test-db
-
-      - name: 0.10.0 Database
-        run: |
-          rm /home/runner/work/InvenTree/db.sqlite3
-          cp test-db/stable_0.10.0.sqlite3 /home/runner/work/InvenTree/db.sqlite3
-          chmod +rw /home/runner/work/InvenTree/db.sqlite3
-          invoke migrate
-
-      - name: 0.11.0 Database
-        run: |
-          rm /home/runner/work/InvenTree/db.sqlite3
-          cp test-db/stable_0.11.0.sqlite3 /home/runner/work/InvenTree/db.sqlite3
-          chmod +rw /home/runner/work/InvenTree/db.sqlite3
-          invoke migrate
-
-      - name: 0.13.5 Database
-        run: |
-          rm /home/runner/work/InvenTree/db.sqlite3
-          cp test-db/stable_0.13.5.sqlite3 /home/runner/work/InvenTree/db.sqlite3
-          chmod +rw /home/runner/work/InvenTree/db.sqlite3
-          invoke migrate
-
-      - name: 0.16.0 Database
-        run: |
-          rm /home/runner/work/InvenTree/db.sqlite3
-          cp test-db/stable_0.16.0.sqlite3 /home/runner/work/InvenTree/db.sqlite3
-          chmod +rw /home/runner/work/InvenTree/db.sqlite3
-          invoke migrate
-
-      - name: 0.17.0 Database
-        run: |
-          rm /home/runner/work/InvenTree/db.sqlite3
-          cp test-db/stable_0.17.0.sqlite3 /home/runner/work/InvenTree/db.sqlite3
-          chmod +rw /home/runner/work/InvenTree/db.sqlite3
-          invoke migrate
-
  zizmor:
    name: Security [Zizmor]
    runs-on: ubuntu-24.04
@@ -1,11 +1,14 @@
 """InvenTree API version information."""

 # InvenTree API version
-INVENTREE_API_VERSION = 544
+INVENTREE_API_VERSION = 545
 """Increment this API version number whenever there is a significant change to the API that any clients need to know about."""

 INVENTREE_API_TEXT = """

+v545 -> 2026-09-08 : https://github.com/inventree/InvenTree/pull/12808
+    - Ensure consistent ordering of SSO options in API documentation
+
 v544 -> 2026-09-07 : https://github.com/inventree/InvenTree/pull/12807
    - Adds filtering by filename on the Attachment list API endpoint

@@ -987,7 +987,7 @@ SOCIAL_BACKENDS = get_setting(
 )

 DEFAULT_SOCIAL = ['saml', 'openid_connect']
-_SOCIAL_BACKENDS = {*DEFAULT_SOCIAL, *SOCIAL_BACKENDS}
+_SOCIAL_BACKENDS = list(dict.fromkeys([*DEFAULT_SOCIAL, *SOCIAL_BACKENDS]))

 # region auth
 for app in _SOCIAL_BACKENDS:  # pragma: no cover
@@ -117,7 +117,16 @@ def create_notes_batch(Note, NotesImage, content_type, model, instances, unlinke
        batch_size=BATCH_SIZE,
    )

-    notes_by_model_id = {instance.pk: note for instance, note in zip(instances, notes)}
+    # bulk_create() only populates the pk on the returned objects on backends
+    # that support RETURNING on bulk insert.
+    # So we re-fetch the notes we just created rather than trusting the returned objects.
+    notes_by_model_id = {
+        note.model_id: note
+        for note in Note.objects.filter(
+            model_type=content_type,
+            model_id__in=[instance.pk for instance in instances],
+        )
+    }

    # Images directly linked to one of these instances
    direct_images = list(
@@ -131,7 +140,8 @@ def create_notes_batch(Note, NotesImage, content_type, model, instances, unlinke
    # Images not directly linked to any instance, but still referenced in the
    # markdown content itself
    embedded_images = []
-    for instance, note in zip(instances, notes):
+    for instance in instances:
+        note = notes_by_model_id[instance.pk]
        matched = [
            image for image in unlinked_images if image.image.url in instance.notes
        ]
@@ -172,7 +182,7 @@ def migrate_orphaned_images(Note, NotesImage, content_type, model):

    model_ids = sorted({image.model_id for image in orphaned_images})

-    notes = Note.objects.bulk_create(
+    Note.objects.bulk_create(
        [
            Note(
                title="Note",
@@ -186,7 +196,15 @@ def migrate_orphaned_images(Note, NotesImage, content_type, model):
        batch_size=BATCH_SIZE,
    )

-    notes_by_model_id = dict(zip(model_ids, notes))
+    # See the matching comment in create_notes_batch() - bulk_create() doesn't
+    # reliably return populated pks across all backends, so re-fetch the notes
+    # we just created rather than trusting the returned objects.
+    notes_by_model_id = {
+        note.model_id: note
+        for note in Note.objects.filter(
+            model_type=content_type, model_id__in=model_ids
+        )
+    }

    for image in orphaned_images:
        image.note = notes_by_model_id[image.model_id]
@@ -3,7 +3,9 @@
 import datetime
 from tqdm import tqdm

+from django.conf import settings
 from django.db import migrations
+from django.utils import timezone


 def set_creation_date(apps, schema_editor):
@@ -74,7 +76,13 @@ def set_creation_date(apps, schema_editor):
            date_options = [make_aware(d) for d in raw_options if d is not None]

            if date_options:
-                item.creation_date = min(date_options)
+                creation_date = min(date_options)
+
+                # Check if timezone-awarae datetimes are being used in the project settings
+                if not settings.USE_TZ:
+                    creation_date = timezone.make_naive(creation_date, utc)
+
+                item.creation_date = creation_date
                process_item(item)

            progress.update(1)
@@ -434,12 +434,13 @@ class TestCreationDateMigration(MigratorTestCase):
        """Create StockItem entries with varied data to exercise all backfill paths."""
        import datetime

+        from django.conf import settings
        from django.db import connection

        Part = self.old_state.apps.get_model('part', 'part')
        StockItemTracking = self.old_state.apps.get_model('stock', 'stockitemtracking')

-        utc = datetime.timezone.utc
+        utc = datetime.timezone.utc if settings.USE_TZ else None

        part = Part.objects.create(
            name='Migration Test Part', level=0, tree_id=1, lft=0, rght=0
@@ -455,19 +456,23 @@ class TestCreationDateMigration(MigratorTestCase):
            Raw SQL also leaves updated=NULL (no DB-level default for auto_now), which
            makes Scenario 6 a clean "no date sources available" case.
            """
+            insert_sql = """
+                INSERT INTO stock_stockitem
+                    (part_id, quantity, level, tree_id, lft, rght,
+                     status, delete_on_deplete, review_needed, is_building,
+                     link, serial_int, barcode_data, barcode_hash)
+                VALUES (%s, 1, 0, 0, 0, 0, 10, false, false, false, '', 0, '', '')
+                """
            with connection.cursor() as cursor:
-                cursor.execute(
-                    """
-                    INSERT INTO stock_stockitem
-                        (part_id, quantity, level, tree_id, lft, rght,
-                         status, delete_on_deplete, review_needed, is_building,
-                         link, serial_int, barcode_data, barcode_hash)
-                    VALUES (%s, 1, 0, 0, 0, 0, 10, false, false, false, '', 0, '', '')
-                    RETURNING id
-                    """,
-                    [part.pk],
-                )
-                pk = cursor.fetchone()[0]
+                # MySQL has no RETURNING support at all (not even a syntax error
+                # workaround) - fall back to cursor.lastrowid there, and use
+                # RETURNING elsewhere since psycopg's cursor has no lastrowid.
+                if connection.features.can_return_rows_from_bulk_insert:
+                    cursor.execute(insert_sql + 'RETURNING id', [part.pk])
+                    pk = cursor.fetchone()[0]
+                else:
+                    cursor.execute(insert_sql, [part.pk])
+                    pk = cursor.lastrowid
                if stocktake_date is not None:
                    cursor.execute(
                        'UPDATE stock_stockitem SET stocktake_date = %s WHERE id = %s',
@@ -545,12 +550,18 @@ class TestCreationDateMigration(MigratorTestCase):
        """Verify creation_date is correctly backfilled for each scenario."""
        import datetime

+        from django.conf import settings
+
        StockItem = self.new_state.apps.get_model('stock', 'stockitem')
-        utc = datetime.timezone.utc
+        utc = datetime.timezone.utc if settings.USE_TZ else None

        def at_utc(dt):
            """Normalise to UTC and strip sub-second precision for comparison."""
-            return dt.astimezone(utc).replace(microsecond=0)
+            return (
+                dt.astimezone(utc).replace(microsecond=0)
+                if settings.USE_TZ
+                else dt.replace(microsecond=0)
+            )

        # Scenario 1: CREATED tracking entry → creation_date = entry date
        item = StockItem.objects.get(pk=self.pk_s1)
@@ -610,19 +621,24 @@ class TestRemoveMpttFieldsMigration(MigratorTestCase):

        def make_item(quantity, parent_id=None):
            """Insert via raw SQL to avoid duplicate status_custom_key ORM bug."""
+            insert_sql = """
+                INSERT INTO stock_stockitem
+                    (part_id, quantity, level, tree_id, lft, rght,
+                     status, delete_on_deplete, is_building,
+                     link, serial_int, barcode_data, barcode_hash, parent_id)
+                VALUES (%s, %s, 0, 0, 0, 0, 10, false, false, '', 0, '', '', %s)
+                """
            with connection.cursor() as cursor:
-                cursor.execute(
-                    """
-                    INSERT INTO stock_stockitem
-                        (part_id, quantity, level, tree_id, lft, rght,
-                         status, delete_on_deplete, is_building,
-                         link, serial_int, barcode_data, barcode_hash, parent_id)
-                    VALUES (%s, %s, 0, 0, 0, 0, 10, false, false, '', 0, '', '', %s)
-                    RETURNING id
-                    """,
-                    [part.pk, quantity, parent_id],
-                )
-                return cursor.fetchone()[0]
+                # MySQL has no RETURNING support at all - fall back to
+                # cursor.lastrowid there, and use RETURNING elsewhere since
+                # psycopg's cursor has no lastrowid.
+                if connection.features.can_return_rows_from_bulk_insert:
+                    cursor.execute(
+                        insert_sql + 'RETURNING id', [part.pk, quantity, parent_id]
+                    )
+                    return cursor.fetchone()[0]
+                cursor.execute(insert_sql, [part.pk, quantity, parent_id])
+                return cursor.lastrowid

        # Root stock item, with no parent
        root_pk = make_item(100)