diff --git a/.github/workflows/migration_tests.yaml b/.github/workflows/migration_tests.yaml new file mode 100644 index 0000000000..1448133170 --- /dev/null +++ b/.github/workflows/migration_tests.yaml @@ -0,0 +1,241 @@ +# Data migration unit tests, run against every supported database backend + +name: Migration Tests + +on: + push: + branches-ignore: ["l10*", "dependabot/**", "backport/**"] + pull_request: + branches-ignore: ["l10*"] + +env: + python_version: 3.12 + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + INVENTREE_MEDIA_ROOT: /home/runner/work/InvenTree/test_inventree_media + INVENTREE_STATIC_ROOT: /home/runner/work/InvenTree/test_inventree_static + INVENTREE_BACKUP_DIR: /home/runner/work/InvenTree/test_inventree_backup + INVENTREE_SITE_URL: http://localhost:8000 + +permissions: + contents: read + +jobs: + paths-filter: + name: Filter + runs-on: ubuntu-latest + + outputs: + migrations: ${{ steps.filter.outputs.migrations }} + force: ${{ steps.force.outputs.force }} + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3 + id: filter + with: + filters: | + migrations: + - '**/test_migrations.py' + - '**/*migrations.py' + - '**/migrations/**' + - '.github/workflows**' + - 'src/backend/requirements.txt' + - name: Is CI being forced? + run: echo "force=true" >> $GITHUB_OUTPUT + id: force + if: | + contains(github.event.pull_request.labels.*.name, 'dependency') || + contains(github.event.pull_request.labels.*.name, 'full-run') + + sqlite: + name: Tests - Migrations [SQLite] + runs-on: ubuntu-latest + needs: paths-filter + if: ${{ (needs.paths-filter.outputs.force == 'true') || (github.ref == 'refs/heads/master' && needs.paths-filter.outputs.migrations == 'true') }} + + env: + INVENTREE_DB_ENGINE: sqlite3 + INVENTREE_DB_NAME: /home/runner/work/InvenTree/db.sqlite3 + INVENTREE_DEBUG: true + INVENTREE_LOG_LEVEL: WARNING + INVENTREE_PLUGINS_ENABLED: false + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Environment Setup + uses: ./.github/actions/setup + with: + apt-dependency: gettext poppler-utils + dev-install: true + update: true + - name: Run Tests + run: invoke dev.test --check --migrations --report --coverage --translations + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 + if: always() + with: + token: ${{ secrets.CODECOV_TOKEN }} + slug: inventree/InvenTree + flags: migrations-sqlite + + mysql: + name: Tests - Migrations [MySQL] + runs-on: ubuntu-24.04 + needs: paths-filter + if: ${{ (needs.paths-filter.outputs.force == 'true') || (github.ref == 'refs/heads/master' && needs.paths-filter.outputs.migrations == 'true') }} + + env: + INVENTREE_DB_ENGINE: django.db.backends.mysql + INVENTREE_DB_NAME: inventree + INVENTREE_DB_USER: root + INVENTREE_DB_PASSWORD: password + INVENTREE_DB_HOST: "127.0.0.1" + INVENTREE_DB_PORT: 3306 + INVENTREE_DEBUG: true + INVENTREE_LOG_LEVEL: WARNING + INVENTREE_PLUGINS_ENABLED: false + + services: + mysql: + image: mysql:9 + env: + MYSQL_ALLOW_EMPTY_PASSWORD: yes + MYSQL_DATABASE: inventree + MYSQL_USER: inventree + MYSQL_PASSWORD: password + MYSQL_ROOT_PASSWORD: password + options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3 + ports: + - 3306:3306 + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Environment Setup + uses: ./.github/actions/setup + with: + apt-dependency: gettext poppler-utils libmysqlclient-dev + pip-dependency: mysqlclient + dev-install: true + update: true + - name: Run Tests + run: invoke dev.test --check --migrations --report --coverage --translations + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 + if: always() + with: + token: ${{ secrets.CODECOV_TOKEN }} + slug: inventree/InvenTree + flags: migrations-mysql + + postgresql: + name: Tests - Migrations [PostgreSQL] + runs-on: ubuntu-latest + needs: paths-filter + if: ${{ (needs.paths-filter.outputs.force == 'true') || (github.ref == 'refs/heads/master' && needs.paths-filter.outputs.migrations == 'true') }} + + env: + INVENTREE_DB_ENGINE: django.db.backends.postgresql + INVENTREE_DB_NAME: inventree + INVENTREE_DB_USER: inventree + INVENTREE_DB_PASSWORD: password + INVENTREE_DB_HOST: "127.0.0.1" + INVENTREE_DB_PORT: 5432 + INVENTREE_DEBUG: false + INVENTREE_LOG_LEVEL: WARNING + INVENTREE_PLUGINS_ENABLED: false + + services: + postgres: + image: postgres:17 + env: + POSTGRES_USER: inventree + POSTGRES_PASSWORD: password + ports: + - 5432:5432 + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Environment Setup + uses: ./.github/actions/setup + with: + apt-dependency: gettext poppler-utils libpq-dev + pip-dependency: psycopg + dev-install: true + update: true + - name: Run Tests + run: invoke dev.test --check --migrations --report --coverage --translations + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 + if: always() + with: + token: ${{ secrets.CODECOV_TOKEN }} + slug: inventree/InvenTree + flags: migrations-postgresql + + migrations-checks: + name: Tests - Database Migrations + runs-on: ubuntu-latest + needs: paths-filter + if: ${{ (needs.paths-filter.outputs.force == 'true') || (github.ref == 'refs/heads/master' && needs.paths-filter.outputs.migrations == 'true') }} + + env: + INVENTREE_DB_ENGINE: sqlite3 + INVENTREE_DB_NAME: /home/runner/work/InvenTree/db.sqlite3 + INVENTREE_DEBUG: true + INVENTREE_LOG_LEVEL: WARNING + INVENTREE_PLUGINS_ENABLED: false + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + name: Checkout Code + - name: Environment Setup + uses: ./.github/actions/setup + with: + install: true + - name: Fetch Database + run: git clone --depth 1 https://github.com/inventree/test-db ./test-db + + - name: 0.10.0 Database + run: | + rm /home/runner/work/InvenTree/db.sqlite3 + cp test-db/stable_0.10.0.sqlite3 /home/runner/work/InvenTree/db.sqlite3 + chmod +rw /home/runner/work/InvenTree/db.sqlite3 + invoke migrate + + - name: 0.11.0 Database + run: | + rm /home/runner/work/InvenTree/db.sqlite3 + cp test-db/stable_0.11.0.sqlite3 /home/runner/work/InvenTree/db.sqlite3 + chmod +rw /home/runner/work/InvenTree/db.sqlite3 + invoke migrate + + - name: 0.13.5 Database + run: | + rm /home/runner/work/InvenTree/db.sqlite3 + cp test-db/stable_0.13.5.sqlite3 /home/runner/work/InvenTree/db.sqlite3 + chmod +rw /home/runner/work/InvenTree/db.sqlite3 + invoke migrate + + - name: 0.16.0 Database + run: | + rm /home/runner/work/InvenTree/db.sqlite3 + cp test-db/stable_0.16.0.sqlite3 /home/runner/work/InvenTree/db.sqlite3 + chmod +rw /home/runner/work/InvenTree/db.sqlite3 + invoke migrate + + - name: 0.17.0 Database + run: | + rm /home/runner/work/InvenTree/db.sqlite3 + cp test-db/stable_0.17.0.sqlite3 /home/runner/work/InvenTree/db.sqlite3 + chmod +rw /home/runner/work/InvenTree/db.sqlite3 + invoke migrate diff --git a/.github/workflows/qc_checks.yaml b/.github/workflows/qc_checks.yaml index e2a87d4368..64a23d7774 100644 --- a/.github/workflows/qc_checks.yaml +++ b/.github/workflows/qc_checks.yaml @@ -33,7 +33,6 @@ jobs: outputs: server: ${{ steps.filter.outputs.server }} - migrations: ${{ steps.filter.outputs.migrations }} frontend: ${{ steps.filter.outputs.frontend }} api: ${{ steps.filter.outputs.api }} force: ${{ steps.force.outputs.force }} @@ -55,11 +54,6 @@ jobs: - 'src/backend/InvenTree/**' - 'src/backend/requirements.txt' - 'src/backend/requirements-dev.txt' - migrations: - - '**/test_migrations.py' - - '**/migrations/**' - - '.github/workflows**' - - 'src/backend/requirements.txt' api: - 'src/backend/InvenTree/InvenTree/api_version.py' frontend: @@ -559,113 +553,6 @@ jobs: - name: Data Export Test uses: ./.github/actions/migration - migration-tests: - name: Tests - Migrations [PostgreSQL] - runs-on: ubuntu-latest - needs: paths-filter - if: ${{ (needs.paths-filter.outputs.force == 'true') || (github.ref == 'refs/heads/master' && needs.paths-filter.outputs.migrations == 'true') }} - - env: - INVENTREE_DB_ENGINE: django.db.backends.postgresql - INVENTREE_DB_NAME: inventree - INVENTREE_DB_USER: inventree - INVENTREE_DB_PASSWORD: password - INVENTREE_DB_HOST: "127.0.0.1" - INVENTREE_DB_PORT: 5432 - INVENTREE_DEBUG: False - INVENTREE_LOG_LEVEL: WARNING - INVENTREE_PLUGINS_ENABLED: false - - services: - postgres: - image: postgres:17 - env: - POSTGRES_USER: inventree - POSTGRES_PASSWORD: password - ports: - - 5432:5432 - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - name: Environment Setup - uses: ./.github/actions/setup - with: - apt-dependency: gettext poppler-utils libpq-dev - pip-dependency: psycopg - dev-install: true - update: true - - name: Run Tests - run: invoke dev.test --check --migrations --report --coverage --translations - - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 - if: always() - with: - token: ${{ secrets.CODECOV_TOKEN }} - slug: inventree/InvenTree - flags: migrations - - migrations-checks: - name: Tests - Full Migration [SQLite] - runs-on: ubuntu-latest - needs: paths-filter - if: ${{ (needs.paths-filter.outputs.force == 'true') || (github.ref == 'refs/heads/master' && needs.paths-filter.outputs.migrations == 'true') }} - - env: - INVENTREE_DB_ENGINE: sqlite3 - INVENTREE_DB_NAME: /home/runner/work/InvenTree/db.sqlite3 - INVENTREE_DEBUG: true - INVENTREE_LOG_LEVEL: WARNING - INVENTREE_PLUGINS_ENABLED: false - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - name: Checkout Code - - name: Environment Setup - uses: ./.github/actions/setup - with: - install: true - - name: Fetch Database - run: git clone --depth 1 https://github.com/inventree/test-db ./test-db - - - name: 0.10.0 Database - run: | - rm /home/runner/work/InvenTree/db.sqlite3 - cp test-db/stable_0.10.0.sqlite3 /home/runner/work/InvenTree/db.sqlite3 - chmod +rw /home/runner/work/InvenTree/db.sqlite3 - invoke migrate - - - name: 0.11.0 Database - run: | - rm /home/runner/work/InvenTree/db.sqlite3 - cp test-db/stable_0.11.0.sqlite3 /home/runner/work/InvenTree/db.sqlite3 - chmod +rw /home/runner/work/InvenTree/db.sqlite3 - invoke migrate - - - name: 0.13.5 Database - run: | - rm /home/runner/work/InvenTree/db.sqlite3 - cp test-db/stable_0.13.5.sqlite3 /home/runner/work/InvenTree/db.sqlite3 - chmod +rw /home/runner/work/InvenTree/db.sqlite3 - invoke migrate - - - name: 0.16.0 Database - run: | - rm /home/runner/work/InvenTree/db.sqlite3 - cp test-db/stable_0.16.0.sqlite3 /home/runner/work/InvenTree/db.sqlite3 - chmod +rw /home/runner/work/InvenTree/db.sqlite3 - invoke migrate - - - name: 0.17.0 Database - run: | - rm /home/runner/work/InvenTree/db.sqlite3 - cp test-db/stable_0.17.0.sqlite3 /home/runner/work/InvenTree/db.sqlite3 - chmod +rw /home/runner/work/InvenTree/db.sqlite3 - invoke migrate - zizmor: name: Security [Zizmor] runs-on: ubuntu-24.04 diff --git a/src/backend/InvenTree/InvenTree/api_version.py b/src/backend/InvenTree/InvenTree/api_version.py index 43124be61b..c97883c52a 100644 --- a/src/backend/InvenTree/InvenTree/api_version.py +++ b/src/backend/InvenTree/InvenTree/api_version.py @@ -1,11 +1,14 @@ """InvenTree API version information.""" # InvenTree API version -INVENTREE_API_VERSION = 544 +INVENTREE_API_VERSION = 545 """Increment this API version number whenever there is a significant change to the API that any clients need to know about.""" INVENTREE_API_TEXT = """ +v545 -> 2026-09-08 : https://github.com/inventree/InvenTree/pull/12808 + - Ensure consistent ordering of SSO options in API documentation + v544 -> 2026-09-07 : https://github.com/inventree/InvenTree/pull/12807 - Adds filtering by filename on the Attachment list API endpoint diff --git a/src/backend/InvenTree/InvenTree/settings.py b/src/backend/InvenTree/InvenTree/settings.py index 36bad35440..4f39bf4f43 100644 --- a/src/backend/InvenTree/InvenTree/settings.py +++ b/src/backend/InvenTree/InvenTree/settings.py @@ -987,7 +987,7 @@ SOCIAL_BACKENDS = get_setting( ) DEFAULT_SOCIAL = ['saml', 'openid_connect'] -_SOCIAL_BACKENDS = {*DEFAULT_SOCIAL, *SOCIAL_BACKENDS} +_SOCIAL_BACKENDS = list(dict.fromkeys([*DEFAULT_SOCIAL, *SOCIAL_BACKENDS])) # region auth for app in _SOCIAL_BACKENDS: # pragma: no cover diff --git a/src/backend/InvenTree/common/migrations/0051_auto_20260525_0956.py b/src/backend/InvenTree/common/migrations/0051_auto_20260525_0956.py index bf07ca720e..561d135531 100644 --- a/src/backend/InvenTree/common/migrations/0051_auto_20260525_0956.py +++ b/src/backend/InvenTree/common/migrations/0051_auto_20260525_0956.py @@ -117,7 +117,16 @@ def create_notes_batch(Note, NotesImage, content_type, model, instances, unlinke batch_size=BATCH_SIZE, ) - notes_by_model_id = {instance.pk: note for instance, note in zip(instances, notes)} + # bulk_create() only populates the pk on the returned objects on backends + # that support RETURNING on bulk insert. + # So we re-fetch the notes we just created rather than trusting the returned objects. + notes_by_model_id = { + note.model_id: note + for note in Note.objects.filter( + model_type=content_type, + model_id__in=[instance.pk for instance in instances], + ) + } # Images directly linked to one of these instances direct_images = list( @@ -131,7 +140,8 @@ def create_notes_batch(Note, NotesImage, content_type, model, instances, unlinke # Images not directly linked to any instance, but still referenced in the # markdown content itself embedded_images = [] - for instance, note in zip(instances, notes): + for instance in instances: + note = notes_by_model_id[instance.pk] matched = [ image for image in unlinked_images if image.image.url in instance.notes ] @@ -172,7 +182,7 @@ def migrate_orphaned_images(Note, NotesImage, content_type, model): model_ids = sorted({image.model_id for image in orphaned_images}) - notes = Note.objects.bulk_create( + Note.objects.bulk_create( [ Note( title="Note", @@ -186,7 +196,15 @@ def migrate_orphaned_images(Note, NotesImage, content_type, model): batch_size=BATCH_SIZE, ) - notes_by_model_id = dict(zip(model_ids, notes)) + # See the matching comment in create_notes_batch() - bulk_create() doesn't + # reliably return populated pks across all backends, so re-fetch the notes + # we just created rather than trusting the returned objects. + notes_by_model_id = { + note.model_id: note + for note in Note.objects.filter( + model_type=content_type, model_id__in=model_ids + ) + } for image in orphaned_images: image.note = notes_by_model_id[image.model_id] diff --git a/src/backend/InvenTree/stock/migrations/0121_auto_20260526_0849.py b/src/backend/InvenTree/stock/migrations/0121_auto_20260526_0849.py index d6e5087a8e..e9860953d6 100644 --- a/src/backend/InvenTree/stock/migrations/0121_auto_20260526_0849.py +++ b/src/backend/InvenTree/stock/migrations/0121_auto_20260526_0849.py @@ -3,7 +3,9 @@ import datetime from tqdm import tqdm +from django.conf import settings from django.db import migrations +from django.utils import timezone def set_creation_date(apps, schema_editor): @@ -74,7 +76,13 @@ def set_creation_date(apps, schema_editor): date_options = [make_aware(d) for d in raw_options if d is not None] if date_options: - item.creation_date = min(date_options) + creation_date = min(date_options) + + # Check if timezone-awarae datetimes are being used in the project settings + if not settings.USE_TZ: + creation_date = timezone.make_naive(creation_date, utc) + + item.creation_date = creation_date process_item(item) progress.update(1) diff --git a/src/backend/InvenTree/stock/test_migrations.py b/src/backend/InvenTree/stock/test_migrations.py index c5372b7204..0a4bebb8ce 100644 --- a/src/backend/InvenTree/stock/test_migrations.py +++ b/src/backend/InvenTree/stock/test_migrations.py @@ -434,12 +434,13 @@ class TestCreationDateMigration(MigratorTestCase): """Create StockItem entries with varied data to exercise all backfill paths.""" import datetime + from django.conf import settings from django.db import connection Part = self.old_state.apps.get_model('part', 'part') StockItemTracking = self.old_state.apps.get_model('stock', 'stockitemtracking') - utc = datetime.timezone.utc + utc = datetime.timezone.utc if settings.USE_TZ else None part = Part.objects.create( name='Migration Test Part', level=0, tree_id=1, lft=0, rght=0 @@ -455,19 +456,23 @@ class TestCreationDateMigration(MigratorTestCase): Raw SQL also leaves updated=NULL (no DB-level default for auto_now), which makes Scenario 6 a clean "no date sources available" case. """ + insert_sql = """ + INSERT INTO stock_stockitem + (part_id, quantity, level, tree_id, lft, rght, + status, delete_on_deplete, review_needed, is_building, + link, serial_int, barcode_data, barcode_hash) + VALUES (%s, 1, 0, 0, 0, 0, 10, false, false, false, '', 0, '', '') + """ with connection.cursor() as cursor: - cursor.execute( - """ - INSERT INTO stock_stockitem - (part_id, quantity, level, tree_id, lft, rght, - status, delete_on_deplete, review_needed, is_building, - link, serial_int, barcode_data, barcode_hash) - VALUES (%s, 1, 0, 0, 0, 0, 10, false, false, false, '', 0, '', '') - RETURNING id - """, - [part.pk], - ) - pk = cursor.fetchone()[0] + # MySQL has no RETURNING support at all (not even a syntax error + # workaround) - fall back to cursor.lastrowid there, and use + # RETURNING elsewhere since psycopg's cursor has no lastrowid. + if connection.features.can_return_rows_from_bulk_insert: + cursor.execute(insert_sql + 'RETURNING id', [part.pk]) + pk = cursor.fetchone()[0] + else: + cursor.execute(insert_sql, [part.pk]) + pk = cursor.lastrowid if stocktake_date is not None: cursor.execute( 'UPDATE stock_stockitem SET stocktake_date = %s WHERE id = %s', @@ -545,12 +550,18 @@ class TestCreationDateMigration(MigratorTestCase): """Verify creation_date is correctly backfilled for each scenario.""" import datetime + from django.conf import settings + StockItem = self.new_state.apps.get_model('stock', 'stockitem') - utc = datetime.timezone.utc + utc = datetime.timezone.utc if settings.USE_TZ else None def at_utc(dt): """Normalise to UTC and strip sub-second precision for comparison.""" - return dt.astimezone(utc).replace(microsecond=0) + return ( + dt.astimezone(utc).replace(microsecond=0) + if settings.USE_TZ + else dt.replace(microsecond=0) + ) # Scenario 1: CREATED tracking entry → creation_date = entry date item = StockItem.objects.get(pk=self.pk_s1) @@ -610,19 +621,24 @@ class TestRemoveMpttFieldsMigration(MigratorTestCase): def make_item(quantity, parent_id=None): """Insert via raw SQL to avoid duplicate status_custom_key ORM bug.""" + insert_sql = """ + INSERT INTO stock_stockitem + (part_id, quantity, level, tree_id, lft, rght, + status, delete_on_deplete, is_building, + link, serial_int, barcode_data, barcode_hash, parent_id) + VALUES (%s, %s, 0, 0, 0, 0, 10, false, false, '', 0, '', '', %s) + """ with connection.cursor() as cursor: - cursor.execute( - """ - INSERT INTO stock_stockitem - (part_id, quantity, level, tree_id, lft, rght, - status, delete_on_deplete, is_building, - link, serial_int, barcode_data, barcode_hash, parent_id) - VALUES (%s, %s, 0, 0, 0, 0, 10, false, false, '', 0, '', '', %s) - RETURNING id - """, - [part.pk, quantity, parent_id], - ) - return cursor.fetchone()[0] + # MySQL has no RETURNING support at all - fall back to + # cursor.lastrowid there, and use RETURNING elsewhere since + # psycopg's cursor has no lastrowid. + if connection.features.can_return_rows_from_bulk_insert: + cursor.execute( + insert_sql + 'RETURNING id', [part.pk, quantity, parent_id] + ) + return cursor.fetchone()[0] + cursor.execute(insert_sql, [part.pk, quantity, parent_id]) + return cursor.lastrowid # Root stock item, with no parent root_pk = make_item(100)