fix(forntend): generate UI coverage again (#12066)

* Attempt to fix UI coverage

* Update CI workflows:

- use test sharding
- Only upload coverage on master

* Restore line

* Simplify test

* Simplify test matrix

* Fix env vars

* Adjust matrix

* Adjust output names

* Fix paths

* Simplify qc_checks

* Revert missing line

* Simplify coverage calls

* Run firefox test against port 8000

* Fix VITE_COVERAGE env var

* Capture browser name in report output

* Increase timeout again

* Enhanced feedback from playwright startup

* Split UI checks into separate file

* Fix workflow deps

* Shard chromium build

* Adjust reporter type

* Reduce uncessesary build steps

* Tweak paths filter

* Reduce retries

* Also generate HTML reports

* Tweak reporter output

* Fix custom splash URLs

* Fix envs for customization tests

* Shard the firefox runner too

* Ignore customization tests for firefox too

* Don't upload if tests fail

* Fix triggers

* Remove merged test coverage

* Pin download action

* Error if no artifact files found

* Update ignore dirs

* Adjust baseFixtures

* Fix for teardown in baseFixtures.ts

* Fix path for coverage files

* include hidden files

src/frontend/playwright.config.ts

@@ -3,9 +3,6 @@ import { defineConfig, devices } from '@playwright/test';
 // Detect if running in CI
 const IS_CI = !!process.env.CI;
 
-const MAX_WORKERS: number = 3;
-const MAX_RETRIES: number = 3;
-
 /* We optionally spin-up services based on the testing mode:
  *
  * Local Development:
@@ -26,21 +23,28 @@ const MAX_RETRIES: number = 3;
  * - WORKERS = 1 (to avoid conflicts with HMR)
  */
 
-const BASE_URL: string = IS_CI
-  ? 'http://localhost:8000'
-  : 'http://localhost:5173';
+const BASE_URL: string =
+  process.env.PLAYWRIGHT_BASE_URL || 'http://localhost:5173';
+
+// If running in "production" mode, we can use multiple workers to speed up the tests
+const MAX_WORKERS: number = BASE_URL.endsWith('8000') ? 3 : 1;
+const MAX_RETRIES: number = IS_CI ? 1 : 2;
 
 console.log('Running Playwright Tests:');
 console.log('- Base URL:', BASE_URL);
+console.log('- Max Workers:', MAX_WORKERS);
+console.log('- Max Retries:', MAX_RETRIES);
 
 export default defineConfig({
   testDir: './tests',
   fullyParallel: false,
   timeout: 90000,
   forbidOnly: !!IS_CI,
-  retries: IS_CI ? MAX_RETRIES : 0,
-  workers: IS_CI ? MAX_WORKERS : 1,
-  reporter: IS_CI ? [['html', { open: 'never' }], ['github']] : 'list',
+  retries: MAX_RETRIES,
+  workers: MAX_WORKERS,
+  reporter: IS_CI
+    ? [['html', { open: 'never' }], ['blob'], ['github']]
+    : 'list',
 
   /* Configure projects for major browsers */
   projects: [
@@ -57,13 +61,6 @@ export default defineConfig({
         ...devices['Desktop Firefox']
       },
       testIgnore: /customization/ // Ignore all tests in the "customization" folder for this project
-    },
-    {
-      name: 'customization',
-      use: {
-        ...devices['Desktop Firefox']
-      },
-      testIgnore: /pui_.*\.spec\.ts/ // Ignore all "pui_*.spec.ts" tests for this project
     }
   ],
 
@@ -89,7 +86,9 @@ export default defineConfig({
         INVENTREE_CORS_ORIGIN_ALLOW_ALL: 'True',
         INVENTREE_COOKIE_SAMESITE: 'False',
         INVENTREE_LOGIN_ATTEMPTS: '100',
-        INVENTREE_PLUGINS_MANDATORY: 'samplelocate'
+        INVENTREE_PLUGINS_MANDATORY: 'samplelocate',
+        INVENTREE_CUSTOM_SPLASH: 'img/playwright_custom_splash.png',
+        INVENTREE_CUSTOM_LOGO: 'img/playwright_custom_logo.png'
       },
       url: 'http://localhost:8000/api/',
       reuseExistingServer: IS_CI,

src/frontend/tests/baseFixtures.ts

@@ -2,9 +2,14 @@ import * as crypto from 'node:crypto';
 import * as fs from 'node:fs';
 import os from 'node:os';
 import * as path from 'node:path';
-import { test as baseTest } from '@playwright/test';
+import { fileURLToPath } from 'node:url';
+import { type BrowserContext, test as baseTest } from '@playwright/test';
 
-const istanbulCLIOutput = path.join(process.cwd(), '.nyc_output');
+const frontendDir = path.resolve(
+  path.dirname(fileURLToPath(import.meta.url)),
+  '..'
+);
+const istanbulCLIOutput = path.join(frontendDir, '.nyc_output');
 const platform = os.platform();
 let systemKeyVar: string;
 if (platform === 'darwin') {
@@ -19,16 +24,16 @@ export function generateUUID(): string {
   return crypto.randomBytes(16).toString('hex');
 }
 
-export const test = baseTest.extend({
-  context: async ({ context }, use) => {
-    await context.addInitScript(() =>
-      window.addEventListener('beforeunload', () =>
-        (window as any).collectIstanbulCoverage(
-          JSON.stringify((window as any).__coverage__)
-        )
+async function setupCoverageCollection(context: BrowserContext) {
+  await context.addInitScript(() =>
+    window.addEventListener('beforeunload', () =>
+      (window as any).collectIstanbulCoverage?.(
+        JSON.stringify((window as any).__coverage__)
       )
-    );
-    await fs.promises.mkdir(istanbulCLIOutput, { recursive: true });
+    )
+  );
+  await fs.promises.mkdir(istanbulCLIOutput, { recursive: true });
+  try {
     await context.exposeFunction(
       'collectIstanbulCoverage',
       (coverageJSON: string) => {
@@ -42,18 +47,68 @@ export const test = baseTest.extend({
           );
       }
     );
+  } catch {
+    // already exposed on this context (e.g. called twice for same context)
+  }
+}
+
+async function collectCoverageFromContext(context: BrowserContext) {
+  await Promise.allSettled(
+    context.pages().map(async (page) => {
+      try {
+        await Promise.race([
+          page.evaluate(() =>
+            (window as any).collectIstanbulCoverage?.(
+              JSON.stringify((window as any).__coverage__)
+            )
+          ),
+          new Promise((_, reject) =>
+            setTimeout(
+              () => reject(new Error('Coverage collection timeout')),
+              2000
+            )
+          )
+        ]);
+      } catch {
+        // page may already be closed or script execution can be blocked during teardown
+      }
+    })
+  );
+}
+
+export const test = baseTest.extend<{}, {}>({
+  // Wrap browser.newPage so contexts created via doCachedLogin also get coverage
+  browser: [
+    async ({ browser }, use) => {
+      const origNewPage = browser.newPage.bind(browser);
+      (browser as any).newPage = async (
+        options?: Parameters<typeof browser.newPage>[0]
+      ) => {
+        const page = await origNewPage(options);
+        await setupCoverageCollection(page.context());
+        return page;
+      };
+      try {
+        await use(browser);
+      } finally {
+        (browser as any).newPage = origNewPage;
+        for (const context of browser.contexts()) {
+          await collectCoverageFromContext(context);
+        }
+      }
+    },
+    { scope: 'worker' }
+  ],
+
+  context: async ({ context }, use) => {
+    await setupCoverageCollection(context);
     await use(context);
-    for (const page of context.pages()) {
-      await page.evaluate(() =>
-        (window as any).collectIstanbulCoverage(
-          JSON.stringify((window as any).__coverage__)
-        )
-      );
-    }
+    await collectCoverageFromContext(context);
   },
+
   // Ensure no errors are thrown in the console
   page: async ({ page }, use) => {
-    const messages = [];
+    const messages: any[] = [];
     page.on('console', (msg) => {
       const url = msg.location().url;
       if (

src/frontend/tests/customization/customization.spec.ts

@@ -1,11 +1,16 @@
-import test, { expect } from '@playwright/test';
+import { expect, test } from '../baseFixtures';
 import { navigate } from '../helpers';
 
 /**
  * Tests for user interface customization functionality.
  *
  * Note: The correct environment variables must be set for these tests to work correctly. See "playwright.config.ts" for details.
- * These tests are designed to run in CI environments where specific environment variables are set to enable custom logos and splash screens. The tests verify that these customizations are correctly applied in the user interface.
+ * These tests are designed to run in CI environments where specific environment variables are set to enable custom logos and splash screens.
+ * The tests verify that these customizations are correctly applied in the user interface.
+ *
+ * If you are running these tests locally, ensure that you have the appropriate environment variables set to enable the customizations.
+ * You may need to modify the "webServer" configuration in "playwright.config.ts" to include the necessary environment variables for local testing.
+ *
  */
 
 test('Customization - Splash', async ({ page }) => {

src/frontend/tests/pages/pui_return.spec.ts

@@ -1,4 +1,4 @@
-import test from '@playwright/test';
+import { test } from '../baseFixtures';
 import { loadTab } from '../helpers';
 import { doCachedLogin } from '../login';
 

src/frontend/tests/pui_exporting.spec.ts

@@ -1,4 +1,4 @@
-import test from '@playwright/test';
+import { test } from './baseFixtures';
 import { stevenuser } from './defaults';
 import { globalSearch, loadTab, navigate } from './helpers';
 import { doCachedLogin } from './login';

src/frontend/tests/pui_forms.spec.ts

@@ -1,6 +1,6 @@
-/** Unit tests for form validation, rendering, etc */
-import { expect, test } from 'playwright/test';
 import { createApi } from './api';
+/** Unit tests for form validation, rendering, etc */
+import { expect, test } from './baseFixtures';
 import { stevenuser } from './defaults';
 import { navigate } from './helpers';
 import { doCachedLogin } from './login';

src/frontend/tests/pui_importing.spec.ts

@@ -1,4 +1,4 @@
-import test from '@playwright/test';
+import { test } from './baseFixtures';
 import { stevenuser } from './defaults';
 import { doCachedLogin } from './login';
 

src/frontend/tests/pui_login.spec.ts

@@ -13,7 +13,8 @@ test('Login - Failures', async ({ page }) => {
     await page.getByRole('button', { name: 'Log In' }).click();
     await page.getByText('Login failed', { exact: true }).waitFor();
     await page.getByText('Check your input and try again').first().waitFor();
-    await page.locator('#login').getByRole('button').click();
+
+    await page.reload();
   };
 
   // Navigate to the 'login' page

src/frontend/tests/pui_machines.spec.ts

@@ -1,4 +1,4 @@
-import test from 'playwright/test';
+import { test } from './baseFixtures';
 import { adminuser } from './defaults';
 import { clickOnRowMenu, navigate } from './helpers';
 import { doCachedLogin } from './login';

src/frontend/tests/pui_permissions.spec.ts

@@ -2,7 +2,7 @@
  * Tests for UI permissions checks
  */
 
-import test from '@playwright/test';
+import { test } from './baseFixtures';
 import { adminuser, readeruser } from './defaults';
 import { clickOnRowMenu, loadTab } from './helpers';
 import { doCachedLogin } from './login';

src/frontend/tests/pui_plugins.spec.ts

@@ -1,4 +1,4 @@
-import test from 'playwright/test';
+import { test } from './baseFixtures';
 
 import { adminuser } from './defaults.js';
 import {

src/frontend/vite.config.ts

@@ -47,7 +47,7 @@ export default defineConfig(({ command, mode }) => {
       }),
       istanbul({
         include: ['src/*', 'lib/*'],
-        exclude: ['node_modules', 'test/'],
+        exclude: ['node_modules/', 'playwright/', 'tests/'],
         extension: ['.js', '.ts', '.tsx'],
         requireEnv: true
       }),