feat: matrix testing for python 3.14 (#10904)

* feat: matrix testing

* remove unneeded args

* remove hashes for now

* remove require-hashes for now

* try manually fixing cffi for now

* remove wrong cache syntax

* remove cache for now

* move to 3.14?

* make version specific install req

* re-add hashes

* Revert "remove require-hashes for now"

This reverts commit 7a7e12130a.

* also compile requirements-dev

* move more install commands

* fix generation setting

* more logging

* remove cache exception

* remove cffi fix

* remove hash inforcement for now

* remove dev for now

* default to 3.14 to val

* Revert "default to 3.14 to val"

This reverts commit 7a28e46604.

* fix constraints

* remove pin requirement from plugins.txt

* do not use uv for now

* do not use uv for now

* fix test file format

* revert small change

* use 3.14 for benchmarks

* keep tests on 3.11

* use invoke for install

* try 3.14 for performance testing again

* prolong time for slower CI on 3.14

* adjust dep versions

* Keep performance ci on 3.12

Updated the Python version in the QC checks workflow.

tasks.py

@@ -417,6 +417,54 @@ def manage(c, cmd, pty: bool = False, env=None):
     run(c, f'python3 manage.py {cmd}', manage_py_dir(), pty, env)
 
 
+def run_install(
+    c,
+    uv: bool,
+    install_file: Path,
+    run_preflight=True,
+    version_check=False,
+    pinned=True,
+):
+    """Run the installation of python packages from a requirements file."""
+    if version_check:
+        # Test if there is a version specific requirements file
+        sys_ver_s = python_version().split('.')
+        sys_string = f'{sys_ver_s[0]}.{sys_ver_s[1]}'
+        install_file_vers = install_file.parent.joinpath(
+            f'{install_file.stem}-{sys_string}{install_file.suffix}'
+        )
+        if install_file_vers.exists():
+            install_file = install_file_vers
+            info(f"Using version-specific requirements file '{install_file_vers}'")
+
+    info(f"Installing required python packages from '{install_file}'")
+    if not Path(install_file).is_file():
+        raise FileNotFoundError(f"Requirements file '{install_file}' not found")
+
+    # Install required Python packages with PIP
+    if not uv:
+        if run_preflight:
+            run(
+                c,
+                'pip3 install --no-cache-dir --disable-pip-version-check -U pip setuptools',
+            )
+        run(
+            c,
+            f'pip3 install --no-cache-dir --disable-pip-version-check -U {"--require-hashes" if pinned else ""} -r {install_file}',
+        )
+    else:
+        if run_preflight:
+            run(
+                c,
+                'pip3 install --no-cache-dir --disable-pip-version-check -U uv setuptools',
+            )
+            info('Installed package manager')
+        run(
+            c,
+            f'uv pip install -U {"--require-hashes" if pinned else ""} -r {install_file}',
+        )
+
+
 def yarn(c, cmd):
     """Runs a given command against the yarn package manager.
 
@@ -492,16 +540,7 @@ def plugins(c, uv=False):
         get_plugin_file,
     )
 
-    plugin_file = get_plugin_file()
-
-    info(f"Installing plugin packages from '{plugin_file}'")
-
-    # Install the plugins
-    if not uv:
-        run(c, f"pip3 install --disable-pip-version-check -U -r '{plugin_file}'")
-    else:
-        run(c, 'pip3 install --no-cache-dir --disable-pip-version-check uv')
-        run(c, f"uv pip install -r '{plugin_file}'")
+    run_install(c, uv, get_plugin_file(), run_preflight=False, pinned=False)
 
     # Collect plugin static files
     manage(c, 'collectplugins')
@@ -511,35 +550,26 @@ def plugins(c, uv=False):
     help={
         'uv': 'Use UV package manager (experimental)',
         'skip_plugins': 'Skip plugin installation',
+        'dev': 'Install development requirements instead of production requirements',
     }
 )
 @state_logger('TASK02')
-def install(c, uv=False, skip_plugins=False):
+def install(c, uv=False, skip_plugins=False, dev=False):
     """Installs required python packages."""
+    if dev:
+        run_install(
+            c,
+            uv,
+            local_dir().joinpath('src/backend/requirements-dev.txt'),
+            version_check=True,
+        )
+        success('Dependency installation complete')
+        return
+
     # Ensure path is relative to *this* directory
-    INSTALL_FILE = local_dir().joinpath('src/backend/requirements.txt')
-
-    info(f"Installing required python packages from '{INSTALL_FILE}'")
-
-    if not Path(INSTALL_FILE).is_file():
-        raise FileNotFoundError(f"Requirements file '{INSTALL_FILE}' not found")
-
-    # Install required Python packages with PIP
-    if not uv:
-        run(
-            c,
-            'pip3 install --no-cache-dir --disable-pip-version-check -U pip setuptools',
-        )
-        run(
-            c,
-            f'pip3 install --no-cache-dir --disable-pip-version-check -U --require-hashes -r {INSTALL_FILE}',
-        )
-    else:
-        run(
-            c,
-            'pip3 install --no-cache-dir --disable-pip-version-check -U uv setuptools',
-        )
-        run(c, f'uv pip install -U --require-hashes  -r {INSTALL_FILE}')
+    run_install(
+        c, uv, local_dir().joinpath('src/backend/requirements.txt'), version_check=True
+    )
 
     # Run plugins install
     if not skip_plugins:
@@ -558,10 +588,8 @@ def install(c, uv=False, skip_plugins=False):
 @task(help={'tests': 'Set up test dataset at the end'})
 def setup_dev(c, tests=False):
     """Sets up everything needed for the dev environment."""
-    info("Installing required python packages from 'src/backend/requirements-dev.txt'")
-
     # Install required Python packages with PIP
-    run(c, 'pip3 install -U --require-hashes -r src/backend/requirements-dev.txt')
+    install(c, uv=False, skip_plugins=True, dev=True)
 
     # Install pre-commit hook
     info('Installing pre-commit for checks before git commits...')