Optimize PUI package delivery for package installs (#7655)

* Package frontend in deb

* Add artifact download

* remove 0.8.0 check

* remove array casting

* fix format once more

* another try

* add brackets again

* add version

* and bash

* and shell

* more debuging

* various style fixes

* small fixes

* and ls for prosperity

* debug

* maybe git as source?

* fix download cmd?

* debug a bit

* debug a bit more

* remove sha download - is not working with GHA restrictions

* write version number

* check if a new frontend must be dowloaded

* write versions into frontend packages

* Matmair/issue7338 (#205)

* Package frontend in deb

* Add artifact download

* remove 0.8.0 check

* remove array casting

* fix format once more

* another try

* add brackets again

* add version

* and bash

* and shell

* more debuging

* various style fixes

* small fixes

* and ls for prosperity

* debug

* maybe git as source?

* fix download cmd?

* debug a bit

* debug a bit more

* remove sha download - is not working with GHA restrictions

* write version number

* check if a new frontend must be dowloaded

* write versions into frontend packages

* change ref dir for tests

* add better build logging

* extend task to get ref from package

* fix downloading syntax

* fix name ref

* make more robust

* more logging

* move import

* turn down unzipping noise

* strip content (spaces, newlines)

* add info what happens now

* fix quite flag

* adjust publisher

.github/workflows/qc_checks.yaml

@@ -567,6 +567,8 @@ jobs:
         run: cd src/frontend && yarn install
       - name: Build frontend
         run: cd src/frontend && yarn run compile && yarn run build
+      - name: Write version file - SHA
+        run: cd src/backend/InvenTree/web/static/web/.vite && echo "$GITHUB_SHA" > sha.txt
       - name: Zip frontend
         run: |
           cd src/backend/InvenTree/web/static

.github/workflows/release.yaml

@@ -43,6 +43,10 @@ jobs:
         run: cd src/frontend && yarn install
       - name: Build frontend
         run: cd src/frontend && npm run compile && npm run build
+      - name: Write version file - SHA
+        run: cd src/backend/InvenTree/web/static/web/.vite && echo "$GITHUB_SHA" > sha.txt
+      - name: Write version file - TAG
+        run: cd src/backend/InvenTree/web/static/web/.vite && echo "${{ github.ref_name }}" > tag.txt
       - name: Zip frontend
         run: |
           cd src/backend/InvenTree/web/static/web

contrib/packager.io/before.sh

@@ -5,33 +5,41 @@
 
 set -eu
 
-VERSION="$APP_PKG_VERSION-$APP_PKG_ITERATION"
-echo "Setting VERSION information to $VERSION"
-echo "$VERSION" > VERSION
-
 # The sha is the second element in APP_PKG_ITERATION
+REPO="inventree/InvenTree"
+VERSION="$APP_PKG_VERSION-$APP_PKG_ITERATION"
 SHA=$(echo $APP_PKG_ITERATION | cut -d'.' -f2)
 
 # Download info
-echo "Getting info from github for commit $SHA"
+echo "INFO collection | Getting info from github for commit $SHA"
-curl -L \
+curl -L -s -f \
   -H "Accept: application/vnd.github+json" \
   -H "X-GitHub-Api-Version: 2022-11-28" \
-  https://api.github.com/repos/InvenTree/InvenTree/commits/$SHA > commit.json
+  https://api.github.com/repos/$REPO/commits/$SHA > commit.json
-curl -L \
+echo "INFO collection | Got commit.json with size $(wc -c commit.json)"
+curl -L -s -f \
   -H "Accept: application/vnd.github+json" \
   -H "X-GitHub-Api-Version: 2022-11-28" \
-  https://api.github.com/repos/InvenTree/InvenTree/commits/$SHA/branches-where-head > branches.json
+  https://api.github.com/repos/$REPO/commits/$SHA/branches-where-head > branches.json
+echo "INFO collection | Got branches.json with size $(wc -c branches.json)"
+curl -L -s -f \
+  -H "Accept: application/vnd.github+json" \
+  -H "X-GitHub-Api-Version: 2022-11-28" \
+  https://api.github.com/repos/$REPO/commits/$APP_PKG_VERSION > tag.json
+echo "INFO collection | Got tag.json with size $(wc -c tag.json)"
 
 # Extract info
-echo "Extracting info from github"
+echo "INFO extract | Extracting info from github"
 DATE=$(jq -r '.commit.committer.date' commit.json)
 BRANCH=$(jq -r '.[].name' branches.json)
 NODE_ID=$(jq -r '.node_id' commit.json)
 SIGNATURE=$(jq -r '.commit.verification.signature' commit.json)
+FULL_SHA=$(jq -r '.sha' commit.json)
 
-echo "Write VERSION information"
+echo "INFO write | Write VERSION information"
+echo "$VERSION" > VERSION
 echo "INVENTREE_COMMIT_HASH='$SHA'" >> VERSION
+echo "INVENTREE_COMMIT_SHA='$FULL_SHA'" >> VERSION
 echo "INVENTREE_COMMIT_DATE='$DATE'" >> VERSION
 echo "INVENTREE_PKG_INSTALLER='PKG'" >> VERSION
 echo "INVENTREE_PKG_BRANCH='$BRANCH'" >> VERSION
@@ -39,5 +47,22 @@ echo "INVENTREE_PKG_TARGET='$TARGET'" >> VERSION
 echo "NODE_ID='$NODE_ID'" >> VERSION
 echo "SIGNATURE='$SIGNATURE'" >> VERSION
 
-echo "Written VERSION information"
+echo "INFO write | Written VERSION information"
+echo "### VERSION ###"
 cat VERSION
+echo "### VERSION ###"
+
+# Try to get frontend
+echo "INFO frontend | Trying to get frontend"
+# Check if tag sha is the same as the commit sha
+TAG_SHA=$(jq -r '.sha' tag.json)
+if [ "$TAG_SHA" != "$FULL_SHA" ]; then
+  echo "INFO frontend | Tag sha '$TAG_SHA' is not the same as commit sha $FULL_SHA, can not download frontend"
+else
+  echo "INFO frontend | Getting frontend from github via tag"
+  curl https://github.com/$REPO/releases/download/$APP_PKG_VERSION/frontend-build.zip -L -O -f
+  mkdir -p src/backend/InvenTree/web/static
+  echo "INFO frontend | Unzipping frontend"
+  unzip -qq frontend-build.zip -d src/backend/InvenTree/web/static/web
+  echo "INFO frontend | Unzipped frontend"
+fi

tasks.py

@@ -1272,6 +1272,34 @@ def frontend_download(
 
             handle_extract(dst.name)
 
+    def check_already_current(tag=None, sha=None):
+        """Check if the currently available frontend is already the requested one."""
+        ref = 'tag' if tag else 'commit'
+
+        if tag:
+            current = managePyDir().joinpath('web', 'static', 'web', '.vite', 'tag.txt')
+        elif sha:
+            current = managePyDir().joinpath('web', 'static', 'web', '.vite', 'sha.txt')
+        else:
+            raise ValueError('Either tag or sha needs to be set')
+
+        if not current.exists():
+            print(
+                f'Current frontend information for {ref} is not available - this is expected in some cases'
+            )
+            return False
+
+        current_content = current.read_text().strip()
+        ref_value = tag or sha
+        if current_content == ref_value:
+            print(f'Frontend {ref} is already `{ref_value}`')
+            return True
+        else:
+            print(
+                f'Frontend {ref} is not expected `{ref_value}` but `{current_content}` - new version will be downloaded'
+            )
+            return False
+
     # if zip file is specified, try to extract it directly
     if file:
         handle_extract(file)
@@ -1288,6 +1316,22 @@ def frontend_download(
                 ['git', 'rev-parse', 'HEAD'], encoding='utf-8'
             ).strip()
         except Exception:
+            # .deb Packages contain extra information in the VERSION file
+            version_file = localDir().joinpath('VERSION')
+            if not version_file.exists():
+                return
+            from dotenv import dotenv_values  # noqa: WPS433
+
+            content = dotenv_values(version_file)
+            if (
+                'INVENTREE_PKG_INSTALLER' in content
+                and content['INVENTREE_PKG_INSTALLER'] == 'PKG'
+            ):
+                ref = content.get('INVENTREE_COMMIT_SHA')
+                print(
+                    f'[INFO] Running in package environment, got commit "{ref}" from VERSION file'
+                )
+            else:
                 print("[ERROR] Cannot get current ref via 'git rev-parse HEAD'")
                 return
 
@@ -1297,6 +1341,8 @@ def frontend_download(
     if tag:
         tag = tag.lstrip('v')
         try:
+            if check_already_current(tag=tag):
+                return
             handle_download(
                 f'https://github.com/{repo}/releases/download/{tag}/frontend-build.zip'
             )
@@ -1312,6 +1358,8 @@ Then try continuing by running: invoke frontend-download --file <path-to-downloa
         return
 
     if ref:
+        if check_already_current(sha=ref):
+            return
         # get workflow run from all workflow runs on that particular ref
         workflow_runs = requests.get(
             f'https://api.github.com/repos/{repo}/actions/runs?head_sha={ref}',