refactor(docs): simplify docs files / pipelines (#9633)

* refactor(doc): only use one command

* restructure

* move all generated helpers to a dedicated directory

* move to pathlib

* and more pathlib

* add empty generated folder

.github/workflows/release.yaml

@@ -121,11 +121,7 @@ jobs:
           pip install --require-hashes -r docs/requirements.txt
       - name: Build documentation
         run: |
-          invoke migrate
+          invoke build-docs --mkdocs
-          invoke int.export-definitions --basedir "docs"
-          invoke dev.schema --filename docs/schema.yml --ignore-warnings
-          python docs/extract_schema.py docs/schema.yml
-          mkdocs build  -f docs/mkdocs.yml
       - name: Zip build docs
         run: |
           cd docs/site

.gitignore

@@ -113,8 +113,3 @@ api.yaml
 # web frontend (static files)
 src/backend/InvenTree/web/static
 InvenTree/web/static
-
-# Exported interim files
-docs/schema.yml
-docs/docs/api/*.yml
-docs/docs/api/schema/*.yml

docs/.gitignore

@@ -13,20 +13,5 @@ site/
 # Generated API schema files
 docs/api/schema/*.yml
 
-# Temporary cache files
-url_cache.txt
-invoke-commands.txt
-
-# Temp files
-releases.json
-versions.json
-inventree_filters.yml
-inventree_settings.json
-inventree_tags.yml
-
 .vscode/
-
+generated/
-inventree_filters.yml
-inventree_report_context.json
-inventree_settings.json
-inventree_tags.yml

docs/docs/hooks.py

@@ -5,9 +5,12 @@ import os
 import re
 from datetime import datetime
 from distutils.version import StrictVersion
+from pathlib import Path
 
 import requests
 
+here = Path(__file__).parent
+
 
 def fetch_rtd_versions():
     """Get a list of RTD docs versions to build the version selector."""
@@ -77,7 +80,7 @@ def fetch_rtd_versions():
             'aliases': [],
         })
 
-    output_filename = os.path.join(os.path.dirname(__file__), 'versions.json')
+    output_filename = here.joinpath('versions.json')
 
     print('Discovered the following versions:')
     print(versions)
@@ -92,11 +95,11 @@ def get_release_data():
     - First look to see if 'releases.json' file exists
     - If data does not exist in this file, request via the github API
     """
-    json_file = os.path.join(os.path.dirname(__file__), 'releases.json')
+    json_file = here.parent.joinpath('generated', 'releases.json')
 
     releases = []
 
-    if os.path.exists(json_file):
+    if json_file.exists():
         # Release information has been cached to file
 
         print("Loading release information from 'releases.json'")
@@ -165,7 +168,7 @@ def on_config(config, *args, **kwargs):
     # Note: version selection is handled by RTD internally
     # Check for 'versions.json' file
     # If it does not exist, we need to fetch it from the RTD API
-    # if os.path.exists(os.path.join(os.path.dirname(__file__), 'versions.json')):
+    # if here.joinpath('versions.json').exists():
     #    print("Found 'versions.json' file")
     # else:
     #    fetch_rtd_versions()
@@ -230,9 +233,9 @@ def on_config(config, *args, **kwargs):
             continue
 
         # Check if there is a local file with release information
-        local_path = os.path.join(os.path.dirname(__file__), 'releases', f'{tag}.md')
+        local_path = here.joinpath('releases', f'{tag}.md')
 
-        if os.path.exists(local_path):
+        if local_path.exists():
             item['local_path'] = local_path
 
         # Extract the date

docs/generated/.gitignore

@@ -0,0 +1 @@
+*

docs/main.py

@@ -4,6 +4,7 @@ import json
 import os
 import subprocess
 import textwrap
+from pathlib import Path
 from typing import Literal, Optional
 
 import requests
@@ -37,29 +38,29 @@ global FILTERS
 global REPORT_CONTEXT
 
 # Read in the InvenTree settings file
-here = os.path.dirname(__file__)
+here = Path(__file__).parent
-settings_file = os.path.join(here, 'inventree_settings.json')
+gen_base = here.joinpath('generated')
 
-with open(settings_file, encoding='utf-8') as sf:
+with open(gen_base.joinpath('inventree_settings.json'), encoding='utf-8') as sf:
     settings = json.load(sf)
 
     GLOBAL_SETTINGS = settings['global']
     USER_SETTINGS = settings['user']
 
 # Tags
-with open(os.path.join(here, 'inventree_tags.yml'), encoding='utf-8') as f:
+with open(gen_base.joinpath('inventree_tags.yml'), encoding='utf-8') as f:
     TAGS = yaml.load(f, yaml.BaseLoader)
 # Filters
-with open(os.path.join(here, 'inventree_filters.yml'), encoding='utf-8') as f:
+with open(gen_base.joinpath('inventree_filters.yml'), encoding='utf-8') as f:
     FILTERS = yaml.load(f, yaml.BaseLoader)
 # Report context
-with open(os.path.join(here, 'inventree_report_context.json'), encoding='utf-8') as f:
+with open(gen_base.joinpath('inventree_report_context.json'), encoding='utf-8') as f:
     REPORT_CONTEXT = json.load(f)
 
 
 def get_repo_url(raw=False):
     """Return the repository URL for the current project."""
-    mkdocs_yml = os.path.join(os.path.dirname(__file__), 'mkdocs.yml')
+    mkdocs_yml = here.joinpath('mkdocs.yml')
 
     with open(mkdocs_yml, encoding='utf-8') as f:
         mkdocs_config = yaml.safe_load(f)
@@ -77,10 +78,10 @@ def check_link(url) -> bool:
     We allow a number attempts and a lengthy timeout,
     as we do not want false negatives.
     """
-    CACHE_FILE = os.path.join(os.path.dirname(__file__), 'url_cache.txt')
+    CACHE_FILE = gen_base.joinpath('url_cache.txt')
 
     # Keep a local cache file of URLs we have already checked
-    if os.path.exists(CACHE_FILE):
+    if CACHE_FILE.exists():
         with open(CACHE_FILE, encoding='utf-8') as f:
             cache = f.read().splitlines()
 
@@ -93,6 +94,7 @@ def check_link(url) -> bool:
 
     while attempts > 0:
         response = requests.head(url, timeout=5000)
+
         # Ensure GH is not causing issues
         if response.status_code in (200, 429):
             # Update the cache file
@@ -147,13 +149,8 @@ def define_env(env):
             dirname = dirname[1:]
 
         # This file exists at ./docs/main.py, so any directory we link to must be relative to the top-level directory
-        here = os.path.dirname(__file__)
+        directory = here.parent.joinpath(dirname)
-        root = os.path.abspath(os.path.join(here, '..'))
+        if not directory.exists() or not directory.is_dir():
-
-        directory = os.path.join(root, dirname)
-        directory = os.path.abspath(directory)
-
-        if not os.path.exists(directory) or not os.path.isdir(directory):
             raise FileNotFoundError(f'Source directory {dirname} does not exist.')
 
         repo_url = get_repo_url()
@@ -188,12 +185,8 @@ def define_env(env):
             filename = filename[1:]
 
         # This file exists at ./docs/main.py, so any file we link to must be relative to the top-level directory
-        here = os.path.dirname(__file__)
+        file_path = here.parent.joinpath(filename)
-        root = os.path.abspath(os.path.join(here, '..'))
+        if not file_path.exists():
-
-        file_path = os.path.join(root, filename)
-
-        if not os.path.exists(file_path):
             raise FileNotFoundError(f'Source file {filename} does not exist.')
 
         # Construct repo URL
@@ -214,11 +207,8 @@ def define_env(env):
     @env.macro
     def invoke_commands():
         """Provides an output of the available commands."""
-        here = os.path.dirname(__file__)
+        tasks = here.parent.joinpath('tasks.py')
-        base = os.path.join(here, '..')
+        output = gen_base.joinpath('invoke-commands.txt')
-        base = os.path.abspath(base)
-        tasks = os.path.join(base, 'tasks.py')
-        output = os.path.join(here, 'invoke-commands.txt')
 
         command = f'invoke -f {tasks} --list > {output}'
 
@@ -232,17 +222,15 @@ def define_env(env):
     @env.macro
     def listimages(subdir):
         """Return a listing of all asset files in the provided subdir."""
-        here = os.path.dirname(__file__)
+        directory = here.joinpath('docs', 'assets', 'images', subdir)
-
-        directory = os.path.join(here, 'docs', 'assets', 'images', subdir)
 
         assets = []
 
         allowed = ['.png', '.jpg']
 
-        for asset in os.listdir(directory):
+        for asset in directory.iterdir():
-            if any(asset.endswith(x) for x in allowed):
+            if any(str(asset).endswith(x) for x in allowed):
-                assets.append(os.path.join(subdir, asset))
+                assets.append(str(subdir / asset.relative_to(directory)))
 
         return assets
 
@@ -255,11 +243,9 @@ def define_env(env):
             title: The title of the collapse block in the documentation
             fmt: The format of the included file (e.g., 'python', 'html', etc.)
         """
-        here = os.path.dirname(__file__)
+        path = here.parent.joinpath(filename)
-        path = os.path.join(here, '..', filename)
-        path = os.path.abspath(path)
 
-        if not os.path.exists(path):
+        if not path.exists():
             raise FileNotFoundError(f'Required file {path} does not exist.')
 
         with open(path, encoding='utf-8') as f:
@@ -276,10 +262,8 @@ def define_env(env):
     @env.macro
     def templatefile(filename):
         """Include code for a provided template file."""
-        base = os.path.basename(filename)
+        base = Path(filename).name
-        fn = os.path.join(
+        fn = Path('src', 'backend', 'InvenTree', 'report', 'templates', filename)
-            'src', 'backend', 'InvenTree', 'report', 'templates', filename
-        )
 
         return includefile(fn, f'Template: {base}', fmt='html')
 

readthedocs.yml

@@ -14,9 +14,6 @@ build:
     python: "3.9"
   jobs:
     post_install:
-      - echo "Generating API schema file"
       - pip install -U invoke
-      - invoke migrate
+      - echo "Generating API schema file"
-      - invoke int.export-definitions --basedir "docs"
+      - invoke build-docs
-      - invoke dev.schema --filename docs/schema.yml --ignore-warnings
-      - python docs/extract_schema.py docs/schema.yml

tasks.py

@@ -1325,12 +1325,13 @@ def export_definitions(c, basedir: str = ''):
     """Export various definitions."""
     if basedir != '' and basedir.endswith('/') is False:
         basedir += '/'
+    base_path = Path(basedir, 'generated').resolve()
 
     filenames = [
-        Path(basedir + 'inventree_settings.json').resolve(),
+        base_path.joinpath('inventree_settings.json'),
-        Path(basedir + 'inventree_tags.yml').resolve(),
+        base_path.joinpath('inventree_tags.yml'),
-        Path(basedir + 'inventree_filters.yml').resolve(),
+        base_path.joinpath('inventree_filters.yml'),
-        Path(basedir + 'inventree_report_context.json').resolve(),
+        base_path.joinpath('inventree_report_context.json'),
     ]
 
     info('Exporting definitions...')
@@ -1704,6 +1705,14 @@ via your signed in browser, or consider using a point release download via invok
         )
 
 
+def doc_schema(c):
+    """Generate schema documentation for the API."""
+    schema(
+        c, ignore_warnings=True, overwrite=True, filename='docs/generated/schema.yml'
+    )
+    run(c, 'python docs/extract_schema.py docs/generated/schema.yml')
+
+
 @task(
     help={
         'address': 'Host and port to run the server on (default: localhost:8080)',
@@ -1716,13 +1725,27 @@ def docs_server(c, address='localhost:8080', compile_schema=False):
     export_definitions(c, basedir='docs')
 
     if compile_schema:
-        # Build the schema docs first
+        doc_schema(c)
-        schema(c, ignore_warnings=True, overwrite=True, filename='docs/schema.yml')
-        run(c, 'python docs/extract_schema.py docs/schema.yml')
 
     run(c, f'mkdocs serve -a {address} -f docs/mkdocs.yml')
 
 
+@task(
+    help={'mkdocs': 'Build the documentation using mkdocs at the end (default: False)'}
+)
+def build_docs(c, mkdocs=False):
+    """Build the required documents for building the docs. Optionally build the documentation using mkdocs."""
+    migrate(c)
+    export_definitions(c, basedir='docs')
+    doc_schema(c)
+
+    if mkdocs:
+        run(c, 'mkdocs build  -f docs/mkdocs.yml')
+        info('Documentation build complete')
+    else:
+        info('Documentation build complete, but mkdocs not requested')
+
+
 @task
 def clear_generated(c):
     """Clear generated files from `invoke update`."""
@@ -1787,6 +1810,7 @@ ns = Collection(
     version,
     wait,
     worker,
+    build_docs,
 )
 
 ns.add_collection(development, 'dev')