Docs: post process to adjust URLs in index.html files for hosting in subdirectories (#3842)

* Docs: post process output to fix paths so that many mdbooks can be served from sub-directories

* Prevent theme from being modified

* Upload docs to Vercel

* Post process docs

* Process local links

* Docs: only process `index.html` files from the root,

All other files have the correct relative paths to serve assets properly and link to files relatively.

---------

Co-authored-by: Mark Sinclair <mmsinclair@users.noreply.github.com>

.github/workflows/cd-docs.yml

@@ -38,6 +38,7 @@ jobs:
       - name: Build all projects in documentation/ & move to ~/dist/docs/
         run: cd documentation && ./build_all_to_dist.sh
         continue-on-error: false
+
       - name: Deploy branch master to dev
         continue-on-error: true
         uses: easingthemes/ssh-deploy@main
@@ -49,6 +50,7 @@ jobs:
           REMOTE_USER: ${{ secrets.CD_WWW_REMOTE_USER }}
           TARGET: ${{ secrets.CD_WWW_REMOTE_TARGET }}/
           EXCLUDE: "/node_modules/"
+
       - name: Deploy branch master to prod
         if: github.ref == 'refs/heads/master'
         uses: easingthemes/ssh-deploy@main
@@ -60,6 +62,44 @@ jobs:
           REMOTE_USER: ${{ secrets.CD_WWW_REMOTE_USER }}
           TARGET: ${{ secrets.CD_WWW_REMOTE_TARGET }}/
           EXCLUDE: "/node_modules/"
+
+      - name: Post process
+        run: cd documentation && ./post_process.sh
+        continue-on-error: false
+
+      - name: Create Vercel project file
+        uses: mobiledevops/secret-to-file-action@v1
+        with:
+          base64-encoded-secret: ${{ secrets.VERCEL_PROJECT_JSON_BASE64 }}
+          filename: "project.json"
+          is-executable: true
+          working-directory: "./dist/docs/.vercel"
+
+      - name: Install Vercel CLI
+        run: npm install --global vercel@latest
+
+      - name: Pull Vercel Environment Information (preview)
+        if: github.ref != 'refs/heads/master'
+        run: vercel pull --yes --environment=preview --token=${{ secrets.VERCEL_TOKEN }}
+        working-directory: dist/docs
+      - name: Pull Vercel Environment Information (production)
+        if: github.ref == 'refs/heads/master'
+        run: vercel pull --yes --environment=production --token=${{ secrets.VERCEL_TOKEN }}
+        working-directory: dist/docs
+
+      - name: Build Project Artifacts
+        run: vercel build --token=${{ secrets.VERCEL_TOKEN }}
+        working-directory: dist/docs
+
+      - name: Deploy Project Artifacts to Vercel (preview)
+        if: github.ref != 'refs/heads/master'
+        run: vercel deploy --prebuilt --token=${{ secrets.VERCEL_TOKEN }}
+        working-directory: dist/docs
+      - name: Deploy Project Artifacts to Vercel (master)
+        if: github.ref == 'refs/heads/master'
+        run: vercel deploy --prebuilt --prod --token=${{ secrets.VERCEL_TOKEN }}
+        working-directory: dist/docs
+
       - name: Matrix - Node Install
         run: npm install
         working-directory: .github/workflows/support-files

documentation/dev-portal/book.toml

@@ -38,7 +38,7 @@ chapter-line-height       = "2em"
 section-line-height       = "1.5em"
 
 # if true, never read and touch the files in theme dir
-turn-off = false
+turn-off = true
 
 
 [preprocessor.admonish]

documentation/docs/book.toml

@@ -38,7 +38,7 @@ chapter-line-height       = "2em"
 section-line-height       = "1.5em"
 
 # if true, never read and touch the files in theme dir: this is used to stop the looping reload issue referred to in the readme
-turn-off = false
+turn-off = true
 
 [preprocessor.admonish]
 command = "mdbook-admonish"
@@ -90,7 +90,7 @@ git-repository-icon = "fa-github"
 # edit-url-template = "https://github.com/rust-lang/mdBook/edit/master/guide/{path}"
 # site-url = "/docs/"
 # cname = "nymtech.net"
-input-404 = "not-found.md" 
+input-404 = "not-found.md"
 
 [output.html.fold]
 enable = true    # whether or not to enable section folding

documentation/operators/book.toml

@@ -38,7 +38,7 @@ chapter-line-height       = "2em"
 section-line-height       = "1.5em"
 
 # if true, never read and touch the files in theme dir: this is used to stop the looping reload issue referred to in the readme
-turn-off = false
+turn-off = true
 
 [preprocessor.admonish]
 command = "mdbook-admonish"

documentation/post_process.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+# this is a script called by the github CI and CD workflows to post process CSS/image/href links for serving
+# several mdbooks from a subdirectory
+
+cd scripts/post-process
+npm install
+node index.mjs

documentation/scripts/post-process/index.mjs

@@ -0,0 +1,88 @@
+import { unified } from "unified";
+import parse from "rehype-parse";
+import inspectUrls from "@jsdevtools/rehype-url-inspector";
+import stringify from "rehype-stringify";
+import { read, write } from "to-vfile";
+import path from "path";
+import fs from "fs";
+import { glob } from "glob";
+
+async function main() {
+    const distDir = "../../../dist/docs";
+
+    const items = [];
+
+    const books = [
+        'developers',
+        'docs',
+        'operators',
+    ];
+
+    for(const book of books)
+    {
+        // only process the root `index.html` files, because they have absolute paths instead of relative paths
+        const filenames = [ path.resolve(distDir, book, 'index.html') ];
+
+        // leaving this here for a future where other files need to be processed
+        // const filenames = await glob(path.resolve(distDir, book) + '/**/*.html');
+
+        for (const f of filenames) {
+            // Create a Rehype processor with the inspectUrls plugin
+            const processor = unified()
+                .use(parse)
+                .use(inspectUrls, {
+                    inspectEach(args) {
+                        const { url: rawUrl, propertyName } = args;
+                        const { tagName } = args.node;
+                        const filename = args.file.history[0];
+
+                        const relativeFilename = path.relative(distDir, filename);
+                        const relativeDirectory = path.dirname(relativeFilename);
+
+                        // remove relative paths from URL
+                        const bareUrl = rawUrl.split('/').filter(c => c !== '.' && c !== '..').join('/');
+                        let url;
+
+                        if(rawUrl.includes('.html#')) {
+                            url = path.join(`/${relativeDirectory}`, bareUrl);
+                        } else if(rawUrl.startsWith('#')) {
+                            url = path.join(`/`, relativeFilename + bareUrl);
+                        } else {
+                            url = path.join(`/${book}`, bareUrl);
+                        }
+
+                        // const item = { filename, relativeDirectory, tagName, propertyName, rawUrl, url };
+                        const item = { tagName, rawUrl, url };
+
+                        // if(tagName === 'a') {
+                        //     console.log(args);
+                        // }
+
+                        if(!rawUrl.startsWith('http')) {
+                            if (tagName === 'link' || tagName === 'script' || tagName === 'a') {
+                                args.node.properties[propertyName] = url;
+                                items.push(item);
+                            }
+                        }
+                    }
+                })
+                .use(stringify);
+
+            // Read the example HTML file
+            const filename = path.resolve(distDir, f);
+            console.log(`${filename}...`);
+
+            let file = await read(filename);
+
+            // Crawl the HTML file and find all the URLs
+            const res = await processor.process(file);
+
+            fs.writeFileSync(filename, res.value);
+        }
+    }
+
+   // console.table(items);
+   // console.log();
+}
+
+main();

documentation/scripts/post-process/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "nym-docs-post-process",
+  "version": "1.0.0",
+  "main": "index.js",
+  "scripts": {
+    "process": "echo \"Error: no test specified\" && exit 1"
+  },
+  "dependencies": {
+    "@jsdevtools/rehype-url-inspector": "^2.0.2",
+    "glob": "^10.3.4",
+    "rehype-parse": "^9.0.0",
+    "rehype-stringify": "^10.0.0",
+    "to-vfile": "^8.0.0",
+    "unified": "^11.0.2"
+  }
+}