Merge pull request #240 from pro-vision/feature/js-data-providers

feat(assemble-lite, pv-stylemark, vscode-pv-handlebars-language-server): data for templates from js

packages/assemble-lite/Assemble.js

@@ -121,8 +121,15 @@ module.exports = class Assemble {
       this.log("modified files:", modifiedFiles);
       if (this.failedPaths.length)
         this.log("failed paths from last build:", this.failedPaths);
+
+      const jsDataPaths = dataPaths.filter((path) => path.endsWith(".js"));
+      // js data files are `require`d, if modified, remove it from cache to get the new content
+      modifiedFiles.forEach((path) => {
+        if (jsDataPaths.includes(path)) delete require.cache[path];
+      });
       // re-try the failed files from the last try
-      modifiedFiles.push(...this.failedPaths);
+      // also add the js data providers, which might return different data, even when the file itself wasn't changed. e.g. fetching or reading from fs
+      modifiedFiles.push(...this.failedPaths, ...jsDataPaths);
       // reset the list from the last run for the next iteration
       this.failedPaths = [];
 
@@ -534,6 +541,8 @@ module.exports = class Assemble {
             dataPool[filename] = load(await readFile(path, "utf-8"), {
               filename,
             });
+          } else if (ext === ".js") {
+            dataPool[filename] = await this._loadJsData(path, filename);
           }
         } catch (error) {
           console.error(
@@ -549,6 +558,40 @@ module.exports = class Assemble {
     return dataPool;
   }
 
+  // extracts data from the js data provider files
+  // some data files are provided via async function,
+  // during watch build, to have faster builds, if the functions takes too long to finish, the old stale data is used,
+  // and in the background the data is fetched and added to the data cache.
+  // this will be used on the next build cycle
+  async _loadJsData(path, name) {
+    let result = require(path);
+    // js returns a json
+    if (typeof result !== "function") return result;
+
+    // js returned a function
+    result = result();
+
+    // sync, can be used right away
+    if (!(result instanceof Promise)) return result;
+
+    // if there is no cached value for this data file,
+    // then there is no other option to wait till the data is generated
+    if (!this.dataPool.hasOwnProperty(name)) return await result;
+
+    const oldValue = this.dataPool[name];
+
+    return Promise.race([
+      // wait 20 ms for the result to be returned,
+      // fallback to the old stale value if it takes longer,
+      // and then update the cache once it is loaded
+      new Promise((resolve) => setTimeout(() => resolve(oldValue), 20)),
+      result.then((newValue) => {
+        this.dataPool[name] = newValue;
+        return newValue;
+      }),
+    ]);
+  }
+
   /**
    * extract front matter and handlebars template from hbs files
    *

packages/assemble-lite/README.md

@@ -42,3 +42,22 @@ assembleLite({
 | data      | glob \| glob[]  | where is the data                         |
 | helpers   | glob \| glob[]  | where are the custom handlebars-helpers (the collection from [handlebars-helpers](https://www.npmjs.com/package/handlebars-helpers) is already included - out of the box)                   |
 | target        | glob \| glob[]  | defines, where to put the rendered files  |
+
+## Data Files
+
+When generating html files, you can provide some data to be passed to the handlebars template and pages.
+
+These data can be local to the template and would only be applied to it, when set as yaml front-matter in the .hbs file.
+Or be global and accessible by all the templates via the handlebars `@root` object. Global data are all `.json`, `.yaml`, `.yml` and `*__data.js` files in the src and pages directory. The js file would need to have a default function that returns a json. This function can also return a promise, but keep in mind that this will slow down the build time and make build caching more difficult.
+
+```js
+// some-component__data.js
+
+module.exports = async function() {
+  await someFileSystemIO();
+
+  return {
+    // the actual data
+  };
+}
+```

packages/assemble-lite/helper/data-helper.js

@@ -18,6 +18,10 @@ const loadData = async (data) => {
         dataPool[filename] = load(await readFile(path, "utf-8"), {
           filename,
         });
+      } else if (ext === ".js") {
+        const provider = require(path);
+        dataPool[filename] =
+          typeof provider === "function" ? await provider() : provider;
       }
     })
   );

packages/pv-stylemark/tasks/clickdummy/assembleClickdummyComponents.js

@@ -6,8 +6,8 @@ const { destPath, cdTemplatesSrc, componentsSrc, hbsHelperSrc } = getAppConfig()
 
 function* composeDataPaths(...fileExtensions) {
   for (const ext of fileExtensions) {
-    yield resolveApp(join(componentsSrc, `**/*.${ext}`));
-    yield resolveApp(join(cdTemplatesSrc, `*.${ext}`));
+    yield resolveApp(join(componentsSrc, `**/*${ext}`));
+    yield resolveApp(join(cdTemplatesSrc, `*${ext}`));
   }
 }
 
@@ -17,7 +17,7 @@ const assembleClickdummyComponents = () => {
     partials: resolveApp(join(componentsSrc, "**/*.hbs")),
     pages: resolveApp(join(componentsSrc, "**/*.hbs")),
     templates: resolveApp(join(cdTemplatesSrc, "**/*.hbs")),
-    data: [...composeDataPaths("json", "yaml", "yml")],
+    data: [...composeDataPaths(".json", ".yaml", ".yml", "__data.js"), ],
     helpers: resolveApp(join(hbsHelperSrc, "*.js")),
     target: resolveApp(join(destPath, "components")),
   });

packages/pv-stylemark/tasks/clickdummy/assembleClickdummyPages.js

@@ -14,6 +14,7 @@ const assembleClickdummyPages = () => {
       resolveApp(join(componentsSrc, "**/*.json")),
       resolveApp(join(componentsSrc, "**/*.yaml")),
       resolveApp(join(componentsSrc, "**/*.yml")),
+      resolveApp(join(componentsSrc, "**/*__data.js")),
     ],
     helpers: resolveApp(join(hbsHelperSrc, "*.js")),
     target: resolveApp(join(destPath, "pages")),

packages/pv-stylemark/webpack-plugin/getFilesToWatch.js

@@ -18,6 +18,7 @@ const getFilesToWatch = async () => {
       // add .yaml/.yml Component files
       ...(await asyncGlob(join(componentsSrc, "**/*.yaml"))),
       ...(await asyncGlob(join(componentsSrc, "**/*.yml"))),
+      ...(await asyncGlob(join(componentsSrc, "**/*__data.js"))),
       // handlebars helpers
       ...(await asyncGlob(join(hbsHelperSrc, "*.js"))),
       // add .hbs Components files

packages/vscode-pv-handlebars-language-server/server/src/helpers.ts

@@ -147,7 +147,7 @@ export async function getPartials(componentsRootPath: string): Promise<Array<{ p
  * @returns {Promise<Array<{ path: string, name: string }>>}
  */
 export async function getDataFiles(componentsRootPath: string): Promise<Array<{ path: string; name: string }>> {
-  const partialPaths = await globby(`${componentsRootPath}/**/*.{json,yaml,yml}`);
+  const partialPaths = await globby(`${componentsRootPath}/**/*{.{json,yaml,yml},__data.js}`);
   return partialPaths.map(filePath => ({ path: filePath, name: basename(filePath) }));
 }