Skip to content

Commit

Permalink
Add imported variables (#751)
Browse files Browse the repository at this point in the history 
We want to be able to import page variables that resides
in inner files. For example, given the following code:

    <include src="foo.md#title" />
    {{ title }}
    
which will import the variable inside foo.md:

    <variable name="title">Title</variable>

Let's implement this feature.
  • Loading branch information
@jamos-tay @yamgent
jamos-tay authored and yamgent committed Apr 29, 2019
1 parent 2a5e645 commit 4412153
Show file tree
Hide file tree
Showing 6 changed files with 142 additions and 29 deletions.
133 changes: 104 additions & 29 deletions src/lib/markbind/src/parser.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ const ATTRIB_CWF = 'cwf';

const BOILERPLATE_FOLDER_NAME = '_markbind/boilerplates';

const VARIABLE_LOOKUP = {};

/*
* Utils
*/
Expand Down Expand Up @@ -142,14 +144,47 @@ function extractPageVariables(fileName, data, userDefinedVariables, includedVari
return;
}
if (!pageVariables[variableName]) {
pageVariables[variableName]
const variableValue
= nunjucks.renderString(md.renderInline(variableElement.html()),
{ ...pageVariables, ...userDefinedVariables, ...includedVariables });
pageVariables[variableName] = variableValue;
if (!VARIABLE_LOOKUP[fileName]) {
VARIABLE_LOOKUP[fileName] = {};
}
VARIABLE_LOOKUP[fileName][variableName] = variableValue;
}
});
return pageVariables;
}

/**
* Extract imported page variables from a page
* @param context of the page
*/
function extractImportedVariables(context) {
if (!context.importedVariables) {
return {};
}
const importedVariables = {};
Object.entries(context.importedVariables).forEach(([src, variables]) => {
variables.forEach((variableName) => {
const actualFilePath = utils.isUrl()
? src
: path.resolve(path.dirname(context.cwf), decodeURIComponent(url.parse(src).path));
if (!VARIABLE_LOOKUP[actualFilePath] || !VARIABLE_LOOKUP[actualFilePath][variableName]) {
// eslint-disable-next-line no-console
console.warn(`Missing variable ${variableName} in ${src} referenced by ${context.cwf}\n`);
return;
}
const variableValue = VARIABLE_LOOKUP[actualFilePath][variableName];
if (!importedVariables[variableName]) {
importedVariables[variableName] = variableValue;
}
});
});
return importedVariables;
}

Parser.prototype.getDynamicIncludeSrc = function () {
return _.clone(this.dynamicIncludeSrc);
};
Expand All @@ -172,13 +207,13 @@ Parser.prototype._preprocess = function (node, context, config) {
element.attribs = element.attribs || {};
element.attribs[ATTRIB_CWF] = path.resolve(context.cwf);

const requiresSrc = ['include'].includes(element.name);
const requiresSrc = ['include', 'import'].includes(element.name);
if (requiresSrc && _.isEmpty(element.attribs.src)) {
const error = new Error(`Empty src attribute in ${element.name} in: ${element.attribs[ATTRIB_CWF]}`);
this._onError(error);
return createErrorNode(element, error);
}
const shouldProcessSrc = ['include', 'panel'].includes(element.name);
const shouldProcessSrc = ['include', 'panel', 'import'].includes(element.name);
const hasSrc = _.hasIn(element.attribs, 'src');
let isUrl;
let includeSrc;
Expand Down Expand Up @@ -211,7 +246,8 @@ Parser.prototype._preprocess = function (node, context, config) {
}
}

if (element.name === 'include') {
if (element.name === 'include' || element.name === 'import') {
const isImport = element.name === 'import';
const isInline = _.hasIn(element.attribs, 'inline');
const isDynamic = _.hasIn(element.attribs, 'dynamic');
const isOptional = _.hasIn(element.attribs, 'optional');
Expand Down Expand Up @@ -263,22 +299,17 @@ Parser.prototype._preprocess = function (node, context, config) {
element.name = 'markdown';
}

let fileContent = self._fileCache[actualFilePath]; // cache the file contents to save some I/O
const fileContent = self._fileCache[actualFilePath]; // cache the file contents to save some I/O
const { parent, relative } = calculateNewBaseUrls(filePath, config.rootPath, config.baseUrlMap);
const userDefinedVariables = config.userDefinedVariablesMap[path.resolve(parent, relative)];

// Extract included variables from the PARENT file
const includeVariables = extractIncludeVariables(element, context.variables);

// Extract page variables from the CHILD file
const pageVariables = extractPageVariables(element.attribs.src, fileContent,
const pageVariables = extractPageVariables(actualFilePath, fileContent,
userDefinedVariables, includeVariables);

// Render inner file content
fileContent = nunjucks.renderString(fileContent,
{ ...pageVariables, ...includeVariables, ...userDefinedVariables },
{ path: actualFilePath });

// Delete variable attributes in include
Object.keys(element.attribs).forEach((attribute) => {
if (attribute.startsWith('var-')) {
Expand All @@ -296,8 +327,28 @@ Parser.prototype._preprocess = function (node, context, config) {
const segmentSrc = cheerio.parseHTML(fileContent, true);
const $ = cheerio.load(segmentSrc);
const hashContent = $(includeSrc.hash).html();
let actualContent = (hashContent && isTrim) ? hashContent.trim() : hashContent;

if (isImport) {
const variableContent = $(`variable[name=${includeSrc.hash.substring(1)}]`).html();
if (!variableContent) {
// eslint-disable-next-line no-console
console.warn(`Missing import variable ${includeSrc.pathname} in ${element.attribs[ATTRIB_CWF]}.`);
return createEmptyNode();
}
if (!context.importedVariables) {
// eslint-disable-next-line no-param-reassign
context.importedVariables = {};
}
if (!context.importedVariables[includeSrc.pathname]) {
// eslint-disable-next-line no-param-reassign
context.importedVariables[includeSrc.pathname] = [];
}
// eslint-disable-next-line no-param-reassign
context.importedVariables[includeSrc.pathname].push(includeSrc.hash.substring(1));
return createEmptyNode();
}

let actualContent = (hashContent && isTrim) ? hashContent.trim() : hashContent;
if (actualContent === null) {
if (isOptional) {
// set empty content for optional segment include that does not exist
Expand Down Expand Up @@ -331,6 +382,12 @@ Parser.prototype._preprocess = function (node, context, config) {
true,
);
} else {
if (isImport) {
// eslint-disable-next-line no-console
console.warn(`Missing hash for import variable ${includeSrc.pathname}`
+ ` in ${element.attribs[ATTRIB_CWF]}.`);
return createEmptyNode();
}
let actualContent = (fileContent && isTrim) ? fileContent.trim() : fileContent;
if (isIncludeSrcMd) {
if (context.mode === 'include') {
Expand All @@ -352,15 +409,30 @@ Parser.prototype._preprocess = function (node, context, config) {
childContext.cwf = filePath;
childContext.source = isIncludeSrcMd ? 'md' : 'html';
childContext.callStack.push(context.cwf);
childContext.variables = includeVariables;
childContext.variables = { ...includeVariables };
childContext.importedVariables = {};

if (element.children && element.children.length > 0) {
if (childContext.callStack.length > CyclicReferenceError.MAX_RECURSIVE_DEPTH) {
const error = new CyclicReferenceError(childContext.callStack);
this._onError(error);
return createErrorNode(element, error);
}
element.children = element.children.map(e => self._preprocess(e, childContext, config));
element.children = element.children.map((e) => {
let processedEle = cheerio.html(self._preprocess(e, childContext, config));
const importedVariables = extractImportedVariables(childContext);
userDefinedVariables.hostBaseUrl = '{{hostBaseUrl}}';
processedEle
= nunjucks.renderString(processedEle,
{
...pageVariables,
...importedVariables,
...includeVariables,
...userDefinedVariables,
},
{ path: actualFilePath });
return cheerio.parseHTML(processedEle)[0];
});
}
} else if ((element.name === 'panel') && hasSrc) {
if (!isUrl && includeSrc.hash) {
Expand Down Expand Up @@ -531,8 +603,17 @@ Parser.prototype.includeFile = function (file, config) {
context.cwf = config.cwf || file; // current working file
context.mode = 'include';
context.callStack = [];
let fileData = null;

return new Promise((resolve, reject) => {
let actualFilePath = file;
if (!utils.fileExists(file)) {
const boilerplateFilePath = calculateBoilerplateFilePath(path.basename(file), file, config);
if (utils.fileExists(boilerplateFilePath)) {
actualFilePath = boilerplateFilePath;
}
}

const handler = new htmlparser.DomHandler((error, dom) => {
if (error) {
reject(error);
Expand All @@ -549,35 +630,29 @@ Parser.prototype.includeFile = function (file, config) {
}
return processed;
});
resolve(cheerio.html(nodes));
const { parent, relative } = calculateNewBaseUrls(file, config.rootPath, config.baseUrlMap);
const userDefinedVariables = config.userDefinedVariablesMap[path.resolve(parent, relative)];
const pageVariables = extractPageVariables(path.basename(file), fileData, userDefinedVariables, {});
const importedVariables = extractImportedVariables(context);
resolve(nunjucks.renderString(cheerio.html(nodes),
{ ...pageVariables, ...importedVariables, ...userDefinedVariables },
{ path: actualFilePath }));
});

const parser = new htmlparser.Parser(handler, {
xmlMode: true,
decodeEntities: true,
});

let actualFilePath = file;
if (!utils.fileExists(file)) {
const boilerplateFilePath = calculateBoilerplateFilePath(path.basename(file), file, config);
if (utils.fileExists(boilerplateFilePath)) {
actualFilePath = boilerplateFilePath;
}
}

// Read files
fs.readFile(actualFilePath, 'utf-8', (err, data) => {
fileData = data;
if (err) {
reject(err);
return;
}
const { parent, relative } = calculateNewBaseUrls(file, config.rootPath, config.baseUrlMap);
const userDefinedVariables = config.userDefinedVariablesMap[path.resolve(parent, relative)];
const pageVariables = extractPageVariables(path.basename(file), data, userDefinedVariables, {});
const fileContent = nunjucks.renderString(data,
{ ...pageVariables, ...userDefinedVariables },
{ path: actualFilePath });
const fileExt = utils.getExt(file);
const fileContent = data;
if (utils.isMarkdownFileExt(fileExt)) {
context.source = 'md';
parser.parseComplete(fileContent.toString());
Expand Down
14 changes: 14 additions & 0 deletions test/functional/test_site/expected/index.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,17 @@ <h1 id="page-variable-referencing-included-variable">Page Variable Referencing I
<div></div>
Page Variable Referencing Included Variable
</div>
<div>
<h1 id="imported-variable">Imported Variable<a class="fa fa-anchor" href="#imported-variable"></a></h1>
<div></div>
Imported Variable
<h1 id="imported-variable-referencing-global-variable">Imported Variable Referencing Global Variable<a class="fa fa-anchor" href="#imported-variable-referencing-global-variable"></a></h1>
<div></div>
Imported Variable Referencing Global Variable
</div>
<h1 id="imported-variable-on-top-level">Imported Variable on Top Level<a class="fa fa-anchor" href="#imported-variable-on-top-level"></a></h1>
<div></div>
Imported Variable on Top Level
<h1 id="heading-with-multiple-keywords">Heading with multiple keywords<a class="fa fa-anchor" href="#heading-with-multiple-keywords"></a></h1>
<p><span class="keyword">keyword 1</span>
<span class="keyword">keyword 2</span></p>
Expand Down Expand Up @@ -475,6 +486,9 @@ <h6 class="always-index" id="level-6-header-outside-headingsearchindex-with-alwa
<a class="nav-link py-1" href="#outer-page-variable-should-not-leak-into-inner-pages">Outer Page Variable Should Not Leak Into Inner Pages&#x200E;</a>
<a class="nav-link py-1" href="#included-variable-overriding-page-variable">Included Variable Overriding Page Variable&#x200E;</a>
<a class="nav-link py-1" href="#page-variable-referencing-included-variable">Page Variable Referencing Included Variable&#x200E;</a>
<a class="nav-link py-1" href="#imported-variable">Imported Variable&#x200E;</a>
<a class="nav-link py-1" href="#imported-variable-referencing-global-variable">Imported Variable Referencing Global Variable&#x200E;</a>
<a class="nav-link py-1" href="#imported-variable-on-top-level">Imported Variable on Top Level&#x200E;</a>
<a class="nav-link py-1" href="#heading-with-multiple-keywords">Heading with multiple keywords&#x200E;</a>
<a class="nav-link py-1" href="#heading-with-keyword-in-panel">Heading with keyword in panel&#x200E;</a>
<a class="nav-link py-1" href="#panel-with-heading-with-keyword">Panel with heading with keyword&#x200E;</a>
Expand Down
3 changes: 3 additions & 0 deletions test/functional/test_site/expected/siteData.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,9 @@
"outer-page-variable-should-not-leak-into-inner-pages": "Outer Page Variable Should Not Leak Into Inner Pages",
"included-variable-overriding-page-variable": "Included Variable Overriding Page Variable",
"page-variable-referencing-included-variable": "Page Variable Referencing Included Variable",
"imported-variable": "Imported Variable",
"imported-variable-referencing-global-variable": "Imported Variable Referencing Global Variable",
"imported-variable-on-top-level": "Imported Variable on Top Level",
"heading-with-multiple-keywords": "Heading with multiple keywords | keyword 1, keyword 2",
"heading-with-keyword-in-panel": "Heading with keyword in panel | panel keyword",
"expanded-panel-without-heading-with-keyword": "Expanded panel without heading with keyword",
Expand Down
6 changes: 6 additions & 0 deletions test/functional/test_site/index.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,12 @@ tags: ["tag-frontmatter-shown", "tag-included-file", "+tag-exp*", "-tag-exp-hidd
<variable name="included_variable_overriding_page_variable">Included Variable Overriding Page Variable</variable>
</include>

<include src="testImportedVariables.md" />

# Imported Variable on Top Level
<import src="testImportedVariablesToImport.md#imported_variable_on_top_level" />
{{ imported_variable_on_top_level }}

# Heading with multiple keywords
<span class="keyword">keyword 1</span>
<span class="keyword">keyword 2</span>
Expand Down
7 changes: 7 additions & 0 deletions test/functional/test_site/testImportedVariables.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
# Imported Variable
<import src="testImportedVariablesToImport.md#imported_variable" />
{{ imported_variable }}

# Imported Variable Referencing Global Variable
<import src="testImportedVariablesToImport.md#imported_variable_referencing_global_variable" />
{{ imported_variable_referencing_global_variable }}
8 changes: 8 additions & 0 deletions test/functional/test_site/testImportedVariablesToImport.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
# **Should not appear**: Imported Variable
<variable name="imported_variable">Imported Variable</variable>

# **Should not appear**: Imported Variable on Top Level
<variable name="imported_variable_on_top_level">Imported Variable on Top Level</variable>

# **Should not appear**: Imported Variable Referencing Global Variable
<variable name="imported_variable_referencing_global_variable">Imported Variable Referencing {{ global_variable }}</variable>

0 comments on commit 4412153

Please sign in to comment.