Skip to content

Commit

Permalink
Merge pull request #2211 from IDEMSInternational/refactor/answer-list…
Browse files Browse the repository at this point in the history
…-parser

Refactor: template parser answer list support
  • Loading branch information
jfmcquade authored Apr 18, 2024
2 parents e963303 + 8752f73 commit d208bed
Show file tree
Hide file tree
Showing 10 changed files with 309 additions and 99 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,6 @@ import { JsonFileCache } from "../cacheStrategy/jsonFile";
import chalk from "chalk";

class BaseProcessor<T = any, V = any> {
/** Used to invalidate cache */
public cacheVersion = 20231001.0;

public logger: Logger;

public cache: JsonFileCache;
Expand All @@ -26,7 +23,7 @@ class BaseProcessor<T = any, V = any> {
* Create a base processor instance. Sets up logging and cache
* @param context.namespace - Name used as prefix for cache and logging
*/
constructor(public context: { namespace: string; paths: IConverterPaths }) {
constructor(public context: { namespace: string; paths: IConverterPaths; cacheVersion: number }) {
const { namespace } = context;
this.logger = createChildFileLogger({ source: namespace });
this.setupCache();
Expand All @@ -36,9 +33,9 @@ class BaseProcessor<T = any, V = any> {
* included relative path, md5 checksum and modified time
*/
private setupCache() {
const { paths, namespace } = this.context;
const { paths, namespace, cacheVersion } = this.context;
const cacheFolder = path.resolve(paths.SHEETS_CACHE_FOLDER, namespace);
this.cache = new JsonFileCache(cacheFolder, this.cacheVersion);
this.cache = new JsonFileCache(cacheFolder, cacheVersion);
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,17 @@ import { IConverterPaths, IFlowHashmapByType, IParsedWorkbookData } from "../../
import { arrayToHashmap, groupJsonByKey, IContentsEntry } from "../../utils";
import BaseProcessor from "../base";

export class FlowParserProcessor extends BaseProcessor<FlowTypes.FlowTypeWithData> {
public cacheVersion = 20230818.3;
const cacheVersion = 20240412.0;

export class FlowParserProcessor extends BaseProcessor<FlowTypes.FlowTypeWithData> {
public parsers: { [flowType in FlowTypes.FlowType]: Parsers.DefaultParser } = {
data_list: new Parsers.DataListParser(this),
data_pipe: new Parsers.DataPipeParser(this),
generator: new Parsers.GeneratorParser(this),
global: new Parsers.DefaultParser(this),
template: new Parsers.TemplateParser(this),
tour: new Parsers.DefaultParser(this),
asset_pack: new Parsers.DefaultParser(this),
};

/** Keep a track of all processed flows by type and name (used in data_pipes)*/
Expand All @@ -30,7 +31,7 @@ export class FlowParserProcessor extends BaseProcessor<FlowTypes.FlowTypeWithDat
} = {};

constructor(paths: IConverterPaths) {
super({ paths, namespace: "flowParser" });
super({ paths, namespace: "flowParser", cacheVersion });
}

public override processInput(flow: FlowTypes.FlowTypeWithData) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,13 +44,16 @@ export class DefaultParser<
this.queue.shift();
}
// Process queue
let rowNumber = 1;
while (this.queue.length > 0) {
// Start rowNumber from 2 to match sheet (without header row)
rowNumber++;
const row = this.queue[0];
try {
const processed = new RowProcessor(row, this, rowDefaultValues).run();
// some rows may be omitted during processing so ignore
if (processed) {
const postProcessed = this.postProcessRow(processed);
const postProcessed = this.postProcessRow(processed, rowNumber);
if (postProcessed) {
processedRows.push(postProcessed);
}
Expand Down Expand Up @@ -87,7 +90,7 @@ export class DefaultParser<

/** Overridable method called by parser to apply any additional processing
* on each individual row. By default the original row is simply returned */
public postProcessRow(row: any) {
public postProcessRow(row: any, rowNumber: number) {
return row;
}

Expand Down Expand Up @@ -232,10 +235,10 @@ class RowProcessor {
// handle custom fields
if (!shouldSkip) {
if (typeof value === "string") {
if (field.endsWith("_list")) {
if (field.endsWith("_list") || field.includes("_list_")) {
this.row[field] = parseAppDataListString(value);
}
if (field.endsWith("_collection")) {
if (field.endsWith("_collection") || field.includes("_collection_")) {
this.row[field] = parseAppDataCollectionString(this.row[field]);
}
if (field.endsWith("action_list")) {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,180 @@
describe("template Parser", () => {
it("TODO - add tests", () => expect(true).toEqual(true));
import { FlowTypes } from "data-models";
import { TemplateParser } from "./template.parser";

const ROW_BASE: FlowTypes.TemplateRow = {
_nested_name: "",
name: "",
type: "" as any,
};

describe("Template Parser PostProcessor", () => {
let parser: TemplateParser;
beforeEach(() => {
parser = new TemplateParser({} as any);
});

it("Remove empty rows", () => {
const res = parser.postProcessRow({} as any);
expect(res).toEqual(undefined);
});

it("Replaces empty row type with set_variable", () => {
const res = parser.postProcessRow({ ...ROW_BASE, type: "" as any });
expect(res.type).toEqual("set_variable");
});

// TODO - notify authors of change (can now auto gen multiple)
it("Generates default row names when omitted for multiple rows", () => {
const res = parser.run({
flow_name: "test_name_generate_multiple",
flow_type: "template",
rows: [
{ ...ROW_BASE, name: "", type: "accordion" },
{ ...ROW_BASE, name: "", type: "accordion" },
],
});
expect(res.rows).toEqual([
// Note - suffix numbers start at 2 as flow sheet would typically have row 1 as header row
{
_nested_name: "accordion_2",
name: "accordion_2",
type: "accordion",
},
{
_nested_name: "accordion_3",
name: "accordion_3",
type: "accordion",
},
]);
});

it("Converts rows ending _list or containing _list_ in name to templated list", () => {
// CASE 0 - Do not convert rows unless explicit includes _list_ or ends _list
const case0_1 = parser.postProcessRow({
...ROW_BASE,
name: "test_listen",
value: "item_1; item_2; item_3;",
});
expect(case0_1.value).toEqual("item_1; item_2; item_3;");

// CASE 1 - List items defined inline
const case1 = parser.postProcessRow({
...ROW_BASE,
name: "test_list",
value: "item_1; item_2; item_3;",
});
expect(case1.value).toEqual(["item_1", "item_2", "item_3"]);
// CASE 2 - List refers to variable (do not parse)
const case2 = parser.postProcessRow({
...ROW_BASE,
name: "test_list_2",
value: "@local.some_list",
});
expect(case2.value).toEqual("@local.some_list");
// CASE 3 - List items include variables
const case3 = parser.postProcessRow({
...ROW_BASE,
name: "test_list_3",
value: "@local.item_1; @local.item_2",
});
expect(case3.value).toEqual(["@local.item_1", "@local.item_2"]);
// CASE 4 - List items as json (mix dynamic and missing)
const case4 = parser.postProcessRow({
...ROW_BASE,
name: "test_list_4",
value: "key_1a: textValue | key_1b: @local.value; key_2a: | key_2b: 5",
});
expect(case4.value).toEqual([
{ key_1a: "textValue", key_1b: "@local.value" },
{ key_2a: "", key_2b: "5" },
]);
});

it("Converts rows with _collection in name to templated collection", () => {
const res = parser.postProcessRow({
...ROW_BASE,
name: "test_collection",
value: "key_1:value_1; key_2:value_2",
});
expect(res.value).toEqual({ key_1: "value_1", key_2: "value_2" });
});

it("Parses parameter lists", () => {
const res = parser.postProcessRow({
...ROW_BASE,
// NOTE - list already array due to defaultParser initial parse
parameter_list: ["key_1:val_1", "key_2: val_trailing_spaces ", "key_3_no_value"] as any,
});
// TODO - anymore advanced cases?
expect(res.parameter_list).toEqual({
key_1: "val_1",
key_2: "val_trailing_spaces",
key_3_no_value: "true",
});
});

it("Replaces action list self-references", () => {
const res = parser.postProcessRow({
...ROW_BASE,
name: "my_action_list",
action_list: [
{ trigger: "click", action_id: "", args: ["@local.my_action_list", "some_value"] },
],
});
expect(res.action_list[0].args).toEqual(["this.value", "some_value"]);
});

it("Extracts dynamic fields", () => {
const res = parser.postProcessRow({
...ROW_BASE,
value: "@local.dynamic_value",
});
expect(res._dynamicFields).toEqual({
value: [
{
fullExpression: "@local.dynamic_value",
matchedExpression: "@local.dynamic_value",
type: "local",
fieldName: "dynamic_value",
},
],
});
});

it("Extracts dynamic dependencies", () => {
const res = parser.postProcessRow({
...ROW_BASE,
value: "@local.dynamic_value",
});
expect(res._dynamicDependencies).toEqual({ "@local.dynamic_value": ["value"] });
});

it("Creates nested path names for child rows", () => {
const rows: FlowTypes.TemplateRow[] = [
{
_nested_name: "my_items",
name: "my_items",
type: "items",
// Handle case of both named and unnamed nested row names
rows: [
{
...ROW_BASE,
type: "text",
name: "named_text",
},
{ ...ROW_BASE, type: "text" },
],
},
];
const res = parser.run({ flow_type: "template", flow_name: "test_nested", rows });
const itemRows = res.rows[0].rows;
const nestedNames = itemRows.map((n) => n._nested_name);
expect(nestedNames).toEqual(["my_items.named_text", "my_items.text_2"]);
});
});

describe("Template Parser [QC]", () => {
// TODO - confirm what checks to include and add to code
// it("Ensures answer_list parameters refer to list variables", () => {
// });
});
Loading

0 comments on commit d208bed

Please sign in to comment.