Merge pull request #147 from hed-standard/lazy-partnered-schemas

Implement support for merging lazy partnered schemas

bids/schema.js

@@ -52,8 +52,6 @@ export function parseSchemasSpec(hedVersion) {
     const [schemaSpec, verIssues] = parseSchemaSpec(schemaVersion)
     if (verIssues.length > 0) {
       issues.push(...verIssues)
-    } else if (schemasSpec.isDuplicate(schemaSpec)) {
-      issues.push(generateIssue('invalidSchemaNickname', { spec: schemaVersion, nickname: schemaSpec.nickname }))
     } else {
       schemasSpec.addSchemaSpec(schemaSpec)
     }

common/issues/data.js

@@ -297,6 +297,16 @@ export default {
     level: 'error',
     message: stringTemplate`Tag "${'tag'}" is declared to use a library schema nicknamed "${'library'}" in the dataset's schema listing, but no such schema was found.`,
   },
+  differentWithStandard: {
+    hedCode: 'SCHEMA_LOAD_FAILED',
+    level: 'error',
+    message: stringTemplate`Could not merge lazy partnered schemas with different "withStandard" values: "${'first'}" and "${'second'}".`,
+  },
+  lazyPartneredSchemasShareTag: {
+    hedCode: 'SCHEMA_LOAD_FAILED',
+    level: 'error',
+    message: stringTemplate`Lazy partnered schemas are incompatible because they share the short tag "${'tag'}". These schemas require different prefixes.`,
+  },
   // BIDS issues
   sidecarKeyMissing: {
     hedCode: 'SIDECAR_KEY_MISSING',

common/schema/types.js

@@ -1,5 +1,7 @@
 /** HED schema classes */
 
+import castArray from 'lodash/castArray'
+
 import { getGenerationForSchemaVersion } from '../../utils/hedData'
 
 /**
@@ -9,6 +11,7 @@ export class Schema {
   /**
    * The schema XML data.
    * @type {Object}
+   * @deprecated Unused. Will be removed in 4.0.0.
    */
   xmlData
   /**
@@ -34,17 +37,18 @@ export class Schema {
 
   /**
    * Constructor.
+   *
    * @param {object} xmlData The schema XML data.
    */
   constructor(xmlData) {
     this.xmlData = xmlData
     const rootElement = xmlData.HED
-    this.version = rootElement.$.version
-    this.library = rootElement.$.library ?? ''
+    this.version = rootElement?.$?.version
+    this.library = rootElement?.$?.library ?? ''
 
     if (this.library) {
       this.generation = 3
-    } else {
+    } else if (this.version) {
       this.generation = getGenerationForSchemaVersion(this.version)
     }
   }
@@ -62,7 +66,7 @@ export class Schema {
 }
 
 /**
- * Hed2Schema class
+ * An imported HED 2 schema.
  */
 export class Hed2Schema extends Schema {
   /**
@@ -73,6 +77,7 @@ export class Hed2Schema extends Schema {
 
   /**
    * Constructor.
+   *
    * @param {object} xmlData The schema XML data.
    * @param {SchemaAttributes} attributes A description of tag attributes.
    */
@@ -95,7 +100,7 @@ export class Hed2Schema extends Schema {
 }
 
 /**
- * Hed3Schema class
+ * An imported HED 3 schema.
  */
 export class Hed3Schema extends Schema {
   /**
@@ -108,16 +113,27 @@ export class Hed3Schema extends Schema {
    * @type {Mapping}
    */
   mapping
+  /**
+   * The standard HED schema version this schema is linked to.
+   * @type {string}
+   */
+  withStandard
 
   /**
    * Constructor.
+   *
    * @param {object} xmlData The schema XML data.
    * @param {SchemaEntries} entries A collection of schema entries.
    * @param {Mapping} mapping A mapping between short and long tags.
    */
   constructor(xmlData, entries, mapping) {
     super(xmlData)
 
+    if (!this.library) {
+      this.withStandard = this.version
+    } else {
+      this.withStandard = xmlData.HED?.$?.withStandard
+    }
     this.entries = entries
     this.mapping = mapping
   }
@@ -134,6 +150,30 @@ export class Hed3Schema extends Schema {
   }
 }
 
+/**
+ * An imported lazy partnered HED 3 schema.
+ */
+export class PartneredSchema extends Hed3Schema {
+  /**
+   * The actual HED 3 schema underlying this partnered schema.
+   * @type {Hed3Schema}
+   */
+  actualSchema
+
+  /**
+   * Constructor.
+   *
+   * @param {Hed3Schema} actualSchema The actual HED 3 schema underlying this partnered schema.
+   */
+  constructor(actualSchema) {
+    super({}, actualSchema.entries, actualSchema.mapping)
+    this.actualSchema = actualSchema
+    this.withStandard = actualSchema.withStandard
+    this.library = undefined
+    this.generation = 3
+  }
+}
+
 /**
  * The collection of active HED schemas.
  */
@@ -327,7 +367,7 @@ export class SchemaSpec {
 export class SchemasSpec {
   /**
    * The specification mapping data.
-   * @type {Map<string, SchemaSpec>}
+   * @type {Map<string, SchemaSpec|SchemaSpec[]>}
    */
   data
 
@@ -338,14 +378,29 @@ export class SchemasSpec {
     this.data = new Map()
   }
 
+  /**
+   * Iterator over the specifications.
+   *
+   * @yields {[string, SchemaSpec[]]}
+   */
+  *[Symbol.iterator]() {
+    for (const [key, value] of this.data.entries()) {
+      yield [key, castArray(value)]
+    }
+  }
+
   /**
    * Add a schema to this specification.
    *
    * @param {SchemaSpec} schemaSpec A schema specification.
    * @returns {SchemasSpec| map} This object.
    */
   addSchemaSpec(schemaSpec) {
-    this.data.set(schemaSpec.nickname, schemaSpec)
+    if (this.data.has(schemaSpec.nickname)) {
+      this.data.get(schemaSpec.nickname).push(schemaSpec)
+    } else {
+      this.data.set(schemaSpec.nickname, [schemaSpec])
+    }
     return this
   }
 

converter/converter.js

@@ -79,16 +79,16 @@ export const convertTagToLong = function (schema, hedTag, hedString, offset) {
     const startingIndex = endingIndex
     endingIndex += tag.length
 
-    const tagEntries = castArray(mapping.mappingData.get(tag))
+    const tagEntries = castArray(mapping.shortToTags.get(tag))
 
     if (foundUnknownExtension) {
-      if (mapping.mappingData.has(tag)) {
+      if (mapping.shortToTags.has(tag)) {
         return generateParentNodeIssue(tagEntries, startingIndex, endingIndex)
       } else {
         continue
       }
     }
-    if (!mapping.mappingData.has(tag)) {
+    if (!mapping.shortToTags.has(tag)) {
       if (foundTagEntry === null) {
         return [hedTag, [generateIssue('invalidTag', hedString, {}, [startingIndex + offset, endingIndex + offset])]]
       }
@@ -153,8 +153,8 @@ export const convertTagToShort = function (schema, hedTag, hedString, offset) {
   let lastFoundIndex = index
 
   for (const tag of splitTag) {
-    if (mapping.mappingData.has(tag)) {
-      foundTagEntry = mapping.mappingData.get(tag)
+    if (mapping.shortToTags.has(tag)) {
+      foundTagEntry = mapping.shortToTags.get(tag)
       lastFoundIndex = index
       index -= tag.length
       break

converter/schema.js

@@ -15,7 +15,11 @@ export const buildMappingObject = function (entries) {
   /**
    * @type {Map<string, TagEntry>}
    */
-  const nodeData = new Map()
+  const shortTagData = new Map()
+  /**
+   * @type {Map<string, TagEntry>}
+   */
+  const longTagData = new Map()
   /**
    * @type {Set<string>}
    */
@@ -32,16 +36,17 @@ export const buildMappingObject = function (entries) {
       continue
     }
     const tagObject = new TagEntry(shortTag, tag.name)
-    if (!nodeData.has(lowercaseShortTag)) {
-      nodeData.set(lowercaseShortTag, tagObject)
+    longTagData.set(tag.name, tagObject)
+    if (!shortTagData.has(lowercaseShortTag)) {
+      shortTagData.set(lowercaseShortTag, tagObject)
     } else {
       throw new IssueError(generateIssue('duplicateTagsInSchema', {}))
     }
   }
   for (const tag of takesValueTags) {
-    nodeData.get(tag).takesValue = true
+    shortTagData.get(tag).takesValue = true
   }
-  return new Mapping(nodeData)
+  return new Mapping(shortTagData, longTagData)
 }
 
 /**

converter/types.js

@@ -40,16 +40,24 @@ export class TagEntry {
  */
 export class Mapping {
   /**
-   * A dictionary mapping forms to TagEntry instances.
+   * A dictionary mapping short forms to TagEntry instances.
    * @type {Map<string, TagEntry>}
    */
-  mappingData
+  shortToTags
+  /**
+   * A dictionary mapping long forms to TagEntry instances.
+   * @type {Map<string, TagEntry>}
+   */
+  longToTags
 
   /**
    * Constructor.
-   * @param {Map<string, TagEntry>} mappingData A dictionary mapping forms to TagEntry instances.
+   *
+   * @param {Map<string, TagEntry>} shortToTags A dictionary mapping short forms to TagEntry instances.
+   * @param {Map<string, TagEntry>} longToTags A dictionary mapping long forms to TagEntry instances.
    */
-  constructor(mappingData) {
-    this.mappingData = mappingData
+  constructor(shortToTags, longToTags) {
+    this.shortToTags = shortToTags
+    this.longToTags = longToTags
   }
 }

tests/bids.spec.data.js

@@ -513,6 +513,19 @@ const sidecars = [
       },
     },
   ],
+  // sub10 - Lazy partnered schemas
+  [
+    {
+      // Valid partnered schemas
+      instruments: {
+        HED: {
+          piano_and_violin: '(Piano-sound, Violin-sound)',
+          flute_and_oboe: '(Flute-sound, Oboe-sound)',
+          choral_piano: '(Piano-sound, Vocalized-sound)',
+        },
+      },
+    },
+  ],
 ]
 
 const hedColumnOnlyHeader = 'onset\tduration\tHED\n'
@@ -678,6 +691,16 @@ const tsvFiles = [
       'onset\tduration\tevent_code\tresponse_time\tresponse_count\n' + '5.0\t0\tface\t1\tn/a\n',
     ],
   ],
+  // sub12 - Lazy partnered schemas
+  [
+    [
+      sidecars[9][0],
+      'onset\tduration\tinstruments\n' +
+        '4.5\t0\tpiano_and_violin\n' +
+        '5.0\t0\tflute_and_oboe\n' +
+        '5.2\t0\tchoral_piano\n',
+    ],
+  ],
 ]
 
 const datasetDescriptions = [
@@ -691,7 +714,14 @@ const datasetDescriptions = [
     { Name: 'OnlyScoreAsBase', BIDSVersion: '1.7.0', HEDVersion: 'score_1.0.0' },
     { Name: 'OnlyScoreAsLib', BIDSVersion: '1.7.0', HEDVersion: 'sc:score_1.0.0' },
     { Name: 'OnlyTestAsBase', BIDSVersion: '1.7.0', HEDVersion: 'testlib_1.0.2' },
+    { Name: 'GoodLazyPartneredSchemas', BIDSVersion: '1.7.0', HEDVersion: ['testlib_2.0.0', 'testlib_3.0.0'] },
+    {
+      Name: 'GoodLazyPartneredSchemasWithStandard',
+      BIDSVersion: '1.7.0',
+      HEDVersion: ['testlib_2.0.0', 'testlib_3.0.0', '8.2.0'],
+    },
   ],
+  // Bad datasetDescription.json files
   [
     { Name: 'NonExistentLibrary', BIDSVersion: '1.7.0', HEDVersion: ['8.1.0', 'ts:badlib_1.0.2'] },
     { Name: 'LeadingColon', BIDSVersion: '1.7.0', HEDVersion: [':testlib_1.0.2', '8.1.0'] },
@@ -704,6 +734,13 @@ const datasetDescriptions = [
     { Name: 'BadRemote1', BIDSVersion: '1.7.0', HEDVersion: ['8.1.0', 'ts:testlib_1.800.2'] },
     { Name: 'BadRemote2', BIDSVersion: '1.7.0', HEDVersion: '8.828.0' },
     { Name: 'NoHedVersion', BIDSVersion: '1.7.0' },
+    { Name: 'BadLazyPartneredSchema1', BIDSVersion: '1.7.0', HEDVersion: ['testlib_2.0.0', 'testlib_2.1.0'] },
+    { Name: 'BadLazyPartneredSchema2', BIDSVersion: '1.7.0', HEDVersion: ['testlib_2.1.0', 'testlib_3.0.0'] },
+    {
+      Name: 'LazyPartneredSchemasWithWrongStandard',
+      BIDSVersion: '1.7.0',
+      HEDVersion: ['testlib_2.0.0', 'testlib_3.0.0', '8.1.0'],
+    },
   ],
 ]
 

tests/bids.spec.js

@@ -726,4 +726,48 @@ describe('BIDS datasets', () => {
       })
     }, 10000)
   })
+
+  describe('HED 3 partnered schema tests', () => {
+    let goodEvent
+    let goodDatasetDescriptions, badDatasetDescriptions
+
+    beforeAll(() => {
+      goodEvent = bidsTsvFiles[11][0]
+      goodDatasetDescriptions = bidsDatasetDescriptions[0]
+      badDatasetDescriptions = bidsDatasetDescriptions[1]
+    })
+
+    it('should validate HED 3 in BIDS event TSV files with JSON sidecar data using tags from merged partnered schemas', () => {
+      const testDatasets = {
+        validPartneredTestlib: new BidsDataset([goodEvent], [], goodDatasetDescriptions[8]),
+        validPartneredTestlibWithStandard: new BidsDataset([goodEvent], [], goodDatasetDescriptions[9]),
+        invalidPartneredTestlib1: new BidsDataset([goodEvent], [], badDatasetDescriptions[11]),
+        invalidPartneredTestlib2: new BidsDataset([goodEvent], [], badDatasetDescriptions[12]),
+        invalidPartneredTestlibWithStandard: new BidsDataset([goodEvent], [], badDatasetDescriptions[13]),
+      }
+      const expectedIssues = {
+        validPartneredTestlib: [],
+        validPartneredTestlibWithStandard: [],
+        invalidPartneredTestlib1: [
+          BidsHedIssue.fromHedIssue(
+            generateIssue('lazyPartneredSchemasShareTag', { tag: 'A-nonextension' }),
+            badDatasetDescriptions[11].file,
+          ),
+        ],
+        invalidPartneredTestlib2: [
+          BidsHedIssue.fromHedIssue(
+            generateIssue('lazyPartneredSchemasShareTag', { tag: 'Piano-sound' }),
+            badDatasetDescriptions[12].file,
+          ),
+        ],
+        invalidPartneredTestlibWithStandard: [
+          BidsHedIssue.fromHedIssue(
+            generateIssue('differentWithStandard', { first: '8.1.0', second: '8.2.0' }),
+            badDatasetDescriptions[13].file,
+          ),
+        ],
+      }
+      return validator(testDatasets, expectedIssues, null)
+    }, 10000)
+  })
 })