Allow commas in headers

lib/reader.js

@@ -217,18 +217,19 @@ class ParquetEnvelopeReader {
   async readRowGroup(schema, rowGroup, columnList) {
     var buffer = {
       rowCount: +rowGroup.num_rows,
-      columnData: {}
+      columnData: []
     };
 
     for (let colChunk of rowGroup.columns) {
-      const colMetadata = colChunk.meta_data;
-      const colKey = colMetadata.path_in_schema;
+      const { path_in_schema } = colChunk.meta_data
 
-      if (columnList.length > 0 && parquet_util.fieldIndexOf(columnList, colKey) < 0) {
+      if (columnList.length > 0 && parquet_util.fieldIndexOf(columnList, path_in_schema) < 0) {
         continue;
       }
 
-      buffer.columnData[colKey] = await this.readColumnChunk(schema, colChunk);
+      const columnChunk = await this.readColumnChunk(schema, colChunk);
+      columnChunk.path_in_schema = path_in_schema
+      buffer.columnData.push(columnChunk)
     }
 
     return buffer;

lib/schema.js

@@ -10,7 +10,6 @@ const PARQUET_COLUMN_KEY_SEPARATOR = '.';
  * A parquet file schema
  */
 class ParquetSchema {
-
   /**
    * Create a new schema from a JSON schema definition
    */
@@ -24,42 +23,35 @@ class ParquetSchema {
    * Retrieve a field definition
    */
   findField(path) {
-    if (path.constructor !== Array) {
-      path = path.split(",");
-    } else {
-      path = path.slice(0); // clone array
-    }
+    let field;
+
+    let { fields } = this;
+    for (const pathSegment of path) {
+      field = fields[pathSegment];
 
-    let n = this.fields;
-    for (; path.length > 1; path.shift()) {
-      n = n[path[0]].fields;
+      ({ fields } = field);
     }
 
-    return n[path[0]];
+    return field;
   }
 
   /**
    * Retrieve a field definition and all the field's ancestors
    */
   findFieldBranch(path) {
-    if (path.constructor !== Array) {
-      path = path.split(",");
-    }
-
     let branch = [];
-    let n = this.fields;
-    for (; path.length > 0; path.shift()) {
-      branch.push(n[path[0]]);
 
-      if (path.length > 1) {
-        n = n[path[0]].fields;
-      }
+    let { fields } = this;
+    for (const pathSegment of path) {
+      const field = fields[pathSegment];
+      branch.push(field);
+
+      ({ fields } = field);
     }
 
     return branch;
   }
-
-};
+}
 
 function buildFields(schema, rLevelParentMax, dLevelParentMax, path) {
   if (!rLevelParentMax) {
@@ -175,4 +167,3 @@ function listFields(fields) {
 }
 
 module.exports = { ParquetSchema };
-

lib/shred.js

@@ -160,34 +160,28 @@ exports.materializeRecords = function(schema, buffer) {
     records.push({});
   }
 
-  for (let k in buffer.columnData) {
-    const field = schema.findField(k);
-    const fieldBranch = schema.findFieldBranch(k);
-    let values = buffer.columnData[k].values[Symbol.iterator]();
+  for (const columnChunk of buffer.columnData) {
+    const { path_in_schema } = columnChunk
+    const field = schema.findField(path_in_schema);
+    const fieldBranch = schema.findFieldBranch(path_in_schema);
+    let values = columnChunk.values[Symbol.iterator]();
 
     let rLevels = new Array(field.rLevelMax + 1);
     rLevels.fill(0);
 
-    for (let i = 0; i < buffer.columnData[k].count; ++i) {
-      const dLevel = buffer.columnData[k].dlevels[i];
-      const rLevel = buffer.columnData[k].rlevels[i];
+    for (let i = 0; i < columnChunk.count; ++i) {
+      const dLevel = columnChunk.dlevels[i];
+      const rLevel = columnChunk.rlevels[i];
 
       rLevels[rLevel]++;
       rLevels.fill(0, rLevel + 1);
 
       let value = null;
       if (dLevel === field.dLevelMax) {
-        value = parquet_types.fromPrimitive(
-            field.originalType || field.primitiveType,
-            values.next().value);
+        value = parquet_types.fromPrimitive(field.originalType || field.primitiveType, values.next().value);
       }
 
-      materializeRecordField(
-          records[rLevels[0] - 1],
-          fieldBranch,
-          rLevels.slice(1),
-          dLevel,
-          value);
+      materializeRecordField(records[rLevels[0] - 1], fieldBranch, rLevels.slice(1), dLevel, value);
     }
   }