diff --git a/README.md b/README.md index 90b7a26d..c7d6a71f 100644 --- a/README.md +++ b/README.md @@ -146,6 +146,14 @@ const params = { let reader = await parquet.ParquetReader.openS3(client,params); ``` +### Reading data from a buffer + +If the complete parquet file is in buffer it can be read directly from memory without incurring any additional I/O. + +``` js +const file = fs.readFileSync('fruits.parquet'); +let reader = await parquet.ParquetReader.openBuffer(file); +``` Encodings --------- diff --git a/lib/reader.js b/lib/reader.js index b7bd315c..77fdf0af 100644 --- a/lib/reader.js +++ b/lib/reader.js @@ -94,6 +94,11 @@ class ParquetReader { return this.openEnvelopeReader(envelopeReader); } + static async openBuffer(buffer) { + let envelopeReader = await ParquetEnvelopeReader.openBuffer(Buffer); + return this.openEnvelopeReader(envelopeReader); + } + /** * Open the parquet file from S3 using the supplied aws client and params * The params have to include `Bucket` and `Key` to the file requested @@ -224,6 +229,12 @@ class ParquetEnvelopeReader { return new ParquetEnvelopeReader(readFn, closeFn, fileStat.size); } + static async openBuffer(buffer) { + let readFn = (offset, length) => buffer.slice(offset,offset+length); + let closeFn = () => ({}); + return new ParquetEnvelopeReader(readFn, closeFn, buffer.length); + } + static async openS3(client, params) { let fileStat = await client.headObject(params).promise();