From 96a810317a0b02a8d774b9358d37a930da304344 Mon Sep 17 00:00:00 2001 From: Florent Vilmart Date: Wed, 23 Mar 2016 23:24:58 -0400 Subject: [PATCH] Initial implementation of GCS adapter --- index.js | 139 ++++++++++++++++++++++++++++++++++++++ package.json | 23 +++++++ spec/support/jasmine.json | 6 ++ spec/test.spec.js | 44 ++++++++++++ 4 files changed, 212 insertions(+) create mode 100644 index.js create mode 100644 package.json create mode 100644 spec/support/jasmine.json create mode 100644 spec/test.spec.js diff --git a/index.js b/index.js new file mode 100644 index 0000000..51deea5 --- /dev/null +++ b/index.js @@ -0,0 +1,139 @@ +'use strict'; +// GCSAdapter +// Store Parse Files in Google Cloud Storage: https://cloud.google.com/storage +const storage = require('gcloud').storage; + +function requiredOrFromEnvironment(options, key, env) { + options[key] = options[key] || process.env[env]; + if (!options[key]) { + throw `GCSAdapter requires an ${key}`; + } + return options; +} + +function fromEnvironmentOrDefault(options, key, env, defaultValue) { + options[key] = options[key] || process.env[env] || defaultValue; + return options; +} + +function optionsFromArguments(args) { + let options = {}; + let projectIdOrOptions = args[0]; + if (typeof projectIdOrOptions == 'string') { + options.projectId = projectIdOrOptions; + options.keyFilename = args[1]; + options.bucket = args[2]; + let otherOptions = args[3]; + if (otherOptions) { + options.bucketPrefix = otherOptions.bucketPrefix; + options.directAccess = otherOptions.directAccess; + } + } else { + options = projectIdOrOptions || {}; + } + options = requiredOrFromEnvironment(options, 'projectId', 'GCP_PROJECT_ID'); + options = requiredOrFromEnvironment(options, 'keyFilename', 'GCP_KEYFILE_PATH'); + options = requiredOrFromEnvironment(options, 'bucket', 'GCS_BUCKET'); + options = fromEnvironmentOrDefault(options, 'bucketPrefix', 'GCS_BUCKET_PREFIX', ''); + options = fromEnvironmentOrDefault(options, 'directAccess', 'GCS_DIRECT_ACCESS', false); + return options; +} + +/* +supported options + +*projectId / 'GCP_PROJECT_ID' +*keyFilename / 'GCP_KEYFILE_PATH' +*bucket / 'GCS_BUCKET' +{ bucketPrefix / 'GCS_BUCKET_PREFIX' defaults to '' +directAccess / 'GCS_DIRECT_ACCESS' defaults to false +*/ +function GCSAdapter() { + let options = optionsFromArguments(arguments); + + this._bucket = options.bucket; + this._bucketPrefix = options.bucketPrefix; + this._directAccess = options.directAccess; + + let storageOptions = { + projectId: options.projectId, + keyFilename: options.keyFilename + }; + + this._gcsClient = new storage(storageOptions); +} + +GCSAdapter.prototype.createFile = function(filename, data, contentType) { + let params = { + contentType: contentType || 'application/octet-stream' + }; + + return new Promise((resolve, reject) => { + let file = this._gcsClient.bucket(this._bucket).file(this._bucketPrefix + filename); + // gcloud supports upload(file) not upload(bytes), so we need to stream. + var uploadStream = file.createWriteStream(params); + uploadStream.on('error', (err) => { + return reject(err); + }).on('finish', () => { + // Second call to set public read ACL after object is uploaded. + if (this._directAccess) { + file.makePublic((err, res) => { + if (err !== null) { + return reject(err); + } + resolve(); + }); + } else { + resolve(); + } + }); + uploadStream.write(data); + uploadStream.end(); + }); +} + +GCSAdapter.prototype.deleteFile = function(filename) { + return new Promise((resolve, reject) => { + let file = this._gcsClient.bucket(this._bucket).file(this._bucketPrefix + filename); + file.delete((err, res) => { + if(err !== null) { + return reject(err); + } + resolve(res); + }); + }); +} + +// Search for and return a file if found by filename. +// Returns a promise that succeeds with the buffer result from GCS, or fails with an error. +GCSAdapter.prototype.getFileData = function(filename) { + return new Promise((resolve, reject) => { + let file = this._gcsClient.bucket(this._bucket).file(this._bucketPrefix + filename); + // Check for existence, since gcloud-node seemed to be caching the result + file.exists((err, exists) => { + if (exists) { + file.download((err, data) => { + if (err !== null) { + return reject(err); + } + return resolve(data); + }); + } else { + reject(err); + } + }); + }); +} + +// Generates and returns the location of a file stored in GCS for the given request and filename. +// The location is the direct GCS link if the option is set, +// otherwise we serve the file through parse-server. +GCSAdapter.prototype.getFileLocation = function(config, filename) { + if (this._directAccess) { + return `https://${this._bucket}.storage.googleapis.com/${this._bucketPrefix + filename}`; + } + return (config.mount + '/files/' + config.applicationId + '/' + encodeURIComponent(filename)); +} + +module.exports = GCSAdapter; +module.exports.default = GCSAdapter; diff --git a/package.json b/package.json new file mode 100644 index 0000000..69c3354 --- /dev/null +++ b/package.json @@ -0,0 +1,23 @@ +{ + "name": "parse-server-gcs-adapter", + "version": "1.0.0", + "description": "", + "main": "index.js", + "dependencies": { + "gcloud": "^0.29.0", + "jasmine": "^2.4.1" + }, + "devDependencies": {}, + "scripts": { + "test": "jasmine" + }, + "keywords": [ + "parse-server", + "google", + "cloud", + "storage", + "gcs" + ], + "author": "Parse", + "license": "MIT" +} diff --git a/spec/support/jasmine.json b/spec/support/jasmine.json new file mode 100644 index 0000000..e8f0320 --- /dev/null +++ b/spec/support/jasmine.json @@ -0,0 +1,6 @@ +{ + "spec_dir": "spec", + "spec_files": [ + "test.spec.js" + ] +} diff --git a/spec/test.spec.js b/spec/test.spec.js new file mode 100644 index 0000000..9942240 --- /dev/null +++ b/spec/test.spec.js @@ -0,0 +1,44 @@ +'use strict'; +let filesAdapterTests = require('parse-server-conformance-tests').files; + +let GCSAdapter = require('../index.js'); + +describe('GCSAdapter tests', () => { + + it('should throw when not initialized properly', () => { + expect(() => { + var gcsAdapter = new GCSAdapter(); + }).toThrow('GCSAdapter requires an projectId') + + expect(() => { + var gcsAdapter = new GCSAdapter('projectId'); + }).toThrow('GCSAdapter requires an keyFilename') + + expect(() => { + var gcsAdapter = new GCSAdapter('projectId', 'keyFilename'); + }).toThrow('GCSAdapter requires an bucket') + + expect(() => { + var gcsAdapter = new GCSAdapter({ projectId: 'projectId'}); + }).toThrow('GCSAdapter requires an keyFilename') + expect(() => { + var gcsAdapter = new GCSAdapter({ projectId: 'projectId' , keyFilename: 'keyFilename'}); + }).toThrow('GCSAdapter requires an bucket') + }) + + it('should not throw when initialized properly', () => { + expect(() => { + var gcsAdapter = new GCSAdapter('projectId', 'keyFilename', 'bucket'); + }).not.toThrow() + + expect(() => { + var gcsAdapter = new GCSAdapter({ projectId: 'projectId' , keyFilename: 'keyFilename', bucket: 'bucket'}); + }).not.toThrow('GCSAdapter requires an bucket') + }) + + if (process.env.GCP_PROJECT_ID && process.env.GCP_KEYFILE_PATH && process.env.GCS_BUCKET) { + // Should be initialized from the env + let gcsAdapter = new GCSAdapter(); + filesAdapterTests.testAdapter("GCSAdapter", gcs); + } +})