From 56c4afbdde99e07c49ca5dccee39e4c9de02bfd6 Mon Sep 17 00:00:00 2001 From: Max Fortun Date: Wed, 18 Jan 2023 14:59:06 -0500 Subject: [PATCH 01/12] Added metadata projection --- lib/agent.js | 1 + lib/submit-request.js | 35 +++++++- test/client/submit.js | 185 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 220 insertions(+), 1 deletion(-) diff --git a/lib/agent.js b/lib/agent.js index 2ad4d383d..eadd7e27d 100644 --- a/lib/agent.js +++ b/lib/agent.js @@ -261,6 +261,7 @@ Agent.prototype._sendOp = function(collection, id, op) { if ('op' in op) message.op = op.op; if (op.create) message.create = op.create; if (op.del) message.del = true; + if (op.m) message.m = op.m; this.send(message); }; diff --git a/lib/submit-request.js b/lib/submit-request.js index 04517513b..aa5879a21 100644 --- a/lib/submit-request.js +++ b/lib/submit-request.js @@ -171,7 +171,7 @@ SubmitRequest.prototype.commit = function(callback) { var op = request.op; op.c = request.collection; op.d = request.id; - op.m = undefined; + op.m = request._metadataProjection(); // Needed for agent to detect if it can ignore sending the op back to // the client that submitted it in subscriptions if (request.collection !== request.index) op.i = request.index; @@ -185,6 +185,39 @@ SubmitRequest.prototype.commit = function(callback) { }); }; +SubmitRequest.prototype._metadataProjection = function() { + var request = this; + + // Default behavior + if (!request.opMetadataProjection) { + return undefined; + } + + // Granular projection + if (typeof request.opMetadataProjection === 'object') { + return request._granularMetadataProjection(); + } + + // Full projection + return request.op.m; +}; + +SubmitRequest.prototype._granularMetadataProjection = function() { + var request = this; + var metadataProjection = {}; + for (var key in request.opMetadataProjection) { + var doProject = request.opMetadataProjection[key]; + if (! doProject ) { + continue; + } + + metadataProjection[key] = request.op.m[key]; + } + + return metadataProjection; +}; + + SubmitRequest.prototype.retry = function(callback) { this.retries++; if (this.maxRetries != null && this.retries > this.maxRetries) { diff --git a/test/client/submit.js b/test/client/submit.js index 15d9e3f49..653f8ddeb 100644 --- a/test/client/submit.js +++ b/test/client/submit.js @@ -7,6 +7,8 @@ var numberType = require('./number-type'); types.register(deserializedType.type); types.register(deserializedType.type2); types.register(numberType.type); +var util = require('../util'); +var errorHandler = util.errorHandler; module.exports = function() { describe('client submit', function() { @@ -1210,5 +1212,188 @@ module.exports = function() { }); }); }); + + describe('metadata projection', function() { + it('passed metadata to connect', function(done) { + var metadata = {username: 'user'}; + + this.backend.use('connect', function(request, next) { + Object.assign(request.agent.custom, request.req); + next(); + }); + + var connection = this.backend.connect(undefined, metadata); + connection.on('connected', function() { + expect(connection.agent.custom).eql(metadata); + done(); + }); + }); + + it('passed metadata to submit', function(done) { + var metadata = {username: 'user'}; + + this.backend.use('connect', function(request, next) { + Object.assign(request.agent.custom, request.req); + next(); + }); + + this.backend.use('submit', function(request) { + expect(request.agent.custom).eql(metadata); + done(); + }); + + var connection = this.backend.connect(undefined, metadata); + var doc = null; + connection.on('connected', function() { + expect(connection.agent.custom).eql(metadata); + doc = connection.get('dogs', 'fido'); + doc.create({name: 'fido'}, function() { + doc.submitOp([{p: ['tricks'], oi: ['fetch']}], {source: 'trainer'}, errorHandler(done)); + }); + }); + }); + + it('received local op without metadata', function(done) { + var metadata = {username: 'user'}; + + this.backend.use('connect', function(request, next) { + Object.assign(request.agent.custom, request.req); + next(); + }); + + this.backend.use('submit', function(request, next) { + expect(request.agent.custom).eql(metadata); + Object.assign(request.op.m, request.agent.custom); + request.opMetadataProjection = {username: true}; + next(); + }); + + var connection = this.backend.connect(undefined, metadata); + var doc = null; + connection.on('connected', function() { + expect(connection.agent.custom).eql(metadata); + doc = connection.get('dogs', 'fido'); + doc.create({name: 'fido'}, function() { + doc.on('op', function(op, source, src, context) { + if (src) { + return; + } + expect(context.op.m).equal(undefined); + done(); + }); + doc.submitOp([{p: ['tricks'], oi: ['fetch']}], {source: 'trainer'}, errorHandler(function() {})); + }); + }); + }); + + it('concurrent changes', function(done) { + this.backend.use('connect', function(request, next) { + expect(request.req).to.have.property('username'); + Object.assign(request.agent.custom, request.req); + next(); + }); + + this.backend.use('submit', function(request, next) { + expect(request.agent.custom).to.have.property('username'); + Object.assign(request.op.m, request.agent.custom); + request.opMetadataProjection = {username: true}; + next(); + }); + + this.backend.use('apply', function(request, next) { + expect(request.op.m).to.have.property('username'); + next(); + }); + + this.backend.use('commit', function(request, next) { + expect(request.op.m).to.have.property('username'); + next(); + }); + + this.backend.use('afterWrite', function(request, next) { + expect(request.op.m).to.have.property('username'); + next(); + }); + + var subscriberCount = 10; + var subscriberOpCount = 10; + + var metadatas = []; + for (var i = 0; i < subscriberCount; i++) { + metadatas[i] = {username: 'user-'+i}; + } + + var ops = []; + for (var i = 0; i < subscriberCount; i++) { + ops[i] = []; + for (var j = 0; j < subscriberOpCount; j++) { + ops[i].push({p: ['tricks '+i+' '+j], oi: 1}); + } + } + + var docs = []; + + function submitOps() { + for (var j = 0; j < subscriberOpCount; j++) { + for (var i = 0; i < subscriberCount; i++) { + var doc = docs[i]; + doc.submitOp([ops[i][j]], {source: 'src-'+i}, errorHandler(doneAfter)); + } + } + } + + function validateAndDone() { + var firstDoc = docs[0]; + // validate that all documents across connections are in sync + for (var i = 1; i < subscriberCount; i++) { + var doc = docs[i]; + expect(doc.data).eql(firstDoc.data); + } + done(); + }; + + var submitOpsAfter = util.callAfter(subscriberCount - 1, submitOps); + var doneAfter = util.callAfter((subscriberCount * subscriberCount * subscriberOpCount) - 1, validateAndDone); + + function getDoc(callback) { + var thisDoc = this; + thisDoc.fetch(function() { + if (!thisDoc.data) { + return thisDoc.create({}, function() { + thisDoc.subscribe(callback); + }); + } + thisDoc.subscribe(callback); + }); + } + + for (var i = 0; i < subscriberCount; i++) { + var metadata = metadatas[i]; + + var connection = this.backend.connect(undefined, Object.assign({}, metadata)); + connection.__test_metadata = Object.assign({}, metadata); + connection.__test_id = i; + + connection.on('connected', function() { + var thisConnection = this; + + expect(thisConnection.agent.custom).eql(thisConnection.__test_metadata); + + thisConnection.doc = docs[thisConnection.__test_id] = thisConnection.get('dogs', 'fido'); + + thisConnection.doc.on('op', function(op, source, src, context) { + if (!src) { // If I am the source there is no metadata to check + return doneAfter(); + } + var id = op[0].p[0].split(' ')[1]; + expect(context.op.m).eql(metadatas[id]); + doneAfter(); + }); + + getDoc.bind(thisConnection.doc)(submitOpsAfter); + }); + } + }); + }); }); }; From 1581dc68dbf304eb86cf3670198ddebf83573f6d Mon Sep 17 00:00:00 2001 From: Max Fortun Date: Wed, 18 Jan 2023 15:31:54 -0500 Subject: [PATCH 02/12] Added metadata projection --- lib/client/doc.js | 10 +++++----- test/client/submit.js | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/client/doc.js b/lib/client/doc.js index cf4f08cc9..59b232ee3 100644 --- a/lib/client/doc.js +++ b/lib/client/doc.js @@ -602,7 +602,7 @@ Doc.prototype._otApply = function(op, source) { // NB: If we need to add another argument to this event, we should consider // the fact that the 'op' event has op.src as its 3rd argument - this.emit('before op batch', op.op, source); + this.emit('before op batch', op.op, source, op.src, {op: op}); // Iteratively apply multi-component remote operations and rollback ops // (source === false) for the default JSON0 OT type. It could use @@ -637,7 +637,7 @@ Doc.prototype._otApply = function(op, source) { this._setData(this.type.apply(this.data, componentOp.op)); this.emit('op', componentOp.op, source, op.src); } - this.emit('op batch', op.op, source); + this.emit('op batch', op.op, source, op.src, {op: op}); // Pop whatever was submitted since we started applying this op this._popApplyStack(stackLength); return; @@ -645,7 +645,7 @@ Doc.prototype._otApply = function(op, source) { // The 'before op' event enables clients to pull any necessary data out of // the snapshot before it gets changed - this.emit('before op', op.op, source, op.src); + this.emit('before op', op.op, source, op.src, {op: op}); // Apply the operation to the local data, mutating it in place this._setData(this.type.apply(this.data, op.op)); // Emit an 'op' event once the local data includes the changes from the @@ -653,8 +653,8 @@ Doc.prototype._otApply = function(op, source) { // submission and before the server or other clients have received the op. // For ops from other clients, this will be after the op has been // committed to the database and published - this.emit('op', op.op, source, op.src); - this.emit('op batch', op.op, source); + this.emit('op', op.op, source, op.src, {op: op}); + this.emit('op batch', op.op, source, op.src, {op: op}); return; } diff --git a/test/client/submit.js b/test/client/submit.js index 653f8ddeb..68280e678 100644 --- a/test/client/submit.js +++ b/test/client/submit.js @@ -1382,7 +1382,7 @@ module.exports = function() { thisConnection.doc = docs[thisConnection.__test_id] = thisConnection.get('dogs', 'fido'); thisConnection.doc.on('op', function(op, source, src, context) { - if (!src) { // If I am the source there is no metadata to check + if (!src || !context) { // If I am the source there is no metadata to check return doneAfter(); } var id = op[0].p[0].split(' ')[1]; From 71f36b1b622fdc2cf651854055bd5ae4caee2e30 Mon Sep 17 00:00:00 2001 From: Max Fortun Date: Wed, 25 Jan 2023 14:23:50 -0500 Subject: [PATCH 03/12] Copy metadata from op to snapshot on apply middleware --- test/client/submit.js | 1 + 1 file changed, 1 insertion(+) diff --git a/test/client/submit.js b/test/client/submit.js index 68280e678..192b03bb5 100644 --- a/test/client/submit.js +++ b/test/client/submit.js @@ -1301,6 +1301,7 @@ module.exports = function() { }); this.backend.use('apply', function(request, next) { + Object.assign(request.snapshot.m, request.op.m); expect(request.op.m).to.have.property('username'); next(); }); From f845c4f605235a6ccac5dc8943e31bba33c24032 Mon Sep 17 00:00:00 2001 From: Max Fortun Date: Wed, 25 Jan 2023 14:46:20 -0500 Subject: [PATCH 04/12] Include metadata when fetching snapshot --- lib/backend.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/backend.js b/lib/backend.js index 0a4f6a884..2022869d2 100644 --- a/lib/backend.js +++ b/lib/backend.js @@ -789,7 +789,8 @@ Backend.prototype._fetchSnapshot = function(collection, id, version, callback) { // - we want to avoid the 'op' middleware, because we later use the 'readSnapshots' middleware in _sanitizeSnapshots // - we handle the projection in _sanitizeSnapshots var from = milestoneSnapshot ? milestoneSnapshot.v : 0; - db.getOps(collection, id, from, version, null, function(error, ops) { + var options = {metadata: true}; + db.getOps(collection, id, from, version, options, function(error, ops) { if (error) return callback(error); backend._buildSnapshotFromOps(id, milestoneSnapshot, ops, function(error, snapshot) { From 06b290db225bb9c604c429af5851bffcb7291309 Mon Sep 17 00:00:00 2001 From: Max Fortun Date: Fri, 27 Jan 2023 10:06:52 -0500 Subject: [PATCH 05/12] Include metadata when fetching snapshot --- lib/backend.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/backend.js b/lib/backend.js index 2022869d2..7b44ad07c 100644 --- a/lib/backend.js +++ b/lib/backend.js @@ -772,9 +772,11 @@ Backend.prototype._fetchSnapshot = function(collection, id, version, callback) { var db = this.db; var backend = this; + var options = {metadata: true}; + var fields = null; var shouldGetLatestSnapshot = version === null; if (shouldGetLatestSnapshot) { - return backend.db.getSnapshot(collection, id, null, null, function(error, snapshot) { + return backend.db.getSnapshot(collection, id, fields, options, function(error, snapshot) { if (error) return callback(error); callback(null, snapshot); @@ -789,7 +791,6 @@ Backend.prototype._fetchSnapshot = function(collection, id, version, callback) { // - we want to avoid the 'op' middleware, because we later use the 'readSnapshots' middleware in _sanitizeSnapshots // - we handle the projection in _sanitizeSnapshots var from = milestoneSnapshot ? milestoneSnapshot.v : 0; - var options = {metadata: true}; db.getOps(collection, id, from, version, options, function(error, ops) { if (error) return callback(error); @@ -844,9 +845,12 @@ Backend.prototype._fetchSnapshotByTimestamp = function(collection, id, timestamp var from = 0; var to = null; + var options = {metadata: true}; + var fields = null; + var shouldGetLatestSnapshot = timestamp === null; if (shouldGetLatestSnapshot) { - return backend.db.getSnapshot(collection, id, null, null, function(error, snapshot) { + return backend.db.getSnapshot(collection, id, fields, options, function(error, snapshot) { if (error) return callback(error); callback(null, snapshot); @@ -862,7 +866,6 @@ Backend.prototype._fetchSnapshotByTimestamp = function(collection, id, timestamp if (error) return callback(error); if (snapshot) to = snapshot.v; - var options = {metadata: true}; db.getOps(collection, id, from, to, options, function(error, ops) { if (error) return callback(error); filterOpsInPlaceBeforeTimestamp(ops, timestamp); From 5939b980b72eea61df6c84d72fd0ede5ae855f91 Mon Sep 17 00:00:00 2001 From: Max Fortun Date: Fri, 27 Jan 2023 15:51:27 -0500 Subject: [PATCH 06/12] Break circular dependency if snapshot data already contains meta --- lib/submit-request.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/submit-request.js b/lib/submit-request.js index aa5879a21..b3da079c0 100644 --- a/lib/submit-request.js +++ b/lib/submit-request.js @@ -266,6 +266,8 @@ SubmitRequest.prototype._addSnapshotMeta = function() { meta.ctime = this.start; } else if (this.op.del) { this.op.m.data = this.snapshot.data; + // break circular dependency if snapshot data already contains metadata + delete this.op.m.data._m; } meta.mtime = this.start; }; From 6ba9af8ad1beb6ac37394ccadc798106aeca032d Mon Sep 17 00:00:00 2001 From: Max Fortun Date: Tue, 31 Jan 2023 10:05:09 -0500 Subject: [PATCH 07/12] Cleaning up and adding documentation --- docs/middleware/op-submission.md | 10 ++++++++++ lib/client/doc.js | 2 -- lib/submit-request.js | 7 +++---- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/docs/middleware/op-submission.md b/docs/middleware/op-submission.md index 57a032af1..374d6f558 100644 --- a/docs/middleware/op-submission.md +++ b/docs/middleware/op-submission.md @@ -37,6 +37,12 @@ backend.use('submit', (context, next) => { if (!userCanChangeDoc(userId, id)) { return next(new Error('Unauthorized')) } + + // add custom metadata to the op + Object.assign(context.op.m, context.agent.custom); + // Explicitly specify which metadata fields to be included when storing the op + context.opMetadataProjection = { userId: true }; + next() }) ``` @@ -61,6 +67,10 @@ backend.use('apply', (context, next) => { if (userId !== ownerId) { return next(new Error('Unauthorized')) } + + // Add op metadata to snapshot before snapshot is stored + Object.assign(context.snapshot.m, context.op.m); + next() }) ``` diff --git a/lib/client/doc.js b/lib/client/doc.js index 59b232ee3..4a6009409 100644 --- a/lib/client/doc.js +++ b/lib/client/doc.js @@ -600,8 +600,6 @@ Doc.prototype._otApply = function(op, source) { ); } - // NB: If we need to add another argument to this event, we should consider - // the fact that the 'op' event has op.src as its 3rd argument this.emit('before op batch', op.op, source, op.src, {op: op}); // Iteratively apply multi-component remote operations and rollback ops diff --git a/lib/submit-request.js b/lib/submit-request.js index b3da079c0..0f753d519 100644 --- a/lib/submit-request.js +++ b/lib/submit-request.js @@ -202,16 +202,15 @@ SubmitRequest.prototype._metadataProjection = function() { return request.op.m; }; +// Specify top level fields to beincluded in a granular projection SubmitRequest.prototype._granularMetadataProjection = function() { var request = this; var metadataProjection = {}; for (var key in request.opMetadataProjection) { var doProject = request.opMetadataProjection[key]; - if (! doProject ) { - continue; + if (doProject ) { + metadataProjection[key] = request.op.m[key]; } - - metadataProjection[key] = request.op.m[key]; } return metadataProjection; From 26c71aeddb99224fd53f65bba8c38798de19179c Mon Sep 17 00:00:00 2001 From: Max Fortun Date: Tue, 31 Jan 2023 10:11:55 -0500 Subject: [PATCH 08/12] Replacing tabs with spaces --- lib/submit-request.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/submit-request.js b/lib/submit-request.js index 0f753d519..730807b35 100644 --- a/lib/submit-request.js +++ b/lib/submit-request.js @@ -265,8 +265,8 @@ SubmitRequest.prototype._addSnapshotMeta = function() { meta.ctime = this.start; } else if (this.op.del) { this.op.m.data = this.snapshot.data; - // break circular dependency if snapshot data already contains metadata - delete this.op.m.data._m; + // break circular dependency if snapshot data already contains metadata + delete this.op.m.data._m; } meta.mtime = this.start; }; From 3bcbb05b96cc62dd8e01360bd20a4eb397eb5498 Mon Sep 17 00:00:00 2001 From: Max Fortun Date: Tue, 31 Jan 2023 11:13:00 -0500 Subject: [PATCH 09/12] Reverting. Will work on this in a separate PR. --- lib/backend.js | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/lib/backend.js b/lib/backend.js index 7b44ad07c..0a4f6a884 100644 --- a/lib/backend.js +++ b/lib/backend.js @@ -772,11 +772,9 @@ Backend.prototype._fetchSnapshot = function(collection, id, version, callback) { var db = this.db; var backend = this; - var options = {metadata: true}; - var fields = null; var shouldGetLatestSnapshot = version === null; if (shouldGetLatestSnapshot) { - return backend.db.getSnapshot(collection, id, fields, options, function(error, snapshot) { + return backend.db.getSnapshot(collection, id, null, null, function(error, snapshot) { if (error) return callback(error); callback(null, snapshot); @@ -791,7 +789,7 @@ Backend.prototype._fetchSnapshot = function(collection, id, version, callback) { // - we want to avoid the 'op' middleware, because we later use the 'readSnapshots' middleware in _sanitizeSnapshots // - we handle the projection in _sanitizeSnapshots var from = milestoneSnapshot ? milestoneSnapshot.v : 0; - db.getOps(collection, id, from, version, options, function(error, ops) { + db.getOps(collection, id, from, version, null, function(error, ops) { if (error) return callback(error); backend._buildSnapshotFromOps(id, milestoneSnapshot, ops, function(error, snapshot) { @@ -845,12 +843,9 @@ Backend.prototype._fetchSnapshotByTimestamp = function(collection, id, timestamp var from = 0; var to = null; - var options = {metadata: true}; - var fields = null; - var shouldGetLatestSnapshot = timestamp === null; if (shouldGetLatestSnapshot) { - return backend.db.getSnapshot(collection, id, fields, options, function(error, snapshot) { + return backend.db.getSnapshot(collection, id, null, null, function(error, snapshot) { if (error) return callback(error); callback(null, snapshot); @@ -866,6 +861,7 @@ Backend.prototype._fetchSnapshotByTimestamp = function(collection, id, timestamp if (error) return callback(error); if (snapshot) to = snapshot.v; + var options = {metadata: true}; db.getOps(collection, id, from, to, options, function(error, ops) { if (error) return callback(error); filterOpsInPlaceBeforeTimestamp(ops, timestamp); From 2cb16ee4ab068851d33a40f625960f7bf8fa7c4c Mon Sep 17 00:00:00 2001 From: Max Fortun Date: Tue, 31 Jan 2023 13:00:41 -0500 Subject: [PATCH 10/12] Reverting. Will work on this in a separate PR. --- lib/submit-request.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/submit-request.js b/lib/submit-request.js index 730807b35..5f291d56d 100644 --- a/lib/submit-request.js +++ b/lib/submit-request.js @@ -265,8 +265,6 @@ SubmitRequest.prototype._addSnapshotMeta = function() { meta.ctime = this.start; } else if (this.op.del) { this.op.m.data = this.snapshot.data; - // break circular dependency if snapshot data already contains metadata - delete this.op.m.data._m; } meta.mtime = this.start; }; From 7954c44b84f0fd4d02ab55112af4943fd3ebff9a Mon Sep 17 00:00:00 2001 From: Max Fortun Date: Wed, 8 Feb 2023 15:55:35 -0500 Subject: [PATCH 11/12] Limiting changes to ops only. Snapshots to be addressed in own pr --- docs/middleware/op-submission.md | 9 +-------- lib/agent.js | 2 +- lib/submit-request.js | 26 +++----------------------- 3 files changed, 5 insertions(+), 32 deletions(-) diff --git a/docs/middleware/op-submission.md b/docs/middleware/op-submission.md index 374d6f558..bec6fad19 100644 --- a/docs/middleware/op-submission.md +++ b/docs/middleware/op-submission.md @@ -39,10 +39,7 @@ backend.use('submit', (context, next) => { } // add custom metadata to the op - Object.assign(context.op.m, context.agent.custom); - // Explicitly specify which metadata fields to be included when storing the op - context.opMetadataProjection = { userId: true }; - + Object.assign(context.op.m, context.agent.custom.metadata); next() }) ``` @@ -67,10 +64,6 @@ backend.use('apply', (context, next) => { if (userId !== ownerId) { return next(new Error('Unauthorized')) } - - // Add op metadata to snapshot before snapshot is stored - Object.assign(context.snapshot.m, context.op.m); - next() }) ``` diff --git a/lib/agent.js b/lib/agent.js index eadd7e27d..05aca54ef 100644 --- a/lib/agent.js +++ b/lib/agent.js @@ -261,7 +261,7 @@ Agent.prototype._sendOp = function(collection, id, op) { if ('op' in op) message.op = op.op; if (op.create) message.create = op.create; if (op.del) message.del = true; - if (op.m) message.m = op.m; + if (op.m) message.m = Object.assign({}, op.m); this.send(message); }; diff --git a/lib/submit-request.js b/lib/submit-request.js index 5f291d56d..f45adc37f 100644 --- a/lib/submit-request.js +++ b/lib/submit-request.js @@ -189,34 +189,14 @@ SubmitRequest.prototype._metadataProjection = function() { var request = this; // Default behavior - if (!request.opMetadataProjection) { + if (!request.agent.custom.metadata) { return undefined; } - // Granular projection - if (typeof request.opMetadataProjection === 'object') { - return request._granularMetadataProjection(); - } - - // Full projection - return request.op.m; + // Copy metadata + return Object.assign({}, request.op.m); }; -// Specify top level fields to beincluded in a granular projection -SubmitRequest.prototype._granularMetadataProjection = function() { - var request = this; - var metadataProjection = {}; - for (var key in request.opMetadataProjection) { - var doProject = request.opMetadataProjection[key]; - if (doProject ) { - metadataProjection[key] = request.op.m[key]; - } - } - - return metadataProjection; -}; - - SubmitRequest.prototype.retry = function(callback) { this.retries++; if (this.maxRetries != null && this.retries > this.maxRetries) { From fb28eed0d1703523c0a0b3efe89dbebc399470c2 Mon Sep 17 00:00:00 2001 From: Max Fortun Date: Thu, 9 Feb 2023 17:14:53 -0500 Subject: [PATCH 12/12] Rework tests --- docs/middleware/op-submission.md | 3 - lib/agent.js | 2 +- lib/client/doc.js | 6 + lib/submit-request.js | 15 +-- test/client/submit.js | 200 +++++++------------------------ 5 files changed, 54 insertions(+), 172 deletions(-) diff --git a/docs/middleware/op-submission.md b/docs/middleware/op-submission.md index bec6fad19..57a032af1 100644 --- a/docs/middleware/op-submission.md +++ b/docs/middleware/op-submission.md @@ -37,9 +37,6 @@ backend.use('submit', (context, next) => { if (!userCanChangeDoc(userId, id)) { return next(new Error('Unauthorized')) } - - // add custom metadata to the op - Object.assign(context.op.m, context.agent.custom.metadata); next() }) ``` diff --git a/lib/agent.js b/lib/agent.js index 05aca54ef..eadd7e27d 100644 --- a/lib/agent.js +++ b/lib/agent.js @@ -261,7 +261,7 @@ Agent.prototype._sendOp = function(collection, id, op) { if ('op' in op) message.op = op.op; if (op.create) message.create = op.create; if (op.del) message.del = true; - if (op.m) message.m = Object.assign({}, op.m); + if (op.m) message.m = op.m; this.send(message); }; diff --git a/lib/client/doc.js b/lib/client/doc.js index 4a6009409..bf893a24e 100644 --- a/lib/client/doc.js +++ b/lib/client/doc.js @@ -864,6 +864,12 @@ Doc.prototype.submitOp = function(component, options, callback) { callback = options; options = null; } + + // If agent has metadata, append on client level so that both client and backend have access to it + if(this.connection.agent && this.connection.agent.custom && typeof this.connection.agent.custom.metadata === "object") { + component.m = Object.assign({}, component.m, this.connection.agent.custom.metadata); + } + var op = {op: component}; var source = options && options.source; this._submit(op, source, callback); diff --git a/lib/submit-request.js b/lib/submit-request.js index f45adc37f..7aa3450c2 100644 --- a/lib/submit-request.js +++ b/lib/submit-request.js @@ -171,7 +171,8 @@ SubmitRequest.prototype.commit = function(callback) { var op = request.op; op.c = request.collection; op.d = request.id; - op.m = request._metadataProjection(); + op.m = request.agent.custom.metadata ? request.op.m : undefined; + // Needed for agent to detect if it can ignore sending the op back to // the client that submitted it in subscriptions if (request.collection !== request.index) op.i = request.index; @@ -185,18 +186,6 @@ SubmitRequest.prototype.commit = function(callback) { }); }; -SubmitRequest.prototype._metadataProjection = function() { - var request = this; - - // Default behavior - if (!request.agent.custom.metadata) { - return undefined; - } - - // Copy metadata - return Object.assign({}, request.op.m); -}; - SubmitRequest.prototype.retry = function(callback) { this.retries++; if (this.maxRetries != null && this.retries > this.maxRetries) { diff --git a/test/client/submit.js b/test/client/submit.js index 192b03bb5..62eaad760 100644 --- a/test/client/submit.js +++ b/test/client/submit.js @@ -7,8 +7,6 @@ var numberType = require('./number-type'); types.register(deserializedType.type); types.register(deserializedType.type2); types.register(numberType.type); -var util = require('../util'); -var errorHandler = util.errorHandler; module.exports = function() { describe('client submit', function() { @@ -1213,187 +1211,79 @@ module.exports = function() { }); }); - describe('metadata projection', function() { - it('passed metadata to connect', function(done) { - var metadata = {username: 'user'}; + describe('op metadata', function() { - this.backend.use('connect', function(request, next) { - Object.assign(request.agent.custom, request.req); - next(); - }); - - var connection = this.backend.connect(undefined, metadata); - connection.on('connected', function() { - expect(connection.agent.custom).eql(metadata); - done(); - }); - }); + it('metadata disabled', async function() { + let resolveTest = null; + let rejectTest = null; + const testPromise = new Promise((resolve, reject) => { resolveTest = resolve; rejectTest = reject; }); - it('passed metadata to submit', function(done) { - var metadata = {username: 'user'}; - - this.backend.use('connect', function(request, next) { - Object.assign(request.agent.custom, request.req); + this.backend.use('afterWrite', function(request, next) { + expect(request.op.m).to.be.undefined; next(); + doneAfter(); }); - this.backend.use('submit', function(request) { - expect(request.agent.custom).eql(metadata); - done(); - }); + const docs = []; + for(let i = 0; i < 2; i++) { + const doc = this.backend.connect().get('dogs', 'fido').on('error', rejectTest); + docs.push(doc); + } - var connection = this.backend.connect(undefined, metadata); - var doc = null; - connection.on('connected', function() { - expect(connection.agent.custom).eql(metadata); - doc = connection.get('dogs', 'fido'); - doc.create({name: 'fido'}, function() { - doc.submitOp([{p: ['tricks'], oi: ['fetch']}], {source: 'trainer'}, errorHandler(done)); - }); - }); - }); + let left = docs.size; + const doneAfter = () => { if(!left--) { resolveTest(); } }; - it('received local op without metadata', function(done) { - var metadata = {username: 'user'}; + await new Promise((resolve, reject) => docs[0].create({ age: 1 }, err => err?reject(err):resolve())).catch(err => rejectTest(err)); - this.backend.use('connect', function(request, next) { - Object.assign(request.agent.custom, request.req); - next(); - }); + for(let i = 0; i < docs.length; i++) { + const doc = docs[i]; + await new Promise((resolve, reject) => doc.subscribe(err => err?reject(err):resolve())).catch(err => rejectTest(err)); + } - this.backend.use('submit', function(request, next) { - expect(request.agent.custom).eql(metadata); - Object.assign(request.op.m, request.agent.custom); - request.opMetadataProjection = {username: true}; - next(); - }); + const doc = docs[0]; + doc.submitOp({ p: ['age'], na: 1, m: { clientMeta: 'client0' } }); - var connection = this.backend.connect(undefined, metadata); - var doc = null; - connection.on('connected', function() { - expect(connection.agent.custom).eql(metadata); - doc = connection.get('dogs', 'fido'); - doc.create({name: 'fido'}, function() { - doc.on('op', function(op, source, src, context) { - if (src) { - return; - } - expect(context.op.m).equal(undefined); - done(); - }); - doc.submitOp([{p: ['tricks'], oi: ['fetch']}], {source: 'trainer'}, errorHandler(function() {})); - }); - }); + return testPromise; }); - it('concurrent changes', function(done) { + it('metadata enabled', async function() { + let resolveTest = null; + let rejectTest = null; + const testPromise = new Promise((resolve, reject) => { resolveTest = resolve; rejectTest = reject; }); + this.backend.use('connect', function(request, next) { - expect(request.req).to.have.property('username'); + expect(request.req.metadata).to.be.ok; Object.assign(request.agent.custom, request.req); next(); }); - this.backend.use('submit', function(request, next) { - expect(request.agent.custom).to.have.property('username'); - Object.assign(request.op.m, request.agent.custom); - request.opMetadataProjection = {username: true}; - next(); - }); - - this.backend.use('apply', function(request, next) { - Object.assign(request.snapshot.m, request.op.m); - expect(request.op.m).to.have.property('username'); - next(); - }); - - this.backend.use('commit', function(request, next) { - expect(request.op.m).to.have.property('username'); - next(); - }); - this.backend.use('afterWrite', function(request, next) { - expect(request.op.m).to.have.property('username'); + expect(request.op.m).to.be.ok; next(); + doneAfter(); }); - var subscriberCount = 10; - var subscriberOpCount = 10; - - var metadatas = []; - for (var i = 0; i < subscriberCount; i++) { - metadatas[i] = {username: 'user-'+i}; + const docs = []; + for(let i = 0; i < 2; i++) { + const connOptions = { metadata: { agentMeta: 'agent'+i } }; + const doc = this.backend.connect(undefined, connOptions).get('dogs', 'fido').on('error', rejectTest); + docs.push(doc); } - var ops = []; - for (var i = 0; i < subscriberCount; i++) { - ops[i] = []; - for (var j = 0; j < subscriberOpCount; j++) { - ops[i].push({p: ['tricks '+i+' '+j], oi: 1}); - } - } + let left = docs.size; + const doneAfter = () => { if(!left--) { resolveTest(); } }; - var docs = []; + await new Promise((resolve, reject) => docs[0].create({ age: 1 }, err => err?reject(err):resolve())).catch(err => rejectTest(err)); - function submitOps() { - for (var j = 0; j < subscriberOpCount; j++) { - for (var i = 0; i < subscriberCount; i++) { - var doc = docs[i]; - doc.submitOp([ops[i][j]], {source: 'src-'+i}, errorHandler(doneAfter)); - } - } + for(let i = 0; i < docs.length; i++) { + const doc = docs[i]; + await new Promise((resolve, reject) => doc.subscribe(err => err?reject(err):resolve())).catch(err => rejectTest(err)); } - function validateAndDone() { - var firstDoc = docs[0]; - // validate that all documents across connections are in sync - for (var i = 1; i < subscriberCount; i++) { - var doc = docs[i]; - expect(doc.data).eql(firstDoc.data); - } - done(); - }; - - var submitOpsAfter = util.callAfter(subscriberCount - 1, submitOps); - var doneAfter = util.callAfter((subscriberCount * subscriberCount * subscriberOpCount) - 1, validateAndDone); + const doc = docs[0]; + doc.submitOp({ p: ['age'], na: 1, m: { clientMeta: 'client0' } }); - function getDoc(callback) { - var thisDoc = this; - thisDoc.fetch(function() { - if (!thisDoc.data) { - return thisDoc.create({}, function() { - thisDoc.subscribe(callback); - }); - } - thisDoc.subscribe(callback); - }); - } - - for (var i = 0; i < subscriberCount; i++) { - var metadata = metadatas[i]; - - var connection = this.backend.connect(undefined, Object.assign({}, metadata)); - connection.__test_metadata = Object.assign({}, metadata); - connection.__test_id = i; - - connection.on('connected', function() { - var thisConnection = this; - - expect(thisConnection.agent.custom).eql(thisConnection.__test_metadata); - - thisConnection.doc = docs[thisConnection.__test_id] = thisConnection.get('dogs', 'fido'); - - thisConnection.doc.on('op', function(op, source, src, context) { - if (!src || !context) { // If I am the source there is no metadata to check - return doneAfter(); - } - var id = op[0].p[0].split(' ')[1]; - expect(context.op.m).eql(metadatas[id]); - doneAfter(); - }); - - getDoc.bind(thisConnection.doc)(submitOpsAfter); - }); - } + return testPromise; }); }); });