From 834729afeecfce6349ad71b5b88f3d6e77b74405 Mon Sep 17 00:00:00 2001 From: Micah Riggan Date: Tue, 17 Jul 2018 14:38:36 -0400 Subject: [PATCH 1/7] Cleanup script --- packages/bitcore-node/src/utils/cleanup.ts | 23 ++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 packages/bitcore-node/src/utils/cleanup.ts diff --git a/packages/bitcore-node/src/utils/cleanup.ts b/packages/bitcore-node/src/utils/cleanup.ts new file mode 100644 index 00000000000..e533b4f03ae --- /dev/null +++ b/packages/bitcore-node/src/utils/cleanup.ts @@ -0,0 +1,23 @@ +import { BlockModel } from '../models/block'; +import { TransactionModel } from '../models/transaction'; +import { CoinModel } from '../models/coin'; +import { Storage } from '../services/storage'; + +Storage.start({}).then(() => { + BlockModel.collection + .find({}) + .sort({ height: -1 }) + .stream({ + transform: async block => { + const txs = await TransactionModel.collection.find({ blockHash: block.hash }).toArray(); + for(let tx of txs) { + let mints = await CoinModel.collection.find({ mintTxid: tx.txid }).toArray(); + for (let mint of mints) { + if (mint.mintHeight != block.height && block.height > mint.mintHeight) { + console.log(mint); + } + } + } + } + }) +}); From 0f29db2ba94916f8433f09af05b716d87f235573 Mon Sep 17 00:00:00 2001 From: Micah Riggan Date: Tue, 17 Jul 2018 14:46:57 -0400 Subject: [PATCH 2/7] Adding better messages --- packages/bitcore-node/src/utils/cleanup.ts | 36 +++++++++++++--------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/packages/bitcore-node/src/utils/cleanup.ts b/packages/bitcore-node/src/utils/cleanup.ts index e533b4f03ae..625563ed177 100644 --- a/packages/bitcore-node/src/utils/cleanup.ts +++ b/packages/bitcore-node/src/utils/cleanup.ts @@ -3,21 +3,27 @@ import { TransactionModel } from '../models/transaction'; import { CoinModel } from '../models/coin'; import { Storage } from '../services/storage'; -Storage.start({}).then(() => { - BlockModel.collection - .find({}) - .sort({ height: -1 }) - .stream({ - transform: async block => { - const txs = await TransactionModel.collection.find({ blockHash: block.hash }).toArray(); - for(let tx of txs) { - let mints = await CoinModel.collection.find({ mintTxid: tx.txid }).toArray(); - for (let mint of mints) { - if (mint.mintHeight != block.height && block.height > mint.mintHeight) { - console.log(mint); +Storage.start({}) + .then(() => { + BlockModel.collection + .find({}) + .sort({ height: -1 }) + .stream({ + transform: async block => { + console.log(`Finding transactions for block ${block.hash}`); + const txs = await TransactionModel.collection.find({ blockHash: block.hash }).toArray(); + for (let tx of txs) { + console.log(`Finding coins for tx ${tx.txid}`); + let mints = await CoinModel.collection.find({ mintTxid: tx.txid }).toArray(); + for (let mint of mints) { + if (mint.mintHeight != block.height && block.height > mint.mintHeight) { + console.log(mint); + } } } } - } - }) -}); + }); + }) + .catch(e => { + console.error('fatal', e); + }); From f33eb65c24d3624f41cf63b6b8c9f4de36dfbb90 Mon Sep 17 00:00:00 2001 From: Micah Riggan Date: Tue, 17 Jul 2018 15:03:02 -0400 Subject: [PATCH 3/7] Moving to transform stream --- packages/bitcore-node/src/utils/cleanup.ts | 45 +++++++++++++--------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/packages/bitcore-node/src/utils/cleanup.ts b/packages/bitcore-node/src/utils/cleanup.ts index 625563ed177..8c2adc15156 100644 --- a/packages/bitcore-node/src/utils/cleanup.ts +++ b/packages/bitcore-node/src/utils/cleanup.ts @@ -1,28 +1,37 @@ import { BlockModel } from '../models/block'; +import { Transform } from 'stream'; import { TransactionModel } from '../models/transaction'; import { CoinModel } from '../models/coin'; import { Storage } from '../services/storage'; +class CleanupTransform extends Transform { + constructor() { + super({ objectMode: true }); + } + + async _transform(block, _, done) { + console.log(`Finding transactions for block ${block.hash}`); + const txs = await TransactionModel.collection.find({ blockHash: block.hash }).toArray(); + for (let tx of txs) { + console.log(`Finding coins for tx ${tx.txid}`); + let mints = await CoinModel.collection.find({ mintTxid: tx.txid }).toArray(); + for (let mint of mints) { + if (mint.mintHeight != block.height && block.height > mint.mintHeight) { + console.log(mint); + } + } + } + done(); + } +} + Storage.start({}) .then(() => { - BlockModel.collection - .find({}) - .sort({ height: -1 }) - .stream({ - transform: async block => { - console.log(`Finding transactions for block ${block.hash}`); - const txs = await TransactionModel.collection.find({ blockHash: block.hash }).toArray(); - for (let tx of txs) { - console.log(`Finding coins for tx ${tx.txid}`); - let mints = await CoinModel.collection.find({ mintTxid: tx.txid }).toArray(); - for (let mint of mints) { - if (mint.mintHeight != block.height && block.height > mint.mintHeight) { - console.log(mint); - } - } - } - } - }); + let cursor = BlockModel.collection.find({}); + cursor.addCursorFlag('noCursorTimeout', true); + cursor.pipe(new CleanupTransform()); + cursor.on('data', console.log); + cursor.on('end', () => console.log('done')); }) .catch(e => { console.error('fatal', e); From 800c16462f5c3900439d39a3c9eaa453387221f7 Mon Sep 17 00:00:00 2001 From: Micah Riggan Date: Tue, 17 Jul 2018 15:04:28 -0400 Subject: [PATCH 4/7] Transform works now --- packages/bitcore-node/src/utils/cleanup.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/bitcore-node/src/utils/cleanup.ts b/packages/bitcore-node/src/utils/cleanup.ts index 8c2adc15156..69dba08111e 100644 --- a/packages/bitcore-node/src/utils/cleanup.ts +++ b/packages/bitcore-node/src/utils/cleanup.ts @@ -10,14 +10,12 @@ class CleanupTransform extends Transform { } async _transform(block, _, done) { - console.log(`Finding transactions for block ${block.hash}`); const txs = await TransactionModel.collection.find({ blockHash: block.hash }).toArray(); for (let tx of txs) { - console.log(`Finding coins for tx ${tx.txid}`); let mints = await CoinModel.collection.find({ mintTxid: tx.txid }).toArray(); for (let mint of mints) { if (mint.mintHeight != block.height && block.height > mint.mintHeight) { - console.log(mint); + this.push(mint); } } } From 5b8a4a5b66c9d978b0746a5a9583734974583cdf Mon Sep 17 00:00:00 2001 From: Micah Riggan Date: Tue, 17 Jul 2018 15:08:29 -0400 Subject: [PATCH 5/7] self? --- packages/bitcore-node/src/utils/cleanup.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/bitcore-node/src/utils/cleanup.ts b/packages/bitcore-node/src/utils/cleanup.ts index 69dba08111e..05e4d3f57c3 100644 --- a/packages/bitcore-node/src/utils/cleanup.ts +++ b/packages/bitcore-node/src/utils/cleanup.ts @@ -10,12 +10,13 @@ class CleanupTransform extends Transform { } async _transform(block, _, done) { + const self = this; const txs = await TransactionModel.collection.find({ blockHash: block.hash }).toArray(); for (let tx of txs) { let mints = await CoinModel.collection.find({ mintTxid: tx.txid }).toArray(); for (let mint of mints) { if (mint.mintHeight != block.height && block.height > mint.mintHeight) { - this.push(mint); + self.push(mint); } } } From 2d7b8df1838917eef0188c19e287d32d09d86631 Mon Sep 17 00:00:00 2001 From: Micah Riggan Date: Tue, 17 Jul 2018 15:33:22 -0400 Subject: [PATCH 6/7] Piping on wrong thing --- packages/bitcore-node/src/utils/cleanup.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/bitcore-node/src/utils/cleanup.ts b/packages/bitcore-node/src/utils/cleanup.ts index 05e4d3f57c3..2a02f66e024 100644 --- a/packages/bitcore-node/src/utils/cleanup.ts +++ b/packages/bitcore-node/src/utils/cleanup.ts @@ -28,9 +28,9 @@ Storage.start({}) .then(() => { let cursor = BlockModel.collection.find({}); cursor.addCursorFlag('noCursorTimeout', true); - cursor.pipe(new CleanupTransform()); - cursor.on('data', console.log); - cursor.on('end', () => console.log('done')); + cursor.pipe(new CleanupTransform()) + .on('data', console.log) + .on('end', () => console.log('done')); }) .catch(e => { console.error('fatal', e); From 05eb97a80a0c3004067bef6729940badd89666f3 Mon Sep 17 00:00:00 2001 From: Micah Riggan Date: Tue, 17 Jul 2018 16:20:58 -0400 Subject: [PATCH 7/7] Adding sorting back --- packages/bitcore-node/src/utils/cleanup.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/bitcore-node/src/utils/cleanup.ts b/packages/bitcore-node/src/utils/cleanup.ts index 2a02f66e024..f7c8087767e 100644 --- a/packages/bitcore-node/src/utils/cleanup.ts +++ b/packages/bitcore-node/src/utils/cleanup.ts @@ -26,7 +26,7 @@ class CleanupTransform extends Transform { Storage.start({}) .then(() => { - let cursor = BlockModel.collection.find({}); + let cursor = BlockModel.collection.find({}).sort({height: -1}); cursor.addCursorFlag('noCursorTimeout', true); cursor.pipe(new CleanupTransform()) .on('data', console.log)