Merge branch 'testnet4' of github.com:kajoseph/bitcore

packages/bitcore-node/scripts/purgeNetworkData.js

@@ -0,0 +1,118 @@
+#!/usr/bin/env node
+
+const { Storage } = require('../build/src/services/storage');
+const { BitcoinBlockStorage: BlockStorage } = require('../build/src/models/block');
+const { TransactionStorage } = require('../build/src/models/transaction');
+const { CoinStorage } = require('../build/src/models/coin');
+const rl = require('readline').createInterface({ input: process.stdin, output: process.stdout });
+const { wait } = require('../build/src/utils');
+
+function usage(errMsg) {
+  console.log('USAGE: ./purgeNetworkData.js [options');
+  console.log('OPTIONS:');
+  console.log('  --chain <value>      REQUIRED - e.g. BTC, BCH, DOGE, LTC...');
+  console.log('  --network <value>    REQUIRED - e.g. mainnet, testnet3, regtest...');
+  console.log('  --limit <value>      Number of documents to delete at a time. Default: 1000');
+  console.log('  --sleep <value>      Sleep time in milliseconds between deletions. Default: 200');
+  console.log('  --every <value>      Sleep for --sleep milliseconds every --every loop iteration. Default: 10');
+  if (errMsg) {
+    console.log('\nERROR: ' + errMsg);
+  }
+  process.exit();
+}
+
+const args = process.argv.slice(2);
+
+if (args.includes('--help') || args.includes('-h')) {
+  usage();
+}
+
+
+const chainIdx = args.indexOf('--chain');
+const networkIdx = args.indexOf('--network');
+const chain = args[chainIdx + 1]?.toUpperCase();
+const network = args[networkIdx + 1]?.toLowerCase();
+
+if (chainIdx === -1 || networkIdx === -1 || !chain || !network) {
+  usage('Missing required options.');
+}
+
+const limitIdx = args.indexOf('--limit');
+const limit = (limitIdx > -1 && parseInt(args[limitIdx + 1])) || 1000;
+const sleepIdx = args.indexOf('--sleep');
+const sleepMs = (sleepIdx > -1 && parseInt(args[sleepIdx + 1])) || 200;
+const everyIdx = args.indexOf('--every');
+const nSleep = (everyIdx > -1 && parseInt(args[everyIdx + 1])) || 10;
+
+console.log('Connecting to database...');
+
+Storage.start()
+  .then(async () => {
+    const blkCount = await BlockStorage.collection.countDocuments({ chain, network });
+    const txCount = await TransactionStorage.collection.countDocuments({ chain, network });
+    const coinCount = await CoinStorage.collection.countDocuments({ chain, network });
+    let progressCnt = 0;
+
+    console.log(`If you continue, the following ${chain}:${network} data will be purged:`);
+    console.log('Blocks:', blkCount);
+    console.log('Transactions:', txCount);
+    console.log('Coins:', coinCount);
+    console.log(`Data will be deleted in batches of ${limit} documents with a sleep time of ${sleepMs}ms every ${nSleep} iterations.`);
+
+    const ans = await new Promise(r => rl.question('Do you want to continue? (y/N): ', r));
+    if (ans?.toLowerCase() !== 'y') {
+      console.log('Aborted.');
+      return;
+    }
+
+    {
+      progressCnt = 0;
+      let blkIds = await BlockStorage.collection.find({ chain, network }).project({ _id: 1 }).sort({ _id: 1 }).limit(limit).toArray();
+      while (blkIds.length) {
+        process.stdout.write(`Blocks: ${(progressCnt / blkCount).toFixed(2)}% (${progressCnt} / ${blkCount})\r`);
+        const res = await BlockStorage.collection.deleteMany({ _id: { $in: blkIds.map(a => a._id) } });
+        progressCnt += blkIds.length;
+        if (progressCnt % nSleep === 0) {
+          await wait(sleepMs);
+        }
+        blkIds = await BlockStorage.collection.find({ chain, network }).project({ _id: 1 }).sort({ _id: 1 }).limit(limit).toArray();
+      };
+      console.log('\nBlocks purged.');
+    }
+
+    {
+      progressCnt = 0;
+      let txIds = await TransactionStorage.collection.find({ chain, network }).project({ _id: 1 }).sort({ _id: 1 }).limit(limit).toArray();
+      while (txIds.length){
+        process.stdout.write(`Transactions: ${(progressCnt / txCount).toFixed(2)}% (${progressCnt} / ${txCount})\r`);
+        const res = await TransactionStorage.collection.deleteMany({ _id: { $in: txIds.map(a => a._id) } });
+        progressCnt += txIds.length;
+        if (progressCnt % nSleep === 0) {
+          await wait(sleepMs);
+        }
+        txIds = await TransactionStorage.collection.find({ chain, network }).project({ _id: 1 }).sort({ _id: 1 }).limit(limit).toArray();
+      }
+      console.log('\nTransactions purged.');
+    }
+
+    {
+      progressCnt = 0;
+      let coinIds = await CoinStorage.collection.find({ chain, network }).project({ _id: 1 }).sort({ _id: 1 }).limit(limit).toArray();
+      while (coinIds.length) {
+        process.stdout.write(`Coins: ${(progressCnt / coinCount).toFixed(2)}% (${progressCnt} / ${coinCount})\r`);
+        const res = await CoinStorage.collection.deleteMany({ _id: { $in: coinIds.map(a => a._id) } });
+        progressCnt += coinIds.length;
+        if (progressCnt % nSleep === 0) {
+          await wait(sleepMs);
+        }
+        coinIds = await CoinStorage.collection.find({ chain, network }).project({ _id: 1 }).sort({ _id: 1 }).limit(limit).toArray();
+      }
+      console.log('\nCoins purged.');
+    }
+    console.log('Data purged.');
+  })
+  .catch(console.error)
+  .finally(() => {
+    rl.close();
+    Storage.stop();
+  });

packages/bitcore-wallet-service/src/scripts/migrateWalletsNetwork.js

@@ -0,0 +1,212 @@
+#!/usr/bin/env node
+
+const config = require('../../ts_build/config').default;
+const { Storage } = require('../../ts_build');
+const rl = require('readline').createInterface({ input: process.stdin, output: process.stdout });
+const fs = require('fs');
+const os = require('os');
+
+const startDate = new Date('2011-01-01T00:00:00.000Z');
+const endDate = new Date();
+
+function usage(errMsg) {
+  console.log('USAGE: ./migrateWalletsNetwork.js [options]');
+  console.log('OPTIONS:');
+  console.log('  --chain <value>         REQUIRED - e.g. BTC, BCH, DOGE, LTC...');
+  console.log('  --oldNetwork <value>    REQUIRED - e.g. testnet3');
+  console.log('  --newNetwork <value>    REQUIRED - e.g. testnet4');
+  console.log('  --doit                  Save the migration to the db');
+  console.log('  --out <value>           Output file (default: <chain>-<oldNetwork>-<newNetwork>-migrate.json)');
+  if (errMsg) {
+    console.log('\nERROR: ' + errMsg);
+  }
+  process.exit();
+}
+
+const args = process.argv.slice(2);
+
+if (args.includes('--help') || args.includes('-h')) {
+  usage();
+}
+
+const chainIdx = args.indexOf('--chain');
+const oldNetworkIdx = args.indexOf('--oldNetwork');
+const newNetworkIdx = args.indexOf('--newNetwork');
+const chain = args[chainIdx + 1]?.toLowerCase(); // wallet.chain/coin is lowercase
+const oldNetwork = args[oldNetworkIdx + 1]?.toLowerCase();
+const newNetwork = args[newNetworkIdx + 1]?.toLowerCase();
+
+if (
+  chainIdx === -1 ||
+  oldNetworkIdx === -1 ||
+  newNetworkIdx === -1 ||
+  !chain ||
+  !oldNetwork ||
+  !newNetwork
+) {
+  usage('Missing required options.');
+}
+
+const outIdx = args.indexOf('--out');
+let outFile = (outIdx > -1 && args[outIdx + 1]) || `${chain}-${oldNetwork}-${newNetwork}-migrate.json`;
+
+if (outFile.startsWith('~')) {
+  outFile = outFile.replace('~', os.homedir());
+}
+if (outFile.startsWith('$HOME')) {
+  outFile = outFile.replace('$HOME', os.homedir());
+}
+if (!outFile.startsWith('/') && !outFile.startsWith('./') && !outFile.startsWith('../')) {
+  outFile = __dirname + '/' + outFile;
+}
+
+const doit = args.includes('--doit');
+
+if (!doit) {
+  console.log('Dry run (pass --doit to actually do it)');
+} else {
+  console.log('LET\'S DO IT FOR REAL');
+}
+
+const storage = new Storage();
+storage.connect(config.storageOpts, async (err) => {
+  if (err) {
+    console.log(err);
+    return;
+  }
+
+  function done(err) {
+    if (err) { console.log(err) }
+    rl.close();
+    storage.disconnect(() => { console.log('done'); });
+  }
+
+  try {
+    // Get all wallets
+    const walletCnt = await storage.db.collection(Storage.collections.WALLETS).count({});
+    const walletStream = storage.db.collection(Storage.collections.WALLETS).find({});
+
+    let fixAddressCount = 0;
+    let fixWalletsCount = 0;
+    let fixTxsCount = 0;
+    let fixCacheCount = 0;
+    let count = 0;
+
+    console.log(`  ${doit ? 'REAL:' : 'DRY RUN:'} Found ${Intl.NumberFormat().format(walletCnt)} total wallets to scan`);
+    console.log(`  Migrating ${chain}:${oldNetwork}->${newNetwork}`);
+    console.log(`  Output file: ${outFile}`);
+    const ans = await new Promise(r => rl.question('Would you like to continue? (y/N): ', r));
+    if (ans?.toLowerCase() !== 'y') {
+      return done('Good bye.');
+    }
+
+    for await (const wallet of walletStream) {
+      count++;
+      if (count % 100 === 0) {
+        process.stdout.write(`Processed ${(count / walletCnt * 100).toFixed(4)}% wallets (${count}) - fixed ${fixWalletsCount}\r`); // shows how fast things are working
+        await new Promise(resolve => setTimeout(resolve, 250)); // cooldown
+      }
+      // if wallet chain is not covered or if network isn't testnet then skip
+      if (wallet.chain !== chain || (!wallet.chain && wallet.coin !== chain) || wallet.network !== oldNetwork) {
+        continue;
+      }
+
+      fs.appendFileSync(outFile, wallet.id + '\n');
+
+      if (doit) {
+        // Update Wallets collection
+        const resWallet = await storage.db.collection(Storage.collections.WALLETS).updateMany({
+          id: wallet.id,
+          network: oldNetwork
+        }, {
+          $set: { network: newNetwork }
+        });
+
+        if (resWallet?.result?.nModified > 0) {
+          fixWalletsCount++;
+        } else if (!resWallet?.result?.ok) {
+          console.log(JSON.stringify(resWallet));
+        }
+
+        // Update Addresses collection
+        const resAddress = await storage.db.collection(Storage.collections.ADDRESSES).updateMany({
+          walletId: wallet.id,
+          createdOn: { $gte: startDate.getTime() / 1000, $lte: endDate.getTime() / 1000 }, // Included only to use index
+          network: oldNetwork
+        }, {
+          $set: { network: newNetwork }
+        });
+
+        if (resAddress?.result?.nModified > 0) {
+          fixAddressCount++;
+        } else if (!resAddress?.result?.ok) {
+          console.log(JSON.stringify(resAddress));
+        }
+
+        // Delete Txs for old network
+        const resTxs = await storage.db.collection(Storage.collections.TXS).deleteMany({
+          walletId: wallet.id,
+          createdOn: { $gte: startDate.getTime() / 1000, $lte: endDate.getTime() / 1000 }, // Included only to use index
+          network: oldNetwork
+        });
+
+        if (resTxs?.result?.n > 0) {
+          fixTxsCount++;
+        } else if (!resTxs?.result?.ok) {
+          console.log(JSON.stringify(resTxs));
+        }
+
+        const resCache = await storage.db.collection(Storage.collections.CACHE).deleteMany({
+          walletId: wallet.id
+        });
+        if (resCache?.result?.n > 0) {
+          fixCacheCount++;
+        } else if (!resCache?.result?.ok) {
+          console.log(JSON.stringify(resTxs));
+        }
+      } else {
+        // Update Wallets collection
+        const walletCount = await storage.db.collection(Storage.collections.WALLETS).countDocuments({
+          id: wallet.id,
+          network: oldNetwork
+        });
+
+        fixWalletsCount += walletCount;
+
+        // Update Addresses collection
+        const addressCount = await storage.db.collection(Storage.collections.ADDRESSES).countDocuments({
+          walletId: wallet.id,
+          createdOn: { $gte: startDate.getTime() / 1000, $lte: endDate.getTime() / 1000 }, // Included only to use index
+          network: oldNetwork
+        });
+
+        fixAddressCount += addressCount;
+
+        // Delete Txs collection
+        const txsCount = await storage.db.collection(Storage.collections.TXS).countDocuments({
+          walletId: wallet.id,
+          createdOn: { $gte: startDate.getTime() / 1000, $lte: endDate.getTime() / 1000 }, // Included only to use index
+          network: oldNetwork
+        });
+
+        fixTxsCount += txsCount;
+
+        // Delete Cache collection
+        const cacheCount = await storage.db.collection(Storage.collections.CACHE).countDocuments({
+          walletId: wallet.id
+        });
+
+        fixCacheCount += cacheCount;
+      }
+    }
+    process.stdout.write('\n');
+
+    console.log(`Fixed ${fixWalletsCount} wallets`);
+    console.log(`Fixed ${fixAddressCount} Addresses`);
+    console.log(`Fixed ${fixTxsCount} Txs`);
+
+    return done();
+  } catch (err) {
+    return done(err);
+  }
+});