add utilities to use transifex api v3 for pull, pullAll, push commands (

#585)

* add utilities to use transifex api v3 for pull, pullAll, push commands

* clean up

* upgrade version

Co-authored-by: Dima Solyanik <[email protected]>

CHANGELOG.md

@@ -1,3 +1,9 @@
+# [14.1]
+
+- Adds utilities to use Transifex api v3 for pull, pullAll, push commands. Introduces temporary --v3 argument
+  --version3, --v3 Use Transifex v3 under the hood [boolean] [default: false]
+  to specify that new utilities should be used.
+
 # [14.0]
 
 -   **Removed** Retire newrelic integration

Makefile

@@ -1,5 +1,5 @@
 CI_BUILD_NUMBER ?= $(USER)-snapshot
-VERSION ?= 14.0.$(CI_BUILD_NUMBER)
+VERSION ?= 14.1.$(CI_BUILD_NUMBER)
 
 version:
 	@echo $(VERSION)

package.json

@@ -34,6 +34,7 @@
     "isomorphic-style-loader": "4.0.0",
     "memoize-one": "5.1.1",
     "mkdirp": "0.5.1",
+    "node-fetch": "2.6.1",
     "node-sass": "4.13.1",
     "postcss": "8.3.0",
     "postcss-css-variables": "0.13.0",

src/commands/txCommands/pull.js

@@ -1,5 +1,7 @@
 const chalk = require('chalk');
 const { pullResourceContent } = require('./util');
+const { version3 } = require('./util/constants');
+const utilsV3 = require('./util/utilsV3');
 
 module.exports = {
 	command: 'pull',
@@ -12,10 +14,22 @@ module.exports = {
 				describe: 'slugs (names) for each resource to pull',
 				type: 'array',
 			},
+			version3,
 		}),
 	handler: argv => {
 		console.log(chalk.blue('Pulling resource content from transifex'));
 
+		if (argv.v3) {
+			console.log(chalk.magenta('Using Transifex v3'));
+			argv.resource.forEach(slug =>
+				utilsV3
+					.pullResourceContent(slug)
+					.then(() => console.log(chalk.green(`\n${slug} done`)))
+			);
+
+			return;
+		}
+
 		argv.resource.forEach(slug =>
 			pullResourceContent(slug).then(() =>
 				console.log(chalk.green(`\n${slug} done`))

src/commands/txCommands/pullAll.js

@@ -1,6 +1,9 @@
 const chalk = require('chalk');
 const txlib = require('./util');
 const gitHelpers = require('./util/gitHelpers');
+const { TransifexApi } = require('./util/transifexApi');
+const utilsV3 = require('./util/utilsV3');
+const { version3 } = require('./util/constants');
 
 const getProjectResourcesList = () =>
 	txlib.tfx.resource
@@ -48,6 +51,24 @@ const makeDeferredPull = doCommit => slug => () =>
 		return gitHelpers.commit(commitMessage, `--no-verify`);
 	});
 
+const makeDeferredPullV3 = doCommit => slug => () =>
+	pullResourceV3(slug).then(slug => {
+		if (!doCommit) {
+			return;
+		}
+		const commitMessage = `tx:pull for ${slug.replace(/-/g, '_')}`;
+		return gitHelpers.commit(commitMessage, `--no-verify`);
+	});
+
+const pullResourceV3 = slug => {
+	console.log(chalk.cyan(`Starting tx:pull for '${slug}'`));
+	return utilsV3.pullResourceContent(slug).then(() => {
+		// wait until all content has been downloaded before moving on to next tasks
+		console.log(chalk.green(`\nCompleted tx:pull for '${slug}'`));
+		return slug;
+	});
+};
+
 module.exports = {
 	command: 'pullAll',
 	description:
@@ -61,9 +82,34 @@ module.exports = {
 				type: 'boolean',
 				default: false,
 			},
+			version3,
 		}),
 	handler: argv => {
 		console.log(chalk.magenta('Pulling all resources...'));
+		if (argv.v3) {
+			console.log(chalk.magenta('Using Transifex v3'));
+			TransifexApi.fetchProjectResourcesList()
+				// We want to sort the array of resources so that the ALL_TRANSLATIONS_RESOURCE is
+				// downloaded first, this will allow other resources to be applied on top of
+				// any changes in that resource. This should prevent any changes in
+				// feature branches from being overwritten by this resource
+				.then(slugs =>
+					slugs.sort(a =>
+						a === TransifexApi.ALL_TRANSLATIONS_RESOURCE ? -1 : 1
+					)
+				)
+				.then(slugs =>
+					promiseSerial(slugs.map(makeDeferredPullV3(argv.commit)))
+				)
+				.then(
+					() => console.log(chalk.green('All resources pulled.')),
+					err => {
+						console.error(err);
+						process.exit(1);
+					}
+				);
+			return;
+		}
 
 		getProjectResourcesList()
 			.then(slugs => promiseSerial(slugs.map(makeDeferredPull(argv.commit))))

src/commands/txCommands/push.js

@@ -4,6 +4,9 @@ const tfx = require('./util/transifex');
 
 const pushTxMaster = require('./util/pushTxMaster');
 const { gitBranch, exitOnMaster } = require('./util/gitHelpers');
+const { version3 } = require('./util/constants');
+const utilsV3 = require('./util/utilsV3');
+const { TransifexApi } = require('./util/transifexApi');
 
 // syncs content in po format to Transifex
 const pushSrcDiff = poData => {
@@ -30,7 +33,32 @@ const pushSrcDiff = poData => {
 					)
 				);
 		}
-		return; // no data, no branch, no problem
+	});
+};
+// syncs content in po format to Transifex
+const pushSrcDiffV3 = poData => {
+	const branch = gitBranch();
+	return TransifexApi.isResourceExists(branch).then(async branchResourceExists => {
+		if (Object.keys(poData).length) {
+			console.log(chalk.blue('Pushing new source content to Transifex'));
+			console.log('keys:', Object.keys(poData).join(', '));
+			console.log(branchResourceExists ? 'Updating' : 'Creating', 'resource');
+			if (!branchResourceExists) {
+				await TransifexApi.createResource(branch);
+			}
+
+			return TransifexApi.uploadsResourceStrings(branch, poData);
+		}
+		// If there's no translation `po` data but the branch resource exists in Transifex,
+		// delete it from Transifex
+		if (branchResourceExists) {
+			return TransifexApi.deleteResource(branch).then(() =>
+				console.log(
+					'Local branch trn data is empty',
+					` - deleted ${TransifexApi.PROJECT}/${branch}`
+				)
+			);
+		}
 	});
 };
 
@@ -42,13 +70,29 @@ const masterAndResourceTrns = () =>
 		Object.assign({}, masterTrns, resourceTrns)
 	);
 
+const masterAndResourceTrnsv3 = () =>
+	Promise.all([
+		TransifexApi.getResourceStrings(
+			TransifexApi.MASTER_RESOURCE,
+			TransifexApi.PROJECT_MASTER
+		),
+		TransifexApi.getAllStrings(resource => resource !== gitBranch()),
+	]).then(([masterTrns, resourceTrns]) =>
+		Object.assign({}, masterTrns, resourceTrns)
+	);
+
 // Upload any TRN source content that is defined locally but _not_ present on
 // any other resource in Transifex
 const pushTrnSrc = () => {
 	return txlib
 		.trnSrcDiffVerbose(masterAndResourceTrns(), txlib.getLocalTrnSourcePo())
 		.then(pushSrcDiff);
 };
+const pushTrnSrcV3 = () => {
+	return utilsV3
+		.trnSrcDiffVerbose(masterAndResourceTrnsv3(), utilsV3.getLocalTrnSourcePo())
+		.then(pushSrcDiffV3);
+};
 
 module.exports = {
 	command: 'push',
@@ -63,10 +107,16 @@ module.exports = {
 			all: {
 				alias: 'a',
 				default: false,
-				type: 'boolean'
+				type: 'boolean',
 			},
+			version3,
 		}),
 	handler: argv => {
+		if (argv.v3) {
+			console.log(chalk.magenta('Using Transifex v3'));
+			return pushUsingV3(argv);
+		}
+
 		if (argv.project === tfx.MASTER_RESOURCE) {
 			return pushTxMaster();
 		}
@@ -87,3 +137,26 @@ module.exports = {
 		});
 	},
 };
+
+const pushUsingV3 = argv => {
+	if (argv.project === tfx.MASTER_RESOURCE) {
+		// todo is it still being used ? add logic : remove condition
+		console.error('ERROR:', 'is it still being used ?');
+		process.exit(1);
+	}
+
+	if (argv.all) {
+		console.log(chalk.blue('Pushing content to all_translations'));
+		return utilsV3.updateTfxSrcAllTranslations().catch(error => {
+			console.error('ERROR:', error);
+			process.exit(1);
+		});
+	}
+
+	// all other commands should not be run on master
+	exitOnMaster();
+	pushTrnSrcV3().catch(err => {
+		console.error('ERROR:', err);
+		process.exit(1);
+	});
+};

src/commands/txCommands/util/constants.js

@@ -0,0 +1,10 @@
+const version3 = {
+	alias: 'v3',
+	describe: 'Use Transifex v3 under the hood',
+	type: 'boolean',
+	default: false,
+};
+
+module.exports = {
+	version3,
+};

src/commands/txCommands/util/poFormatters.js

@@ -116,10 +116,52 @@ const poObjToMsgObj = trns =>
 		return acc;
 	}, {});
 
+const resourceStringToPoObj = resourceString => ({
+	[resourceString.attributes.key]: {
+		msgid: resourceString.attributes.key,
+		comments: {
+			translator: resourceString.attributes.developer_comment,
+			reference: resourceString.attributes.occurrences,
+		},
+		msgstr: [resourceString.attributes.strings.other],
+	},
+});
+
+const resourceStringsToPoObj = (data, source = {}) => {
+	return data.reduce((acc, item) => {
+		return {
+			...acc,
+			...resourceStringToPoObj(item),
+		};
+	}, source);
+};
+
+const resourceTranslationsToPoObj = (response, source) =>
+	response.included.reduce((acc, inc, i) => {
+		const translatedStrings = response.data[i].attributes.strings;
+		return {
+			...acc,
+			[inc.attributes.key]: {
+				msgid: inc.attributes.key,
+				comments: {
+					translator: inc.attributes.developer_comment,
+					reference: inc.attributes.occurrences,
+				},
+				msgstr: [
+					translatedStrings
+						? translatedStrings.other
+						: inc.attributes.strings.other,
+				],
+			},
+		};
+	}, source);
+
 module.exports = {
 	poStringToPoObj,
 	poObjToPoString,
 	poObjToMsgObj,
 	poStringToPoResource,
 	msgDescriptorsToPoObj,
+	resourceStringsToPoObj,
+	resourceTranslationsToPoObj,
 };