From 8d08283d541d8f63f8143eb92b39dbcfa694db76 Mon Sep 17 00:00:00 2001 From: nbasili Date: Thu, 18 Feb 2021 10:02:45 +0200 Subject: [PATCH 1/2] Add cache invalidation command to CLI --- packages/cli/README.md | 56 ++++++++-- packages/cli/src/api/invalidate.js | 47 ++++++++ packages/cli/src/commands/invalidate.js | 103 ++++++++++++++++++ packages/cli/src/commands/push.js | 14 +-- packages/cli/test/commands/invalidate.test.js | 44 ++++++++ 5 files changed, 250 insertions(+), 14 deletions(-) create mode 100644 packages/cli/src/api/invalidate.js create mode 100644 packages/cli/src/commands/invalidate.js create mode 100644 packages/cli/test/commands/invalidate.test.js diff --git a/packages/cli/README.md b/packages/cli/README.md index 156b9d85..c9220af6 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -52,6 +52,7 @@ $ npm run push # Commands * [`txjs-cli help [COMMAND]`](#txjs-cli-help-command) * [`txjs-cli push [PATTERN]`](#txjs-cli-push-pattern) +* [`txjs-cli invalidate`](#txjs-cli-invalidate) ## `txjs-cli help [COMMAND]` @@ -70,7 +71,7 @@ OPTIONS ## `txjs-cli push [PATTERN]` -Detect translatable strings and push content to Transifex +detect and push source content to Transifex ``` USAGE @@ -80,13 +81,13 @@ ARGUMENTS PATTERN [default: **/*.{js,jsx,ts,tsx}] file pattern to scan for strings OPTIONS - -v, --verbose Verbose output + -v, --verbose verbose output --cds-host=cds-host CDS host URL - --dry-run Dry run, do not push to Transifex - --purge Purge content on Transifex - --secret=secret Native project secret - --token=token Native project public token - --tags=tags Globally add tags to strings + --dry-run dry run, do not push to Transifex + --purge purge content on Transifex + --secret=secret native project secret + --tags=tags globally tag strings + --token=token native project public token DESCRIPTION Parse .js, .ts, .jsx and .tsx files and detect phrases marked for @@ -113,6 +114,47 @@ DESCRIPTION TRANSIFEX_TOKEN=mytoken TRANSIFEX_SECRET=mysecret txjs-cli push ``` +## `txjs-cli invalidate` + +invalidate and refresh CDS cache + +``` +USAGE + $ txjs-cli invalidate + +OPTIONS + --cds-host=cds-host CDS host URL + --purge force delete CDS cached content + --secret=secret native project secret + --token=token native project public token + +DESCRIPTION + Content for delivery is cached in CDS and refreshed automatically every hour. + This command triggers a refresh of cached content on the fly. + + By default, invalidation does not remove existing cached content, but + starts the process of updating with latest translations from Transifex. + + Passing the --purge option, cached content will be forced to be deleted, + however use that with caution, as it may introduce downtime of + translation delivery to the apps until fresh content is cached in the CDS. + + To invalidate translations some environment variables must be set: + TRANSIFEX_TOKEN= + TRANSIFEX_SECRET= + (optional) TRANSIFEX_CDS_HOST= + + or passed as --token= --secret= parameters + + Default CDS Host is https://cds.svc.transifex.net + + Examples: + txjs-cli invalidate + txjs-cli invalidate --purge + txjs-cli invalidate --token=mytoken --secret=mysecret + TRANSIFEX_TOKEN=mytoken TRANSIFEX_SECRET=mysecret txjs-cli invalidate +``` + # License Licensed under Apache License 2.0, see [LICENSE](https://github.com/transifex/transifex-javascript/blob/HEAD/LICENSE) file. diff --git a/packages/cli/src/api/invalidate.js b/packages/cli/src/api/invalidate.js new file mode 100644 index 00000000..d961a66c --- /dev/null +++ b/packages/cli/src/api/invalidate.js @@ -0,0 +1,47 @@ +const axios = require('axios'); +const { version } = require('../../package.json'); + +/** + * Invalidate CDS cache + * + * @param {Object} params + * @param {String} params.url + * @param {String} params.token + * @param {String} params.secret + * @param {Boolean} params.purge + * @returns {Object} Data + * @returns {Boolean} Data.success + * @returns {String} Data.status + * @returns {Number} Data.data.count + * @returns {Number} Data.data.status + * @returns {Number} Data.data.token + */ +async function invalidateCDS(params) { + const action = params.purge ? 'purge' : 'invalidate'; + try { + const res = await axios.post(`${params.url}/${action}`, { + }, { + headers: { + Authorization: `Bearer ${params.token}:${params.secret}`, + 'Content-Type': 'application/json;charset=utf-8', + 'X-NATIVE-SDK': `txjs/cli/${version}`, + }, + }); + return { + success: true, + status: res.status, + data: res.data, + }; + } catch (error) { + if (error.response) { + return { + success: false, + status: error.response.status, + data: error.response.data, + }; + } + throw new Error(error.message); + } +} + +module.exports = invalidateCDS; diff --git a/packages/cli/src/commands/invalidate.js b/packages/cli/src/commands/invalidate.js new file mode 100644 index 00000000..52ef764f --- /dev/null +++ b/packages/cli/src/commands/invalidate.js @@ -0,0 +1,103 @@ +/* eslint no-shadow: 0 */ + +require('colors'); +const { Command, flags } = require('@oclif/command'); +const { cli } = require('cli-ux'); +const invalidateCDS = require('../api/invalidate'); + +class InvalidateCommand extends Command { + async run() { + const { flags } = this.parse(InvalidateCommand); + + let cdsHost = process.env.TRANSIFEX_CDS_HOST || 'https://cds.svc.transifex.net'; + let projectToken = process.env.TRANSIFEX_TOKEN; + let projectSecret = process.env.TRANSIFEX_SECRET; + + if (flags.token) projectToken = flags.token; + if (flags.secret) projectSecret = flags.secret; + if (flags['cds-host']) cdsHost = flags['cds-host']; + + if (!projectToken || !projectSecret) { + this.log(`${'✘'.red} Cannot invalidate CDS, credentials are missing.`); + this.log('Tip: Set TRANSIFEX_TOKEN and TRANSIFEX_SECRET environment variables'.yellow); + process.exit(); + } + + if (flags.purge) { + cli.action.start('Invalidating and purging CDS cache', '', { stdout: true }); + } else { + cli.action.start('Invalidating CDS cache', '', { stdout: true }); + } + + try { + const res = await invalidateCDS({ + url: cdsHost, + token: projectToken, + secret: projectSecret, + purge: flags.purge, + }); + if (res.success) { + cli.action.stop('Success'.green); + this.log(`${(res.data.count || 0).toString().green} records invalidated`); + this.log('Note: It might take a few minutes for fresh content to be available'.yellow); + } else { + cli.action.stop('Failed'.red); + this.log(`Status code: ${res.status}`.red); + this.error(JSON.stringify(res.data)); + } + } catch (err) { + cli.action.stop('Failed'.red); + throw err; + } + } +} + +InvalidateCommand.description = `invalidate and refresh CDS cache +Content for delivery is cached in CDS and refreshed automatically every hour. +This command triggers a refresh of cached content on the fly. + +By default, invalidation does not remove existing cached content, but +starts the process of updating with latest translations from Transifex. + +Passing the --purge option, cached content will be forced to be deleted, +however use that with caution, as it may introduce downtime of +translation delivery to the apps until fresh content is cached in the CDS. + +To invalidate translations some environment variables must be set: +TRANSIFEX_TOKEN= +TRANSIFEX_SECRET= +(optional) TRANSIFEX_CDS_HOST= + +or passed as --token= --secret= parameters + +Default CDS Host is https://cds.svc.transifex.net + +Examples: +txjs-cli invalidate +txjs-cli invalidate --purge +txjs-cli invalidate --token=mytoken --secret=mysecret +TRANSIFEX_TOKEN=mytoken TRANSIFEX_SECRET=mysecret txjs-cli invalidate +`; + +InvalidateCommand.args = []; + +InvalidateCommand.flags = { + purge: flags.boolean({ + description: 'force delete CDS cached content', + default: false, + }), + token: flags.string({ + description: 'native project public token', + default: '', + }), + secret: flags.string({ + description: 'native project secret', + default: '', + }), + 'cds-host': flags.string({ + description: 'CDS host URL', + default: '', + }), +}; + +module.exports = InvalidateCommand; diff --git a/packages/cli/src/commands/push.js b/packages/cli/src/commands/push.js index f73bf220..1e1e27dd 100644 --- a/packages/cli/src/commands/push.js +++ b/packages/cli/src/commands/push.js @@ -170,7 +170,7 @@ class PushCommand extends Command { } } -PushCommand.description = `Detect translatable strings and push content to Transifex +PushCommand.description = `detect and push source content to Transifex Parse .js, .ts, .jsx and .tsx files and detect phrases marked for translation by Transifex Native toolkit for Javascript and upload them to Transifex for translation. @@ -204,28 +204,28 @@ PushCommand.args = [{ PushCommand.flags = { 'dry-run': flags.boolean({ - description: 'Dry run, do not push to Transifex', + description: 'dry run, do not push to Transifex', default: false, }), verbose: flags.boolean({ char: 'v', - description: 'Verbose output', + description: 'verbose output', default: false, }), purge: flags.boolean({ - description: 'Purge content on Transifex', + description: 'purge content on Transifex', default: false, }), token: flags.string({ - description: 'Native project public token', + description: 'native project public token', default: '', }), secret: flags.string({ - description: 'Native project secret', + description: 'native project secret', default: '', }), tags: flags.string({ - description: 'Globally tag strings', + description: 'globally tag strings', default: '', }), 'cds-host': flags.string({ diff --git a/packages/cli/test/commands/invalidate.test.js b/packages/cli/test/commands/invalidate.test.js new file mode 100644 index 00000000..33fe4587 --- /dev/null +++ b/packages/cli/test/commands/invalidate.test.js @@ -0,0 +1,44 @@ +/* globals describe */ +const { expect, test } = require('@oclif/test'); + +describe('invalidate command', () => { + test + .nock('https://cds.svc.transifex.net', (api) => api + .post('/invalidate') + .reply(200, { + status: 'success', + token: 't', + count: 5, + })) + .stdout() + .command(['invalidate', '--secret=s', '--token=t']) + .it('invalidates content', (ctx) => { + expect(ctx.stdout).to.contain('5 records invalidated'); + }); + + test + .nock('https://cds.svc.transifex.net', (api) => api + .post('/purge') + .reply(200, { + status: 'success', + token: 't', + count: 10, + })) + .stdout() + .command(['invalidate', '--purge', '--secret=s', '--token=t']) + .it('invalidates content', (ctx) => { + expect(ctx.stdout).to.contain('10 records invalidated'); + }); + + test + .nock('https://cds.svc.transifex.net', (api) => api + .post('/invalidate') + .reply(403)) + .stdout() + .stderr() + .command(['invalidate', '--secret=s', '--token=t']) + .exit(2) + .it('handles invalidate error', (ctx) => { + expect(ctx.stdout).to.contain('Status code: 403'); + }); +}); From 5181b6aeec74e5cbd80bddd722979f3821c8786e Mon Sep 17 00:00:00 2001 From: nbasili Date: Thu, 18 Feb 2021 12:54:16 +0200 Subject: [PATCH 2/2] Add tag filtering to cli push command --- packages/cli/README.md | 20 ++++++----- packages/cli/src/api/extract.js | 47 +++++++++++++++++++++---- packages/cli/src/commands/push.js | 28 ++++++++++++--- packages/cli/test/api/extract.test.js | 4 +-- packages/cli/test/commands/push.test.js | 26 ++++++++++++++ packages/cli/test/fixtures/tags.js | 5 +++ 6 files changed, 108 insertions(+), 22 deletions(-) create mode 100644 packages/cli/test/fixtures/tags.js diff --git a/packages/cli/README.md b/packages/cli/README.md index c9220af6..b64b0b42 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -81,13 +81,15 @@ ARGUMENTS PATTERN [default: **/*.{js,jsx,ts,tsx}] file pattern to scan for strings OPTIONS - -v, --verbose verbose output - --cds-host=cds-host CDS host URL - --dry-run dry run, do not push to Transifex - --purge purge content on Transifex - --secret=secret native project secret - --tags=tags globally tag strings - --token=token native project public token + -v, --verbose verbose output + --append-tags=append-tags append tags to strings + --cds-host=cds-host CDS host URL + --dry-run dry run, do not push to Transifex + --purge purge content on Transifex + --secret=secret native project secret + --token=token native project public token + --with-tags-only=with-tags-only push strings with specific tags + --without-tags-only=without-tags-only push strings without specific tags DESCRIPTION Parse .js, .ts, .jsx and .tsx files and detect phrases marked for @@ -109,7 +111,9 @@ DESCRIPTION txjs-cli push /home/repo/src txjs-cli push "*.js" txjs-cli push --dry-run - txjs-cli push --tags="master,release:2.5" + txjs-cli push --append-tags="master,release:2.5" + txjs-cli push --with-tags-only="home,error" + txjs-cli push --without-tags-only="custom" txjs-cli push --token=mytoken --secret=mysecret TRANSIFEX_TOKEN=mytoken TRANSIFEX_SECRET=mysecret txjs-cli push ``` diff --git a/packages/cli/src/api/extract.js b/packages/cli/src/api/extract.js index 96f75496..61b8dc74 100644 --- a/packages/cli/src/api/extract.js +++ b/packages/cli/src/api/extract.js @@ -19,7 +19,7 @@ const { stringToArray, mergeArrays } = require('./utils'); * @param {Number} params._charlimit * @param {Number} params._tags * @param {String} occurence - * @param {String[]} globalTags + * @param {String[]} appendTags * @returns {Object} Payload * @returns {String} Payload.string * @returns {String} Payload.key @@ -29,7 +29,7 @@ const { stringToArray, mergeArrays } = require('./utils'); * @returns {String[]} Payload.meta.tags * @returns {String[]} Payload.meta.occurrences */ -function createPayload(string, params, occurence, globalTags) { +function createPayload(string, params, occurence, appendTags) { return { string, key: generateKey(string, params), @@ -37,12 +37,36 @@ function createPayload(string, params, occurence, globalTags) { context: stringToArray(params._context), developer_comment: params._comment, character_limit: params._charlimit ? parseInt(params._charlimit, 10) : undefined, - tags: mergeArrays(stringToArray(params._tags), globalTags), + tags: mergeArrays(stringToArray(params._tags), appendTags), occurrences: [occurence], }, _.isNil), }; } +/** + * Check if payload coming from createPayload is valid based on tag filters + * + * @param {Object} payload + * @param {String[]} options.filterWithTags + * @param {String[]} options.filterWithoutTags + * @returns {Boolean} + */ +function isPayloadValid(payload, options = {}) { + const { filterWithTags, filterWithoutTags } = options; + let isValid = true; + _.each(filterWithTags, (tag) => { + if (!_.includes(payload.meta.tags, tag)) { + isValid = false; + } + }); + _.each(filterWithoutTags, (tag) => { + if (_.includes(payload.meta.tags, tag)) { + isValid = false; + } + }); + return isValid; +} + /** * Check if callee is a valid Transifex Native function * @@ -92,10 +116,14 @@ function _parse(source) { * * @param {String} file absolute file path * @param {String} relativeFile occurence - * @param {String[]} globalTags + * @param {Object} options + * @param {String[]} options.appendTags + * @param {String[]} options.filterWithTags + * @param {String[]} options.filterWithoutTags * @returns {Object} */ -function extractPhrases(file, relativeFile, globalTags) { +function extractPhrases(file, relativeFile, options = {}) { + const { appendTags } = options; const HASHES = {}; const source = fs.readFileSync(file, 'utf8'); const ast = _parse(source); @@ -124,7 +152,9 @@ function extractPhrases(file, relativeFile, globalTags) { }); } - const partial = createPayload(string, params, relativeFile, globalTags); + const partial = createPayload(string, params, relativeFile, appendTags); + if (!isPayloadValid(partial, options)) return; + mergePayload(HASHES, { [partial.key]: { string: partial.string, @@ -156,7 +186,10 @@ function extractPhrases(file, relativeFile, globalTags) { }); if (!string) return; - const partial = createPayload(string, params, relativeFile, globalTags); + + const partial = createPayload(string, params, relativeFile, appendTags); + if (!isPayloadValid(partial, options)) return; + mergePayload(HASHES, { [partial.key]: { string: partial.string, diff --git a/packages/cli/src/commands/push.js b/packages/cli/src/commands/push.js index 1e1e27dd..d73d699f 100644 --- a/packages/cli/src/commands/push.js +++ b/packages/cli/src/commands/push.js @@ -40,7 +40,9 @@ class PushCommand extends Command { filePattern = path.join(filePattern, '**/*.{js,jsx,ts,tsx}'); } - const globalTags = stringToArray(flags.tags); + const appendTags = stringToArray(flags['append-tags']); + const filterWithTags = stringToArray(flags['with-tags-only']); + const filterWithoutTags = stringToArray(flags['without-tags-only']); this.log('Parsing all files to detect translatable content...'); @@ -64,11 +66,17 @@ class PushCommand extends Command { }); bar.start(allFiles.length, 0); + const extractOptions = { + appendTags, + filterWithTags, + filterWithoutTags, + }; + _.each(allFiles, (file) => { const relativeFile = file.replace(pwd, ''); bar.increment({ file: relativeFile.gray }); try { - const data = extractPhrases(file, relativeFile, globalTags); + const data = extractPhrases(file, relativeFile, extractOptions); tree[relativeFile] = data; if (_.isEmpty(data)) { emptyFiles += 1; @@ -190,7 +198,9 @@ txjs-cli push src/ txjs-cli push /home/repo/src txjs-cli push "*.js" txjs-cli push --dry-run -txjs-cli push --tags="master,release:2.5" +txjs-cli push --append-tags="master,release:2.5" +txjs-cli push --with-tags-only="home,error" +txjs-cli push --without-tags-only="custom" txjs-cli push --token=mytoken --secret=mysecret TRANSIFEX_TOKEN=mytoken TRANSIFEX_SECRET=mysecret txjs-cli push `; @@ -224,8 +234,16 @@ PushCommand.flags = { description: 'native project secret', default: '', }), - tags: flags.string({ - description: 'globally tag strings', + 'append-tags': flags.string({ + description: 'append tags to strings', + default: '', + }), + 'with-tags-only': flags.string({ + description: 'push strings with specific tags', + default: '', + }), + 'without-tags-only': flags.string({ + description: 'push strings without specific tags', default: '', }), 'cds-host': flags.string({ diff --git a/packages/cli/test/api/extract.test.js b/packages/cli/test/api/extract.test.js index 37bc50fa..f93aa492 100644 --- a/packages/cli/test/api/extract.test.js +++ b/packages/cli/test/api/extract.test.js @@ -32,8 +32,8 @@ describe('extractPhrases', () => { }); }); - it('works with global tags', async () => { - expect(await extractPhrases('test/fixtures/webpack.js', 'webpack.js', ['g1', 'g2'])) + it('works with append tags', async () => { + expect(await extractPhrases('test/fixtures/webpack.js', 'webpack.js', { appendTags: ['g1', 'g2'] })) .to.deep.equal({ '6f48100ca5a57d2db9b685a8373be8a6': { string: 'Text 1', diff --git a/packages/cli/test/commands/push.test.js b/packages/cli/test/commands/push.test.js index b618760a..59804dae 100644 --- a/packages/cli/test/commands/push.test.js +++ b/packages/cli/test/commands/push.test.js @@ -42,6 +42,32 @@ describe('push command', () => { expect(ctx.stdout).to.not.contain('f2138b2131e064313c369b20006549df: Text 1'); }); + test + .stdout() + .command(['push', 'test/fixtures/tags.js', '--dry-run', '-v', '--append-tags=custom']) + .it('append tags', (ctx) => { + expect(ctx.stdout).to.contain('tags: ["tag1","tag2","custom"]'); + expect(ctx.stdout).to.contain('tags: ["tag2","tag3","custom"]'); + expect(ctx.stdout).to.contain('tags: ["custom"]'); + }); + + test + .stdout() + .command(['push', 'test/fixtures/tags.js', '--dry-run', '-v', '--with-tags-only=tag1']) + .it('filters-in tags', (ctx) => { + expect(ctx.stdout).to.not.contain('tag3'); + expect(ctx.stdout).to.contain('tag1'); + }); + + test + .stdout() + .command(['push', 'test/fixtures/tags.js', '--dry-run', '-v', '--without-tags-only=tag1']) + .it('filters-out tags', (ctx) => { + expect(ctx.stdout).to.contain('tag2'); + expect(ctx.stdout).to.contain('tag3'); + expect(ctx.stdout).to.not.contain('tag1'); + }); + test .nock('https://cds.svc.transifex.net', (api) => api .post('/content') diff --git a/packages/cli/test/fixtures/tags.js b/packages/cli/test/fixtures/tags.js new file mode 100644 index 00000000..30136758 --- /dev/null +++ b/packages/cli/test/fixtures/tags.js @@ -0,0 +1,5 @@ +import { t } from '@transifex/native'; + +t('Text 1', { _tags: 'tag1,tag2' }); +t('Text 2', { _tags: 'tag2,tag3' }); +t('Text 3');