From 6845e0adea615c8159aed33f84d2f3212997c93d Mon Sep 17 00:00:00 2001 From: Semisol Date: Thu, 25 Apr 2024 01:25:23 +0300 Subject: [PATCH] translate: refactor, add noswhere provider Also includes tests. Signed-off-by: Semisol Co-authored-by: William Casarin Signed-off-by: William Casarin --- README.md | 7 ++- package.json | 9 ++-- src/translate.js | 97 ++++++++++++++++----------------------- src/translate/deepl.js | 57 +++++++++++++++++++++++ src/translate/mock.js | 13 ++++++ src/translate/noswhere.js | 61 ++++++++++++++++++++++++ test/translate.test.js | 55 ++++++++++++++-------- test_utils/mock_deepl.js | 15 ------ 8 files changed, 218 insertions(+), 96 deletions(-) create mode 100644 src/translate/deepl.js create mode 100644 src/translate/mock.js create mode 100644 src/translate/noswhere.js delete mode 100644 test_utils/mock_deepl.js diff --git a/README.md b/README.md index 239440d..84b1eb2 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,14 @@ The Damus API backend for Damus Purple and other functionality. #### Essential - `DB_PATH`: Path to the folder where to save mdb files. -- `DEEPL_KEY`: API key for DeepL translation service (Can be set to something bogus for local testing with mock translations) - `TESTFLIGHT_URL`: URL for the TestFlight app (optional) +#### Translations + +- `TRANSLATION_PROVIDER`: The translation provider to use, can be: `mock`, `deepl`, `noswhere` +- `DEEPL_KEY`: The DeepL key to use for DeepL translations if enabled. +- `NOSWHERE_KEY`: The Noswhere key to use for Noswhere translations if enabled. + #### Apple In-App Purchase (IAP) - `ENABLE_IAP_PAYMENTS`: Set to `"true"` to enable Apple In-App Purchase payment endpoints. diff --git a/package.json b/package.json index 2f0b63f..56c2ebe 100644 --- a/package.json +++ b/package.json @@ -10,14 +10,13 @@ "damus-api": "./src/index.js" }, "scripts": { - "test": "ALLOW_HTTP_AUTH=\"true\" DEEPL_KEY=123 tap test/*.test.js", + "test": "ALLOW_HTTP_AUTH=\"true\" DEEPL_KEY=123 TRANSLATION_PROVIDER=mock tap test/*.test.js", "test-translate": "ALLOW_HTTP_AUTH=\"true\" TRANSLATION_PROVIDER=mock tap test/translate.test.js", + "test-noswhere": "ALLOW_HTTP_AUTH=\"true\" TRANSLATION_PROVIDER=noswhere tap test/translate.test.js", "debug-test": "ALLOW_HTTP_AUTH=\"true\" DEEPL_KEY=123 tap test/*.test.js --timeout=2400", "start": "node src/index.js", - "mock_deepl": "node test_utils/mock_deepl.js", - "start_with_mock": "DEEPL_KEY=123 DEEPL_URL=http://localhost:8990 ENABLE_HTTP_AUTH=\"true\" node src/index.js", - "dev": "npm run mock_deepl & npm run start_with_mock", - "dev-debug": "npm run mock_deepl & DEEPL_KEY=123 DEEPL_URL=http://localhost:8990 ENABLE_HTTP_AUTH=\"true\" node inspect src/index.js", + "dev": "TRANSLATION_PROVIDER=mock ENABLE_HTTP_AUTH=\"true\" node src/index.js", + "dev-debug": "TRANSLATION_PROVIDER=mock ENABLE_HTTP_AUTH=\"true\" node --inspect src/index.js", "type-check": "tsc --checkJs --allowJs src/*.js --noEmit --skipLibCheck", "type-check-path": "tsc --checkJs --allowJs --noEmit --skipLibCheck" }, diff --git a/src/translate.js b/src/translate.js index c46e8ed..4d08345 100644 --- a/src/translate.js +++ b/src/translate.js @@ -2,27 +2,29 @@ const util = require('./server_helpers') const crypto = require('crypto') const current_time = require('./utils').current_time +const SUPPORTED_TRANSLATION_PROVIDERS = new Set(["mock", "noswhere", "deepl"]) +let translation_provider = null -const translate_sources = new Set(['BG', 'CS', 'DA', 'DE', 'EL', 'EN', 'ES', 'ET', 'FI', 'FR', 'HU', 'ID', 'IT', 'JA', 'KO', 'LT', 'LV', 'NB', 'NL', 'PL', 'PT', 'RO', 'RU', 'SK', 'SL', 'SV', 'TR', 'UK', 'ZH']) -const translate_targets = new Set(['BG', 'CS', 'DA', 'DE', 'EL', 'EN', 'EN-GB', 'EN-US', 'ES', 'ET', 'FI', 'FR', 'HU', 'ID', 'IT', 'JA', 'KO', 'LT', 'LV', 'NB', 'NL', 'PL', 'PT', 'PT-BR', 'PT-PT', 'RO', 'RU', 'SK', 'SL', 'SV', 'TR', 'UK', 'ZH']) +if (!process.env.TRANSLATION_PROVIDER) { + throw new Error("expected TRANSLATION_PROVIDER") +} -const DEEPL_KEY = process.env.DEEPL_KEY -const DEEPL_URL = process.env.DEEPL_URL || 'https://api.deepl.com/v2/translate' +if (!SUPPORTED_TRANSLATION_PROVIDERS.has(process.env.TRANSLATION_PROVIDER)) { + throw new Error("translation provider not supported") +} -if (!DEEPL_KEY) - throw new Error("expected DEEPL_KEY env var") +// this is safe, as the input value is restricted to known good ones. +translation_provider = new (require("./translate/" + process.env.TRANSLATION_PROVIDER + ".js"))() async function validate_payload(payload) { - if (!payload.source) - return { ok: false, message: 'missing source' } - if (!payload.target) - return { ok: false, message: 'missing target' } - if (!payload.q) - return { ok: false, message: 'missing q' } - if (!translate_sources.has(payload.source)) - return { ok: false, message: 'invalid translation source' } - if (!translate_targets.has(payload.target)) - return { ok: false, message: 'invalid translation target' } + if (typeof payload.source !== "string") + return { ok: false, message: 'bad source' } + if (typeof payload.target !== "string") + return { ok: false, message: 'bad target' } + if (typeof payload.q !== "string") + return { ok: false, message: 'bad q' } + if (!translation_provider.canTranslate(payload.source, payload.target)) + return { ok: false, message: 'invalid translation source/target' } return { ok: true, message: 'valid' } } @@ -30,43 +32,22 @@ async function validate_payload(payload) { function hash_payload(payload) { const hash = crypto.createHash('sha256') hash.update(payload.q) - hash.update(payload.source) - hash.update(payload.target) + hash.update(payload.source.toUpperCase()) + hash.update(payload.target.toUpperCase()) return hash.digest() } -async function deepl_translate_text(payload) { - let resp = await fetch(DEEPL_URL, { - method: 'POST', - headers: { - 'Authorization': `DeepL-Auth-Key ${DEEPL_KEY}`, - 'Content-Type': 'application/json' - }, - body: JSON.stringify({ - text: [payload.q], - source_lang: payload.source, - target_lang: payload.target, - }) - }) - - let data = await resp.json() - - if (data.translations && data.translations.length > 0) { - return data.translations[0].text; - } - - return null -} - async function translate_payload(api, res, payload, trans_id) { // we might already be translating this const job = api.translation.queue[trans_id] if (job) { - let text = await job - if (text === null) - return util.error_response(res, 'deepl translation error') - - return util.json_response(res, { text }) + try { + let result = await job + return util.json_response(res, { text: result.text }) + } catch(e) { + console.error("translation error: %o", e) + return util.error_response(res, 'translation error') + } } // we might have it in the database already @@ -78,26 +59,28 @@ async function translate_payload(api, res, payload, trans_id) { return util.json_response(res, { text }) } - const new_job = deepl_translate_text(payload) + const new_job = translation_provider.translate(payload.source, payload.target, payload.q) api.translation.queue[trans_id] = new_job - let text = await new_job - if (text === null) { + let result + + try { + result = await new_job + } catch(e) { + console.error("translation error: %o", e) + return util.error_response(res, 'translation error') + } finally { delete api.translation.queue[trans_id] - return util.error_response(res, 'deepl translation error') } - // return results immediately - util.json_response(res, { text }) + util.json_response(res, { text: result.text }) // write result to db await api.dbs.translations.put(trans_id, { - text: text, + text: result.text, translated_at: current_time(), payload: payload }) - - delete api.translation.queue[trans_id] } function payload_is_data(q) { @@ -111,8 +94,8 @@ function payload_is_data(q) { async function handle_translate(api, req, res) { let id try { - const source = req.query.source.toUpperCase() - const target = req.query.target.toUpperCase() + const source = req.query.source.toLowerCase() + const target = req.query.target.toLowerCase() const q = req.query.q if (payload_is_data(q)) return util.invalid_request(res, `payload is data`) diff --git a/src/translate/deepl.js b/src/translate/deepl.js new file mode 100644 index 0000000..9145320 --- /dev/null +++ b/src/translate/deepl.js @@ -0,0 +1,57 @@ +const translate_sources = new Set([ + 'bg', 'cs', 'da', 'de', 'el', + 'en', 'es', 'et', 'fi', 'fr', + 'hu', 'id', 'it', 'ja', 'ko', + 'lt', 'lv', 'nb', 'nl', 'pl', + 'pt', 'ro', 'ru', 'sk', 'sl', + 'sv', 'tr', 'uk', 'zh' +]) +const translate_targets = new Set([ + 'bg', 'cs', 'da', 'de', + 'el', 'en', 'en-gb', 'en-us', + 'es', 'et', 'fi', 'fr', + 'hu', 'id', 'it', 'ja', + 'ko', 'lt', 'lv', 'nb', + 'nl', 'pl', 'pt', 'pt-br', + 'pt-pt', 'ro', 'ru', 'sk', + 'sl', 'sv', 'tr', 'uk', + 'zh' +]) + +module.exports = class DeepLTranslator { + #deeplURL = process.env.DEEPL_URL || 'https://api.deepl.com/v2/translate' + #deeplKey = process.env.DEEPL_KEY + constructor() { + if (!this.#deeplKey) + throw new Error("expected DEEPL_KEY env var") + } + canTranslate(from_lang, to_lang) { + return translate_sources.has(from_lang) && translate_targets.has(to_lang) + } + async translate(from_lang, to_lang, text) { + let resp = await fetch(this.#deeplURL, { + method: 'POST', + headers: { + 'Authorization': `DeepL-Auth-Key ${this.#deeplKey}`, + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + text: [text], + source_lang: from_lang.toUpperCase(), + target_lang: to_lang.toUpperCase(), + }) + }) + + if (!resp.ok) throw new Error("error translating: API failed with " + resp.status + " " + resp.statusText) + + let data = await resp.json() + + if (data.translations && data.translations.length > 0) { + return { + text: data.translations[0].text + } + } + + throw new Error("error translating: no response") + } +} \ No newline at end of file diff --git a/src/translate/mock.js b/src/translate/mock.js new file mode 100644 index 0000000..9f0109c --- /dev/null +++ b/src/translate/mock.js @@ -0,0 +1,13 @@ +module.exports = class MockTranslator { + constructor() { + + } + canTranslate(from_lang, to_lang) { + return true + } + async translate(from_lang, to_lang, text) { + return { + text: "Mock translation" + } + } +} \ No newline at end of file diff --git a/src/translate/noswhere.js b/src/translate/noswhere.js new file mode 100644 index 0000000..becaf75 --- /dev/null +++ b/src/translate/noswhere.js @@ -0,0 +1,61 @@ +module.exports = class NoswhereTranslator { + #noswhereURL = process.env.NOSWHERE_URL || 'https://translate.api.noswhere.com/api' + #noswhereKey = process.env.NOSWHERE_KEY + #type = "default" + #fromLangs = new Set() + #toLangs = new Set() + constructor() { + if (!this.#noswhereKey) + throw new Error("expected NOSWHERE_KEY env var") + this.#loadTranslationLangs() + } + async #loadTranslationLangs() { + let resp = await fetch(this.#noswhereURL + "/langs", { + method: 'GET', + headers: { + 'X-Noswhere-Key': this.#noswhereKey, + 'Content-Type': 'application/json' + } + }) + let data = await resp.json() + if (!resp.ok) { + throw new Error(`error getting translation langs: API failed with ${resp.status} ${data.error} (request: ${resp.headers.get("x-noswhere-request")})`) + } + if (!data[this.#type]) { + throw new Error(`type ${this.#type} not supported for translation`) + } + this.#fromLangs = new Set(data[this.#type].from) + this.#toLangs = new Set(data[this.#type].to) + } + canTranslate(from_lang, to_lang) { + if (this.#fromLangs.size === 0) return true // assume true until we get the list of languages + return this.#fromLangs.has(from_lang) && this.#toLangs.has(to_lang) + } + async translate(from_lang, to_lang, text) { + let resp = await fetch(this.#noswhereURL + "/translate", { + method: 'POST', + headers: { + 'X-Noswhere-Key': this.#noswhereKey, + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + text: text, + src_lang: from_lang, + dst_lang: to_lang, + }) + }) + + let data = await resp.json() + if (!resp.ok) { + throw new Error(`error translating: API failed with ${resp.status} ${data.error} (request: ${resp.headers.get("x-noswhere-request")})`) + } + + if (data.result) { + return { + text: data.result + } + } + + throw new Error("error translating: no response") + } +} \ No newline at end of file diff --git a/test/translate.test.js b/test/translate.test.js index 081ce15..691e98a 100644 --- a/test/translate.test.js +++ b/test/translate.test.js @@ -7,9 +7,11 @@ const current_time = require('../src/utils').current_time; const { supertest_client, TEST_BASE_URL } = require('./controllers/utils'); +const TRANSLATION_PROVIDER = process.env.TRANSLATION_PROVIDER + // MARK: - Tests -tap.test('translate_payload - Existing translation in database (mocked db)', async (t) => { +tap.test(`translate_payload - Existing translation in database (${TRANSLATION_PROVIDER})`, async (t) => { const api = await generate_test_api(t, { simulate_existing_translation_in_db: "", simulate_account_found_in_db: true, @@ -29,7 +31,7 @@ tap.test('translate_payload - Existing translation in database (mocked db)', asy t.end(); }); -tap.test(`translate_payload - reject json data (mocked server)`, async (t) => { +tap.test(`translate_payload - reject json data (${TRANSLATION_PROVIDER} server)`, async (t) => { const api = await generate_test_api(t, { simulate_existing_translation_in_db: false, simulate_account_found_in_db: true, @@ -51,26 +53,43 @@ tap.test(`translate_payload - reject json data (mocked server)`, async (t) => { t.same(res.body, expected_result, 'Response should match expected value'); }) -tap.test('translate_payload - New translation (mocked server)', async (t) => { +tap.test(`translate_payload - New translation (${TRANSLATION_PROVIDER} server)`, async (t) => { const api = await generate_test_api(t, { simulate_existing_translation_in_db: false, simulate_account_found_in_db: true, }); - const expected_result = { - text: "Mock translation", - }; + const expected_text = "Mock translation"; // Create a stub for fetch - const fetchStub = sinon.stub(global, 'fetch').returns(Promise.resolve({ - json: async () => { - return { - translations: [ - expected_result - ] - }; + let fetchStub = null + + switch (TRANSLATION_PROVIDER) { + case "deepl": { + fetchStub = sinon.stub(global, 'fetch').returns(Promise.resolve({ + json: async () => { + return { + translations: [ + {text: expected_text} + ] + }; + }, + ok: true + })); + break } - })); + case "noswhere": { + sinon.stub(global, 'fetch').returns(Promise.resolve({ + json: async () => { + return { + result: expected_text, + }; + }, + ok: true + })); + break + } + } const test_data = await generate_test_request_data(api); @@ -79,15 +98,15 @@ tap.test('translate_payload - New translation (mocked server)', async (t) => { .set('Authorization', 'Nostr ' + test_data.auth_note_base64) t.same(res.statusCode, 200, 'Response should be 200'); - t.same(res.body, expected_result, 'Response should match expected value'); + t.same(res.body, {text: expected_text}, 'Response should match expected value'); // Restore fetch - fetchStub.restore(); + fetchStub?.restore(); t.end(); }); -tap.test('translate - Account not found (mocked db)', async (t) => { +tap.test(`translate - Account not found (${TRANSLATION_PROVIDER})`, async (t) => { const api = await generate_test_api(t, { simulate_existing_translation_in_db: false, simulate_account_found_in_db: false, @@ -100,7 +119,7 @@ tap.test('translate - Account not found (mocked db)', async (t) => { t.end(); }); -tap.test('translate - Account expired (mocked db)', async (t) => { +tap.test(`translate - Account expired (${TRANSLATION_PROVIDER})`, async (t) => { const api = await generate_test_api(t, { simulate_existing_translation_in_db: false, simulate_account_expired: true, diff --git a/test_utils/mock_deepl.js b/test_utils/mock_deepl.js deleted file mode 100644 index 353f0a7..0000000 --- a/test_utils/mock_deepl.js +++ /dev/null @@ -1,15 +0,0 @@ -const http = require('http'); - -const server = http.createServer((req, res) => { - res.setHeader('Content-Type', 'application/json'); - res.end(JSON.stringify({ - translations: [ - { text: 'Mock translation' } - ] - })); -}); - -const port = 8990; -server.listen(port, () => { - console.log(`Server running on port ${port}`); -});