From 3cad4a36a35ba59706078a48ac7bd5ae451db1a0 Mon Sep 17 00:00:00 2001 From: hchawla Date: Wed, 29 Nov 2017 15:05:54 -0800 Subject: [PATCH 1/2] Add Merge Command --- gp-cli/README.md | 20 ++ .../ibm/g11n/pipeline/tools/cli/GPCmd.java | 1 + .../pipeline/tools/cli/MergeBundleCmd.java | 193 ++++++++++++++++++ 3 files changed, 214 insertions(+) create mode 100644 gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/MergeBundleCmd.java diff --git a/gp-cli/README.md b/gp-cli/README.md index 420a668..0b158b9 100644 --- a/gp-cli/README.md +++ b/gp-cli/README.md @@ -240,6 +240,26 @@ but user accounts and translation configurations are not transferred, because they are service instance specific. +#### merge (merge-bundle) + +Merge the changesof source bundle to the specified target bundle. This command allows you to merge a +bundle in the same service instance, or another service instance. +The following example merges the contents of exiting bundle *MyBundle* to +a bundle *MyBundle* in the same service instance (assuming MyBundle already exists). +``` +java -jar gp-cli.jar merge -b MyBundle -d MyNewBundle -j mycreds.json +``` +This command can be used to merge a bundle from a service instance to +another service instance. In this case, you need to specify the destination instance's +credentials with --dest-* options as below (assuming MyBundle already exists on destination). +``` +java -jar gp-cli.jar merge -b MyBundle -d MyBundle --dest-url https://gp-rest.ng.bluemix.net/translate/rest --dest-instance-id 9146abf71bb94513504a0eaf76d57804 --dest-user-id 52858e19ae57ba6f2d2ea7e38e9ab457 --dest-password o75YXQCK2obQLOvedkSslBTAyeUq7/+t -j mycreds.json +``` +Note: This command does not copy service managed properties, such as +updatedBy and updatedAt stored in each entity. The merged bundle +and resource entries will have new timestamp in updatedAt property +and the operator of this command will be set to updatedBy property. + --- ### User Commands diff --git a/gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/GPCmd.java b/gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/GPCmd.java index 73b42bf..b8f28bd 100644 --- a/gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/GPCmd.java +++ b/gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/GPCmd.java @@ -53,6 +53,7 @@ public static void main(String[] args) { jc.addCommand("export", new ExportCmd()); jc.addCommand("import", new ImportCmd()); jc.addCommand("list-mt-languages", new ListMTLanguagesCmd()); + jc.addCommand("merge-bundle", new MergeBundleCmd(), "merge"); //users jc.addCommand("list-users", new ListUsersCmd()); diff --git a/gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/MergeBundleCmd.java b/gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/MergeBundleCmd.java new file mode 100644 index 0000000..b995f96 --- /dev/null +++ b/gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/MergeBundleCmd.java @@ -0,0 +1,193 @@ +/* + * Copyright IBM Corp. 2016 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.ibm.g11n.pipeline.tools.cli; + +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import com.ibm.g11n.pipeline.client.BundleData; +import com.ibm.g11n.pipeline.client.BundleDataChangeSet; +import com.ibm.g11n.pipeline.client.NewBundleData; +import com.ibm.g11n.pipeline.client.NewResourceEntryData; +import com.ibm.g11n.pipeline.client.ResourceEntryData; +import com.ibm.g11n.pipeline.client.ServiceAccount; +import com.ibm.g11n.pipeline.client.ServiceClient; +import com.ibm.g11n.pipeline.client.ServiceException; +import com.ibm.g11n.pipeline.client.TranslationStatus; + +/** + * Merge a bundle. + * + * @author Harpreet K Chawla + */ +@Parameters(commandDescription = "Merge a bundle") +final class MergeBundleCmd extends BundleCmd { + @Parameter( + names = { "--dest-url"}, + description = "The destinaion service instance's URL") + private String destUrl; + + @Parameter( + names = { "--dest-instance-id"}, + description = "The destination service instance ID") + private String destInstanceId; + + @Parameter( + names = { "--dest-user-id"}, + description = "User ID used for the destination service instance") + private String destUserId; + + @Parameter( + names = { "--dest-password"}, + description = "Password used for the destination service instance") + private String destPassword; + + @Parameter( + names = {"-d", "--dest-bundle-id"}, + description = "The destination bundle ID", + required = true) + private String destBundleId; + + + @Override + protected void _execute() { + try { + ServiceClient srcClient = getClient(); + ServiceClient destClient = null; + + if (destUrl == null && destInstanceId == null && destPassword == null && destUserId == null) { + destClient = srcClient; + } else { + ServiceAccount account = ServiceAccount.getInstance(destUrl, destInstanceId, + destUserId, destPassword); + destClient = ServiceClient.getInstance(account); + } + + copyBundle(srcClient, bundleId, destClient, destBundleId); + } catch (ServiceException e) { + throw new RuntimeException(e); + } + + System.out.println("Bundle:" + bundleId + + " was successfull merged to the specified destination."); + } + + static void copyBundle(ServiceClient srcClient, String srcBundleId, + ServiceClient destClient, String destBundleId) throws ServiceException { + + // First, create the destination bundle without target languages + BundleData srcBundleInfo = srcClient.getBundleInfo(srcBundleId); + String srcLang = srcBundleInfo.getSourceLanguage(); + Set targetLangs = srcBundleInfo.getTargetLanguages(); + + NewBundleData newBundleData = new NewBundleData(srcLang); + newBundleData.setNotes(srcBundleInfo.getNotes()); + newBundleData.setMetadata(srcBundleInfo.getMetadata()); + newBundleData.setPartner(srcBundleInfo.getPartner()); + newBundleData.setNoTranslationPattern(srcBundleInfo.getNoTranslationPattern()); + newBundleData.setSegmentSeparatorPattern(srcBundleInfo.getSegmentSeparatorPattern()); + + //destClient.createBundle(destBundleId, newBundleData); + + // Upload resource data for source language and target languages + Set remainingLangs = targetLangs == null ? + Collections.emptySet() : new HashSet<>(targetLangs); + + Map srcResources = srcClient.getResourceEntries( + srcBundleId, srcLang); + + if (!srcResources.isEmpty()) { + uploadResources(destClient, destBundleId, srcLang, srcResources); + + // Upload resource data for target languages + // Removing already processed target languages from remainingLangs + // while iterating through the set, so we need to use Iterator here. + for (Iterator i = remainingLangs.iterator(); i.hasNext();) { + String trgLang = i.next(); + Map resources = srcClient.getResourceEntries(srcBundleId, trgLang); + if (!resources.isEmpty()) { + if (uploadResources(destClient, destBundleId, trgLang, resources)) { + // if something has been uploaded, the language was automatically + // added to the bundle's target language list. So remove the language + // from the 'remaining' language list. + i.remove(); + } + } + } + } + + if (!remainingLangs.isEmpty()) { + // There are missing target languages. + // This happens either when the source bundle is empty, + // or target bundles does not contain any successful translated + // contents. We want to set the set of target languages + // to the destination bundle. + assert targetLangs != null; + BundleDataChangeSet bundleDataChanges = new BundleDataChangeSet(); + bundleDataChanges.setTargetLanguages(targetLangs); + destClient.updateBundle(destBundleId, bundleDataChanges); + } + } + + /** + * Upload resource entries. An entry for a target language with status + * other than 'translated' will be excluded. + * + * @param client GP service client + * @param bundleId Bundle ID + * @param language Language ID + * @param resources Resource entries, originally read from another bundle + * @return true if this method actually uploaded any resource entries, or false + * if nothing uploaded. + * @throws ServiceException + */ + private static boolean uploadResources(ServiceClient client, String bundleId, + String language, Map resources) throws ServiceException { + Map newResources = + new HashMap(resources.size()); + for (Entry res : resources.entrySet()) { + ResourceEntryData resdata = res.getValue(); + TranslationStatus state = resdata.getTranslationStatus(); + if (state != TranslationStatus.SOURCE_LANGUAGE + && state != TranslationStatus.TRANSLATED) { + // Ignore an resource entry, if it's not in successful state + continue; + } + NewResourceEntryData newResData = new NewResourceEntryData(resdata.getValue()); + newResData.setNotes(resdata.getNotes()); + newResData.setSequenceNumber(resdata.getSequenceNumber()); + newResData.setMetadata(resdata.getMetadata()); + if (resdata.isReviewed()) { + newResData.setReviewed(Boolean.TRUE); + } + newResData.setPartnerStatus(resdata.getPartnerStatus()); + newResources.put(res.getKey(), newResData); + } + + if (newResources.isEmpty()) { + return false; + } + client.uploadResourceEntries(bundleId, language, newResources); + return true; + } +} \ No newline at end of file From 7df5f4f3673b38a3b37d3fd18865b19c896b4623 Mon Sep 17 00:00:00 2001 From: hchawla Date: Thu, 30 Nov 2017 15:23:06 -0800 Subject: [PATCH 2/2] Merge TR to Master bundle --- gp-cli/.settings/org.eclipse.jdt.core.prefs | 19 + gp-cli/.settings/org.eclipse.jdt.ui.prefs | 2 +- gp-cli/README.md | 19 +- .../tools/cli/ApproximateMatcher.java | 49 +++ .../ibm/g11n/pipeline/tools/cli/BaseCmd.java | 15 + .../ibm/g11n/pipeline/tools/cli/GPCmd.java | 2 +- .../pipeline/tools/cli/MergeBundleCmd.java | 193 ---------- .../tools/cli/MergeTranslationsCmd.java | 343 ++++++++++++++++++ 8 files changed, 431 insertions(+), 211 deletions(-) create mode 100644 gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/ApproximateMatcher.java delete mode 100644 gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/MergeBundleCmd.java create mode 100644 gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/MergeTranslationsCmd.java diff --git a/gp-cli/.settings/org.eclipse.jdt.core.prefs b/gp-cli/.settings/org.eclipse.jdt.core.prefs index 25a1a85..3836513 100644 --- a/gp-cli/.settings/org.eclipse.jdt.core.prefs +++ b/gp-cli/.settings/org.eclipse.jdt.core.prefs @@ -6,6 +6,7 @@ org.eclipse.jdt.core.compiler.problem.assertIdentifier=error org.eclipse.jdt.core.compiler.problem.enumIdentifier=error org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning org.eclipse.jdt.core.compiler.source=1.7 +org.eclipse.jdt.core.formatter.align_fields_grouping_blank_lines=2147483647 org.eclipse.jdt.core.formatter.align_type_members_on_columns=false org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression=16 org.eclipse.jdt.core.formatter.alignment_for_arguments_in_annotation=0 @@ -19,8 +20,10 @@ org.eclipse.jdt.core.formatter.alignment_for_compact_if=16 org.eclipse.jdt.core.formatter.alignment_for_conditional_expression=80 org.eclipse.jdt.core.formatter.alignment_for_enum_constants=0 org.eclipse.jdt.core.formatter.alignment_for_expressions_in_array_initializer=16 +org.eclipse.jdt.core.formatter.alignment_for_expressions_in_for_loop_header=0 org.eclipse.jdt.core.formatter.alignment_for_method_declaration=0 org.eclipse.jdt.core.formatter.alignment_for_multiple_fields=16 +org.eclipse.jdt.core.formatter.alignment_for_parameterized_type_references=0 org.eclipse.jdt.core.formatter.alignment_for_parameters_in_constructor_declaration=16 org.eclipse.jdt.core.formatter.alignment_for_parameters_in_method_declaration=16 org.eclipse.jdt.core.formatter.alignment_for_resources_in_try=80 @@ -30,6 +33,8 @@ org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_enum_declaration org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_type_declaration=16 org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_constructor_declaration=16 org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_method_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_type_arguments=0 +org.eclipse.jdt.core.formatter.alignment_for_type_parameters=0 org.eclipse.jdt.core.formatter.alignment_for_union_type_in_multicatch=16 org.eclipse.jdt.core.formatter.blank_lines_after_imports=1 org.eclipse.jdt.core.formatter.blank_lines_after_package=1 @@ -56,6 +61,7 @@ org.eclipse.jdt.core.formatter.brace_position_for_switch=end_of_line org.eclipse.jdt.core.formatter.brace_position_for_type_declaration=end_of_line org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_block_comment=false org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_javadoc_comment=false +org.eclipse.jdt.core.formatter.comment.count_line_length_from_starting_position=false org.eclipse.jdt.core.formatter.comment.format_block_comments=true org.eclipse.jdt.core.formatter.comment.format_header=false org.eclipse.jdt.core.formatter.comment.format_html=true @@ -88,6 +94,7 @@ org.eclipse.jdt.core.formatter.indent_statements_compare_to_body=true org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_cases=true org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_switch=false org.eclipse.jdt.core.formatter.indentation.size=4 +org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_enum_constant=insert org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_field=insert org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_local_variable=insert org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_method=insert @@ -282,12 +289,24 @@ org.eclipse.jdt.core.formatter.never_indent_block_comments_on_first_column=false org.eclipse.jdt.core.formatter.never_indent_line_comments_on_first_column=false org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_method_body=0 org.eclipse.jdt.core.formatter.number_of_empty_lines_to_preserve=1 +org.eclipse.jdt.core.formatter.parentheses_positions_in_annotation=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_catch_clause=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_enum_constant_declaration=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_for_statment=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_if_while_statement=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_lambda_declaration=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_method_delcaration=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_method_invocation=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_switch_statement=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_try_clause=common_lines org.eclipse.jdt.core.formatter.put_empty_statement_on_new_line=true org.eclipse.jdt.core.formatter.tabulation.char=space org.eclipse.jdt.core.formatter.tabulation.size=4 org.eclipse.jdt.core.formatter.use_on_off_tags=false org.eclipse.jdt.core.formatter.use_tabs_only_for_leading_indentations=false +org.eclipse.jdt.core.formatter.wrap_before_assignment_operator=false org.eclipse.jdt.core.formatter.wrap_before_binary_operator=true +org.eclipse.jdt.core.formatter.wrap_before_conditional_operator=true org.eclipse.jdt.core.formatter.wrap_before_or_operator_multicatch=true org.eclipse.jdt.core.formatter.wrap_outer_expressions_when_nested=true org.eclipse.jdt.core.javaFormatter=org.eclipse.jdt.core.defaultJavaFormatter diff --git a/gp-cli/.settings/org.eclipse.jdt.ui.prefs b/gp-cli/.settings/org.eclipse.jdt.ui.prefs index 27e5693..b2765b2 100644 --- a/gp-cli/.settings/org.eclipse.jdt.ui.prefs +++ b/gp-cli/.settings/org.eclipse.jdt.ui.prefs @@ -1,5 +1,5 @@ eclipse.preferences.version=1 formatter_profile=_gp -formatter_settings_version=12 +formatter_settings_version=13 org.eclipse.jdt.ui.javadoc=false org.eclipse.jdt.ui.text.custom_code_templates= diff --git a/gp-cli/README.md b/gp-cli/README.md index 0b158b9..b63386c 100644 --- a/gp-cli/README.md +++ b/gp-cli/README.md @@ -242,24 +242,11 @@ they are service instance specific. #### merge (merge-bundle) -Merge the changesof source bundle to the specified target bundle. This command allows you to merge a -bundle in the same service instance, or another service instance. -The following example merges the contents of exiting bundle *MyBundle* to -a bundle *MyBundle* in the same service instance (assuming MyBundle already exists). -``` -java -jar gp-cli.jar merge -b MyBundle -d MyNewBundle -j mycreds.json -``` -This command can be used to merge a bundle from a service instance to -another service instance. In this case, you need to specify the destination instance's -credentials with --dest-* options as below (assuming MyBundle already exists on destination). +Merge translations from the worker instance to the master instance + ``` -java -jar gp-cli.jar merge -b MyBundle -d MyBundle --dest-url https://gp-rest.ng.bluemix.net/translate/rest --dest-instance-id 9146abf71bb94513504a0eaf76d57804 --dest-user-id 52858e19ae57ba6f2d2ea7e38e9ab457 --dest-password o75YXQCK2obQLOvedkSslBTAyeUq7/+t -j mycreds.json +java -jar gp-cli.jar merge-translations -m master-credentials.json -j slave-credentials.json -b test ``` -Note: This command does not copy service managed properties, such as -updatedBy and updatedAt stored in each entity. The merged bundle -and resource entries will have new timestamp in updatedAt property -and the operator of this command will be set to updatedBy property. - --- ### User Commands diff --git a/gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/ApproximateMatcher.java b/gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/ApproximateMatcher.java new file mode 100644 index 0000000..a38e382 --- /dev/null +++ b/gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/ApproximateMatcher.java @@ -0,0 +1,49 @@ +/* + * Copyright IBM Corp. 2015,2016 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.ibm.g11n.pipeline.tools.cli; + +import com.ibm.icu.lang.UCharacter; + +public final class ApproximateMatcher { + private String text; + private String processedText; + + public ApproximateMatcher(String text) { + this.text = text; + this.processedText = processText(text); + } + + private static String processText(String text) { + // fold case + text = UCharacter.foldCase(text, true); + + // replace characters other than letter and number + // with space and trim. + return text.replaceAll("[^\\p{L}\\p{N}]+", " ").trim(); + } + + public boolean matches(String inText) { + if (text.equals(inText)) { + return true; + } + return processedText.equals(processText(inText)); + } + + public static boolean matches(String text1, String text2) { + ApproximateMatcher matcher = new ApproximateMatcher(text1); + return matcher.matches(text2); + } +} \ No newline at end of file diff --git a/gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/BaseCmd.java b/gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/BaseCmd.java index c4534fb..d437ace 100644 --- a/gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/BaseCmd.java +++ b/gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/BaseCmd.java @@ -65,6 +65,21 @@ static class JsonCredentials { String password; } + protected static ServiceClient getClient(String jsonCredsFile) { + JsonCredentials creds; + try (InputStreamReader reader = new InputStreamReader( + new FileInputStream(jsonCredsFile), StandardCharsets.UTF_8)) { + Gson gson = new Gson(); + creds = gson.fromJson(reader, JsonCredentials.class); + } catch (IOException e) { + throw new RuntimeException(e); + } + + ServiceAccount account = ServiceAccount.getInstance( + creds.url, creds.instanceId, creds.userId, creds.password); + return ServiceClient.getInstance(account); + } + protected ServiceClient getClient() { if (jsonCreds != null) { JsonCredentials creds; diff --git a/gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/GPCmd.java b/gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/GPCmd.java index b8f28bd..b0e4860 100644 --- a/gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/GPCmd.java +++ b/gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/GPCmd.java @@ -53,7 +53,7 @@ public static void main(String[] args) { jc.addCommand("export", new ExportCmd()); jc.addCommand("import", new ImportCmd()); jc.addCommand("list-mt-languages", new ListMTLanguagesCmd()); - jc.addCommand("merge-bundle", new MergeBundleCmd(), "merge"); + jc.addCommand("merge-translations", new MergeTranslationsCmd(), "merge-translations"); //users jc.addCommand("list-users", new ListUsersCmd()); diff --git a/gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/MergeBundleCmd.java b/gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/MergeBundleCmd.java deleted file mode 100644 index b995f96..0000000 --- a/gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/MergeBundleCmd.java +++ /dev/null @@ -1,193 +0,0 @@ -/* - * Copyright IBM Corp. 2016 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.ibm.g11n.pipeline.tools.cli; - -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; - -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; -import com.ibm.g11n.pipeline.client.BundleData; -import com.ibm.g11n.pipeline.client.BundleDataChangeSet; -import com.ibm.g11n.pipeline.client.NewBundleData; -import com.ibm.g11n.pipeline.client.NewResourceEntryData; -import com.ibm.g11n.pipeline.client.ResourceEntryData; -import com.ibm.g11n.pipeline.client.ServiceAccount; -import com.ibm.g11n.pipeline.client.ServiceClient; -import com.ibm.g11n.pipeline.client.ServiceException; -import com.ibm.g11n.pipeline.client.TranslationStatus; - -/** - * Merge a bundle. - * - * @author Harpreet K Chawla - */ -@Parameters(commandDescription = "Merge a bundle") -final class MergeBundleCmd extends BundleCmd { - @Parameter( - names = { "--dest-url"}, - description = "The destinaion service instance's URL") - private String destUrl; - - @Parameter( - names = { "--dest-instance-id"}, - description = "The destination service instance ID") - private String destInstanceId; - - @Parameter( - names = { "--dest-user-id"}, - description = "User ID used for the destination service instance") - private String destUserId; - - @Parameter( - names = { "--dest-password"}, - description = "Password used for the destination service instance") - private String destPassword; - - @Parameter( - names = {"-d", "--dest-bundle-id"}, - description = "The destination bundle ID", - required = true) - private String destBundleId; - - - @Override - protected void _execute() { - try { - ServiceClient srcClient = getClient(); - ServiceClient destClient = null; - - if (destUrl == null && destInstanceId == null && destPassword == null && destUserId == null) { - destClient = srcClient; - } else { - ServiceAccount account = ServiceAccount.getInstance(destUrl, destInstanceId, - destUserId, destPassword); - destClient = ServiceClient.getInstance(account); - } - - copyBundle(srcClient, bundleId, destClient, destBundleId); - } catch (ServiceException e) { - throw new RuntimeException(e); - } - - System.out.println("Bundle:" + bundleId - + " was successfull merged to the specified destination."); - } - - static void copyBundle(ServiceClient srcClient, String srcBundleId, - ServiceClient destClient, String destBundleId) throws ServiceException { - - // First, create the destination bundle without target languages - BundleData srcBundleInfo = srcClient.getBundleInfo(srcBundleId); - String srcLang = srcBundleInfo.getSourceLanguage(); - Set targetLangs = srcBundleInfo.getTargetLanguages(); - - NewBundleData newBundleData = new NewBundleData(srcLang); - newBundleData.setNotes(srcBundleInfo.getNotes()); - newBundleData.setMetadata(srcBundleInfo.getMetadata()); - newBundleData.setPartner(srcBundleInfo.getPartner()); - newBundleData.setNoTranslationPattern(srcBundleInfo.getNoTranslationPattern()); - newBundleData.setSegmentSeparatorPattern(srcBundleInfo.getSegmentSeparatorPattern()); - - //destClient.createBundle(destBundleId, newBundleData); - - // Upload resource data for source language and target languages - Set remainingLangs = targetLangs == null ? - Collections.emptySet() : new HashSet<>(targetLangs); - - Map srcResources = srcClient.getResourceEntries( - srcBundleId, srcLang); - - if (!srcResources.isEmpty()) { - uploadResources(destClient, destBundleId, srcLang, srcResources); - - // Upload resource data for target languages - // Removing already processed target languages from remainingLangs - // while iterating through the set, so we need to use Iterator here. - for (Iterator i = remainingLangs.iterator(); i.hasNext();) { - String trgLang = i.next(); - Map resources = srcClient.getResourceEntries(srcBundleId, trgLang); - if (!resources.isEmpty()) { - if (uploadResources(destClient, destBundleId, trgLang, resources)) { - // if something has been uploaded, the language was automatically - // added to the bundle's target language list. So remove the language - // from the 'remaining' language list. - i.remove(); - } - } - } - } - - if (!remainingLangs.isEmpty()) { - // There are missing target languages. - // This happens either when the source bundle is empty, - // or target bundles does not contain any successful translated - // contents. We want to set the set of target languages - // to the destination bundle. - assert targetLangs != null; - BundleDataChangeSet bundleDataChanges = new BundleDataChangeSet(); - bundleDataChanges.setTargetLanguages(targetLangs); - destClient.updateBundle(destBundleId, bundleDataChanges); - } - } - - /** - * Upload resource entries. An entry for a target language with status - * other than 'translated' will be excluded. - * - * @param client GP service client - * @param bundleId Bundle ID - * @param language Language ID - * @param resources Resource entries, originally read from another bundle - * @return true if this method actually uploaded any resource entries, or false - * if nothing uploaded. - * @throws ServiceException - */ - private static boolean uploadResources(ServiceClient client, String bundleId, - String language, Map resources) throws ServiceException { - Map newResources = - new HashMap(resources.size()); - for (Entry res : resources.entrySet()) { - ResourceEntryData resdata = res.getValue(); - TranslationStatus state = resdata.getTranslationStatus(); - if (state != TranslationStatus.SOURCE_LANGUAGE - && state != TranslationStatus.TRANSLATED) { - // Ignore an resource entry, if it's not in successful state - continue; - } - NewResourceEntryData newResData = new NewResourceEntryData(resdata.getValue()); - newResData.setNotes(resdata.getNotes()); - newResData.setSequenceNumber(resdata.getSequenceNumber()); - newResData.setMetadata(resdata.getMetadata()); - if (resdata.isReviewed()) { - newResData.setReviewed(Boolean.TRUE); - } - newResData.setPartnerStatus(resdata.getPartnerStatus()); - newResources.put(res.getKey(), newResData); - } - - if (newResources.isEmpty()) { - return false; - } - client.uploadResourceEntries(bundleId, language, newResources); - return true; - } -} \ No newline at end of file diff --git a/gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/MergeTranslationsCmd.java b/gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/MergeTranslationsCmd.java new file mode 100644 index 0000000..db9e3fa --- /dev/null +++ b/gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/MergeTranslationsCmd.java @@ -0,0 +1,343 @@ +/* + * Copyright IBM Corp. 2016 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.ibm.g11n.pipeline.tools.cli; + +import java.io.FileWriter; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import java.io.IOException; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.stream.JsonWriter; +import com.ibm.g11n.pipeline.client.BundleData; +import com.ibm.g11n.pipeline.client.ResourceEntryData; +import com.ibm.g11n.pipeline.client.ResourceEntryDataChangeSet; +import com.ibm.g11n.pipeline.client.ServiceClient; +import com.ibm.g11n.pipeline.client.ServiceException; + +/** + * Merge a bundle. + * + * @author Harpreet K Chawla + */ +@Parameters(commandDescription = "Merge translations from the worker instance to the master instance") +final class MergeTranslationsCmd extends BundleCmd { + @Parameter(names = { "-m", + "--master-json-credentials" }, description = "JSON file containing credentials of the master service instance", required = true) + private String masterJsonCreds; + + @Parameter(names = { "-bd", "--bundles" }, description = "Bundle IDs") + private String bundleListParam; + + @Parameter(names = { "-d", "--dry-run" }, description = "Dry run") + boolean isDryRun = false; + + @Parameter(names = { "-c", "--change-log-json" }, description = "Output change log file in JSON format") + private String changeLogFile; + + @Parameter(names = { "-a", + "--update-always" }, description = "Update master tranalation value always even the source value has significantly changed.") + private boolean updateAlways = false; + + @Parameter(names = { "-r", + "--includes-reviewed" }, description = "Merge changes even if correponding entries are marked as reviewed in master") + private boolean includesReviewed = false; + + private static final String CAT_UPDATED_AS_REVIEWED = "Updated/As reviewed"; + private static final String CAT_UPDATED_AS_UNREVIEWED = "Updated/As unreviewed"; + private static final String CAT_SKIPPED_SOURCE_CHANGED = "Skipped/Source changed"; + private static final String CAT_SKIPPED_NO_CHANGES = "Skipped/No changes necessary"; + private static final String CAT_SKIPPED_ALREADY_REVIEWED = "Skipped/Already reviewed"; + private static final String CAT_SKIPPED_NOT_AVAILABLE = "Skipped/Not available"; + + private static final String CAT_UPDATED_AS_REVIEWED_OVERWRITTEN = "Updated/As reviewed - Overwritten"; + + @Override + protected void _execute() { + try { + System.out.println("Master credentials: " + masterJsonCreds); + System.out.println("Workbench instance credentials: " + jsonCreds); + System.out.println("Change log file (JSON): " + changeLogFile); + + if (isDryRun) { + System.out.println("========================================"); + System.out.println(" This is a dry run!"); + System.out.println(" The master instance won't be updated."); + } + + ServiceClient masterClient = getClient(masterJsonCreds); + ServiceClient workClient = getClient(); + + Set bundleIds = new TreeSet<>(); + if (bundleListParam != null) { + String[] bundleIdParams = bundleListParam.split(","); + for (String param : bundleIdParams) { + param = param.trim(); + if (param.length() > 0) { + bundleIds.add(param); + } + } + } else { + bundleIds.addAll(workClient.getBundleIds()); + } + + Set masterBundleIds = masterClient.getBundleIds(); + + ChangeLog changeLog = new ChangeLog(); + changeLog.bundles = new TreeMap<>(); + + for (String bundleId : bundleIds) { + System.out.println("========================================"); + System.out.println("Processing bundle: " + bundleId); + + BundleData workBundleData = workClient.getBundleInfo(bundleId); + Set workTrgLangs = workBundleData.getTargetLanguages(); + if (workTrgLangs == null || workTrgLangs.isEmpty()) { + System.out.println("[Warning] No target languages found in the work bundle"); + continue; + } + + Set masterTrgLangs = Collections.emptySet(); + + if (masterBundleIds.contains(bundleId)) { + BundleData masterBundleData = masterClient.getBundleInfo(bundleId); + Set tmpLangs = masterBundleData.getTargetLanguages(); + if (tmpLangs == null || tmpLangs.isEmpty()) { + System.out.println("[Warning] No target languages found in the master bundle."); + } else { + masterTrgLangs = tmpLangs; + } + } else { + System.out.println("[Warning] The bundle does not exist in the master instance."); + } + + TreeMap bundleChanges = new TreeMap<>(); + changeLog.bundles.put(bundleId, bundleChanges); + + for (String trgLang : workTrgLangs) { + + Map updated = new TreeMap<>(); + Map skipped = new TreeMap<>(); + LangChanges langChanges = new LangChanges(); + langChanges.updated = updated; + langChanges.skipped = skipped; + bundleChanges.put(trgLang, langChanges); + + Map workResEntries = workClient.getResourceEntries(bundleId, trgLang); + Map masterResEntries = Collections.emptyMap(); + + if (masterTrgLangs.contains(trgLang)) { + masterResEntries = masterClient.getResourceEntries(bundleId, trgLang); + } + + Map langResChanges = new HashMap<>(); + int cntReviewed = 0; + int cntUnreviewed = 0; + int cntAlreadyReviewed = 0; + int cntSrcChanged = 0; + int cntNoChanges = 0; + int cntNotAvailable = 0; + int cntReviewedOverwritten = 0; + + for (Entry wkEntry : workResEntries.entrySet()) { + String resKey = wkEntry.getKey(); + ResourceEntryData wkResData = wkEntry.getValue(); + + ResChangeInfo changeInfo = new ResChangeInfo(); + + ResSummary wkRes = new ResSummary(); + changeInfo.work = wkRes; + + wkRes.reviewed = wkResData.isReviewed(); + wkRes.source = wkResData.getSourceValue(); + wkRes.translation = wkResData.getValue(); + + ResourceEntryData msResData = masterResEntries.get(resKey); + if (msResData == null) { + // No corresponding master resource entry + changeInfo.category = CAT_SKIPPED_NOT_AVAILABLE; + skipped.put(resKey, changeInfo); + cntNotAvailable++; + continue; + } + + ResSummary msRes = new ResSummary(); + changeInfo.master = msRes; + + msRes.reviewed = msResData.isReviewed(); + msRes.source = msResData.getSourceValue(); + msRes.translation = msResData.getValue(); + + boolean trsMatch = wkRes.translation.equals(msRes.translation); + + if (msRes.reviewed) { + // Master entry is already reviewed + if (!trsMatch && includesReviewed) { + // Overwrites the master value with the updated + // translation, + // forced by the option + ResSummary resolved = new ResSummary(); + changeInfo.resolved = resolved; + resolved.reviewed = true; + resolved.translation = wkRes.translation; + + changeInfo.category = CAT_UPDATED_AS_REVIEWED_OVERWRITTEN; + updated.put(resKey, changeInfo); + cntReviewedOverwritten++; + + // Add change set data to be written in the GP + // instance later + ResourceEntryDataChangeSet changeSet = new ResourceEntryDataChangeSet(); + changeSet.setValue(resolved.translation).setReviewed(Boolean.TRUE); + + langResChanges.put(resKey, changeSet); + continue; + } else { + // Normal case - master entry already marked as + // reviewed remains unchanged. + changeInfo.category = CAT_SKIPPED_ALREADY_REVIEWED; + skipped.put(resKey, changeInfo); + cntAlreadyReviewed++; + continue; + } + } + + boolean exactSrcMatch = wkRes.source.equals(msRes.source); + boolean srcMatch = exactSrcMatch ? true + : ApproximateMatcher.matches(wkRes.source, msRes.source); + + if (!srcMatch && !updateAlways) { + // Significant change in source value, so the + // translation + // in the work copy might be no longer reliable. + changeInfo.category = CAT_SKIPPED_SOURCE_CHANGED; + skipped.put(resKey, changeInfo); + cntSrcChanged++; + continue; + } + + if (trsMatch && !exactSrcMatch) { + // Translation in the master is already identical to + // the work + // copy. The source value in the master does not + // match the source + // value in the work copy exactly, therefore, we + // don't update + // master entry to reviewed:true. + changeInfo.category = CAT_SKIPPED_NO_CHANGES; + skipped.put(resKey, changeInfo); + cntNoChanges++; + continue; + } + + // The master entry will be updated as below + ResSummary resolved = new ResSummary(); + changeInfo.resolved = resolved; + resolved.reviewed = exactSrcMatch; + resolved.translation = wkRes.translation; + if (resolved.reviewed) { + changeInfo.category = CAT_UPDATED_AS_REVIEWED; + cntReviewed++; + } else { + changeInfo.category = CAT_UPDATED_AS_UNREVIEWED; + cntUnreviewed++; + } + updated.put(resKey, changeInfo); + + // Add change set data to be written in the GP instance + // later + ResourceEntryDataChangeSet changeSet = new ResourceEntryDataChangeSet(); + changeSet.setValue(resolved.translation).setReviewed(Boolean.valueOf(resolved.reviewed)); + + langResChanges.put(resKey, changeSet); + } + + System.out.println("Change summary for language: " + trgLang); + System.out.println(" Updated/As reviewed: " + cntReviewed); + System.out.println(" Updated/As reviewed - Overwritten: " + cntReviewedOverwritten); + System.out.println(" Updated/As unreviewed: " + cntUnreviewed); + System.out.println(" Skipped/No changes necessary: " + cntNoChanges); + System.out.println(" Skipped/Source value changed: " + cntSrcChanged); + System.out.println(" Skipped/Already reviewed: " + cntAlreadyReviewed); + System.out.println(" Skipped/Not available: " + cntNotAvailable); + + int numChangeEntries = langResChanges.size(); + + if (numChangeEntries > 0) { + if (isDryRun) { + System.out.println(numChangeEntries + " resource entries will be updated for language (" + + trgLang + ") if not dry run"); + } else { + System.out.println("Updating " + numChangeEntries + " resource entries for language (" + + trgLang + ")"); + masterClient.updateResourceEntries(bundleId, trgLang, langResChanges, false); + } + } else { + System.out.println("Nothing to update for language(" + trgLang + ")"); + } + } + } + + if (changeLogFile != null) { + Gson gson = new Gson(); + try { + JsonWriter jsonLogWriter = new JsonWriter(new FileWriter(changeLogFile)); + jsonLogWriter.setIndent(" "); + gson.toJson(changeLog, ChangeLog.class, jsonLogWriter); + jsonLogWriter.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } else { + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + String jsonChangeLog = gson.toJson(changeLog, ChangeLog.class); + System.out.println(jsonChangeLog); + } + } catch (ServiceException e) { + throw new RuntimeException(e); + } + } + + static class ResSummary { + String source; + String translation; + boolean reviewed; + } + + static class ResChangeInfo { + String category; + ResSummary master; + ResSummary work; + ResSummary resolved; + } + + static class LangChanges { + Map updated; + Map skipped; + } + + static class ChangeLog { + Map> bundles; + } +} \ No newline at end of file