diff --git a/gp-cli/.settings/org.eclipse.jdt.core.prefs b/gp-cli/.settings/org.eclipse.jdt.core.prefs index 25a1a85..3836513 100644 --- a/gp-cli/.settings/org.eclipse.jdt.core.prefs +++ b/gp-cli/.settings/org.eclipse.jdt.core.prefs @@ -6,6 +6,7 @@ org.eclipse.jdt.core.compiler.problem.assertIdentifier=error org.eclipse.jdt.core.compiler.problem.enumIdentifier=error org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning org.eclipse.jdt.core.compiler.source=1.7 +org.eclipse.jdt.core.formatter.align_fields_grouping_blank_lines=2147483647 org.eclipse.jdt.core.formatter.align_type_members_on_columns=false org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression=16 org.eclipse.jdt.core.formatter.alignment_for_arguments_in_annotation=0 @@ -19,8 +20,10 @@ org.eclipse.jdt.core.formatter.alignment_for_compact_if=16 org.eclipse.jdt.core.formatter.alignment_for_conditional_expression=80 org.eclipse.jdt.core.formatter.alignment_for_enum_constants=0 org.eclipse.jdt.core.formatter.alignment_for_expressions_in_array_initializer=16 +org.eclipse.jdt.core.formatter.alignment_for_expressions_in_for_loop_header=0 org.eclipse.jdt.core.formatter.alignment_for_method_declaration=0 org.eclipse.jdt.core.formatter.alignment_for_multiple_fields=16 +org.eclipse.jdt.core.formatter.alignment_for_parameterized_type_references=0 org.eclipse.jdt.core.formatter.alignment_for_parameters_in_constructor_declaration=16 org.eclipse.jdt.core.formatter.alignment_for_parameters_in_method_declaration=16 org.eclipse.jdt.core.formatter.alignment_for_resources_in_try=80 @@ -30,6 +33,8 @@ org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_enum_declaration org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_type_declaration=16 org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_constructor_declaration=16 org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_method_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_type_arguments=0 +org.eclipse.jdt.core.formatter.alignment_for_type_parameters=0 org.eclipse.jdt.core.formatter.alignment_for_union_type_in_multicatch=16 org.eclipse.jdt.core.formatter.blank_lines_after_imports=1 org.eclipse.jdt.core.formatter.blank_lines_after_package=1 @@ -56,6 +61,7 @@ org.eclipse.jdt.core.formatter.brace_position_for_switch=end_of_line org.eclipse.jdt.core.formatter.brace_position_for_type_declaration=end_of_line org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_block_comment=false org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_javadoc_comment=false +org.eclipse.jdt.core.formatter.comment.count_line_length_from_starting_position=false org.eclipse.jdt.core.formatter.comment.format_block_comments=true org.eclipse.jdt.core.formatter.comment.format_header=false org.eclipse.jdt.core.formatter.comment.format_html=true @@ -88,6 +94,7 @@ org.eclipse.jdt.core.formatter.indent_statements_compare_to_body=true org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_cases=true org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_switch=false org.eclipse.jdt.core.formatter.indentation.size=4 +org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_enum_constant=insert org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_field=insert org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_local_variable=insert org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_method=insert @@ -282,12 +289,24 @@ org.eclipse.jdt.core.formatter.never_indent_block_comments_on_first_column=false org.eclipse.jdt.core.formatter.never_indent_line_comments_on_first_column=false org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_method_body=0 org.eclipse.jdt.core.formatter.number_of_empty_lines_to_preserve=1 +org.eclipse.jdt.core.formatter.parentheses_positions_in_annotation=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_catch_clause=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_enum_constant_declaration=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_for_statment=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_if_while_statement=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_lambda_declaration=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_method_delcaration=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_method_invocation=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_switch_statement=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_try_clause=common_lines org.eclipse.jdt.core.formatter.put_empty_statement_on_new_line=true org.eclipse.jdt.core.formatter.tabulation.char=space org.eclipse.jdt.core.formatter.tabulation.size=4 org.eclipse.jdt.core.formatter.use_on_off_tags=false org.eclipse.jdt.core.formatter.use_tabs_only_for_leading_indentations=false +org.eclipse.jdt.core.formatter.wrap_before_assignment_operator=false org.eclipse.jdt.core.formatter.wrap_before_binary_operator=true +org.eclipse.jdt.core.formatter.wrap_before_conditional_operator=true org.eclipse.jdt.core.formatter.wrap_before_or_operator_multicatch=true org.eclipse.jdt.core.formatter.wrap_outer_expressions_when_nested=true org.eclipse.jdt.core.javaFormatter=org.eclipse.jdt.core.defaultJavaFormatter diff --git a/gp-cli/.settings/org.eclipse.jdt.ui.prefs b/gp-cli/.settings/org.eclipse.jdt.ui.prefs index 27e5693..b2765b2 100644 --- a/gp-cli/.settings/org.eclipse.jdt.ui.prefs +++ b/gp-cli/.settings/org.eclipse.jdt.ui.prefs @@ -1,5 +1,5 @@ eclipse.preferences.version=1 formatter_profile=_gp -formatter_settings_version=12 +formatter_settings_version=13 org.eclipse.jdt.ui.javadoc=false org.eclipse.jdt.ui.text.custom_code_templates= diff --git a/gp-cli/README.md b/gp-cli/README.md index 420a668..b63386c 100644 --- a/gp-cli/README.md +++ b/gp-cli/README.md @@ -240,6 +240,13 @@ but user accounts and translation configurations are not transferred, because they are service instance specific. +#### merge (merge-bundle) + +Merge translations from the worker instance to the master instance + +``` +java -jar gp-cli.jar merge-translations -m master-credentials.json -j slave-credentials.json -b test +``` --- ### User Commands diff --git a/gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/ApproximateMatcher.java b/gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/ApproximateMatcher.java new file mode 100644 index 0000000..a38e382 --- /dev/null +++ b/gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/ApproximateMatcher.java @@ -0,0 +1,49 @@ +/* + * Copyright IBM Corp. 2015,2016 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.ibm.g11n.pipeline.tools.cli; + +import com.ibm.icu.lang.UCharacter; + +public final class ApproximateMatcher { + private String text; + private String processedText; + + public ApproximateMatcher(String text) { + this.text = text; + this.processedText = processText(text); + } + + private static String processText(String text) { + // fold case + text = UCharacter.foldCase(text, true); + + // replace characters other than letter and number + // with space and trim. + return text.replaceAll("[^\\p{L}\\p{N}]+", " ").trim(); + } + + public boolean matches(String inText) { + if (text.equals(inText)) { + return true; + } + return processedText.equals(processText(inText)); + } + + public static boolean matches(String text1, String text2) { + ApproximateMatcher matcher = new ApproximateMatcher(text1); + return matcher.matches(text2); + } +} \ No newline at end of file diff --git a/gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/BaseCmd.java b/gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/BaseCmd.java index c4534fb..d437ace 100644 --- a/gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/BaseCmd.java +++ b/gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/BaseCmd.java @@ -65,6 +65,21 @@ static class JsonCredentials { String password; } + protected static ServiceClient getClient(String jsonCredsFile) { + JsonCredentials creds; + try (InputStreamReader reader = new InputStreamReader( + new FileInputStream(jsonCredsFile), StandardCharsets.UTF_8)) { + Gson gson = new Gson(); + creds = gson.fromJson(reader, JsonCredentials.class); + } catch (IOException e) { + throw new RuntimeException(e); + } + + ServiceAccount account = ServiceAccount.getInstance( + creds.url, creds.instanceId, creds.userId, creds.password); + return ServiceClient.getInstance(account); + } + protected ServiceClient getClient() { if (jsonCreds != null) { JsonCredentials creds; diff --git a/gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/GPCmd.java b/gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/GPCmd.java index 73b42bf..b0e4860 100644 --- a/gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/GPCmd.java +++ b/gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/GPCmd.java @@ -53,6 +53,7 @@ public static void main(String[] args) { jc.addCommand("export", new ExportCmd()); jc.addCommand("import", new ImportCmd()); jc.addCommand("list-mt-languages", new ListMTLanguagesCmd()); + jc.addCommand("merge-translations", new MergeTranslationsCmd(), "merge-translations"); //users jc.addCommand("list-users", new ListUsersCmd()); diff --git a/gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/MergeTranslationsCmd.java b/gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/MergeTranslationsCmd.java new file mode 100644 index 0000000..db9e3fa --- /dev/null +++ b/gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/MergeTranslationsCmd.java @@ -0,0 +1,343 @@ +/* + * Copyright IBM Corp. 2016 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.ibm.g11n.pipeline.tools.cli; + +import java.io.FileWriter; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import java.io.IOException; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.stream.JsonWriter; +import com.ibm.g11n.pipeline.client.BundleData; +import com.ibm.g11n.pipeline.client.ResourceEntryData; +import com.ibm.g11n.pipeline.client.ResourceEntryDataChangeSet; +import com.ibm.g11n.pipeline.client.ServiceClient; +import com.ibm.g11n.pipeline.client.ServiceException; + +/** + * Merge a bundle. + * + * @author Harpreet K Chawla + */ +@Parameters(commandDescription = "Merge translations from the worker instance to the master instance") +final class MergeTranslationsCmd extends BundleCmd { + @Parameter(names = { "-m", + "--master-json-credentials" }, description = "JSON file containing credentials of the master service instance", required = true) + private String masterJsonCreds; + + @Parameter(names = { "-bd", "--bundles" }, description = "Bundle IDs") + private String bundleListParam; + + @Parameter(names = { "-d", "--dry-run" }, description = "Dry run") + boolean isDryRun = false; + + @Parameter(names = { "-c", "--change-log-json" }, description = "Output change log file in JSON format") + private String changeLogFile; + + @Parameter(names = { "-a", + "--update-always" }, description = "Update master tranalation value always even the source value has significantly changed.") + private boolean updateAlways = false; + + @Parameter(names = { "-r", + "--includes-reviewed" }, description = "Merge changes even if correponding entries are marked as reviewed in master") + private boolean includesReviewed = false; + + private static final String CAT_UPDATED_AS_REVIEWED = "Updated/As reviewed"; + private static final String CAT_UPDATED_AS_UNREVIEWED = "Updated/As unreviewed"; + private static final String CAT_SKIPPED_SOURCE_CHANGED = "Skipped/Source changed"; + private static final String CAT_SKIPPED_NO_CHANGES = "Skipped/No changes necessary"; + private static final String CAT_SKIPPED_ALREADY_REVIEWED = "Skipped/Already reviewed"; + private static final String CAT_SKIPPED_NOT_AVAILABLE = "Skipped/Not available"; + + private static final String CAT_UPDATED_AS_REVIEWED_OVERWRITTEN = "Updated/As reviewed - Overwritten"; + + @Override + protected void _execute() { + try { + System.out.println("Master credentials: " + masterJsonCreds); + System.out.println("Workbench instance credentials: " + jsonCreds); + System.out.println("Change log file (JSON): " + changeLogFile); + + if (isDryRun) { + System.out.println("========================================"); + System.out.println(" This is a dry run!"); + System.out.println(" The master instance won't be updated."); + } + + ServiceClient masterClient = getClient(masterJsonCreds); + ServiceClient workClient = getClient(); + + Set bundleIds = new TreeSet<>(); + if (bundleListParam != null) { + String[] bundleIdParams = bundleListParam.split(","); + for (String param : bundleIdParams) { + param = param.trim(); + if (param.length() > 0) { + bundleIds.add(param); + } + } + } else { + bundleIds.addAll(workClient.getBundleIds()); + } + + Set masterBundleIds = masterClient.getBundleIds(); + + ChangeLog changeLog = new ChangeLog(); + changeLog.bundles = new TreeMap<>(); + + for (String bundleId : bundleIds) { + System.out.println("========================================"); + System.out.println("Processing bundle: " + bundleId); + + BundleData workBundleData = workClient.getBundleInfo(bundleId); + Set workTrgLangs = workBundleData.getTargetLanguages(); + if (workTrgLangs == null || workTrgLangs.isEmpty()) { + System.out.println("[Warning] No target languages found in the work bundle"); + continue; + } + + Set masterTrgLangs = Collections.emptySet(); + + if (masterBundleIds.contains(bundleId)) { + BundleData masterBundleData = masterClient.getBundleInfo(bundleId); + Set tmpLangs = masterBundleData.getTargetLanguages(); + if (tmpLangs == null || tmpLangs.isEmpty()) { + System.out.println("[Warning] No target languages found in the master bundle."); + } else { + masterTrgLangs = tmpLangs; + } + } else { + System.out.println("[Warning] The bundle does not exist in the master instance."); + } + + TreeMap bundleChanges = new TreeMap<>(); + changeLog.bundles.put(bundleId, bundleChanges); + + for (String trgLang : workTrgLangs) { + + Map updated = new TreeMap<>(); + Map skipped = new TreeMap<>(); + LangChanges langChanges = new LangChanges(); + langChanges.updated = updated; + langChanges.skipped = skipped; + bundleChanges.put(trgLang, langChanges); + + Map workResEntries = workClient.getResourceEntries(bundleId, trgLang); + Map masterResEntries = Collections.emptyMap(); + + if (masterTrgLangs.contains(trgLang)) { + masterResEntries = masterClient.getResourceEntries(bundleId, trgLang); + } + + Map langResChanges = new HashMap<>(); + int cntReviewed = 0; + int cntUnreviewed = 0; + int cntAlreadyReviewed = 0; + int cntSrcChanged = 0; + int cntNoChanges = 0; + int cntNotAvailable = 0; + int cntReviewedOverwritten = 0; + + for (Entry wkEntry : workResEntries.entrySet()) { + String resKey = wkEntry.getKey(); + ResourceEntryData wkResData = wkEntry.getValue(); + + ResChangeInfo changeInfo = new ResChangeInfo(); + + ResSummary wkRes = new ResSummary(); + changeInfo.work = wkRes; + + wkRes.reviewed = wkResData.isReviewed(); + wkRes.source = wkResData.getSourceValue(); + wkRes.translation = wkResData.getValue(); + + ResourceEntryData msResData = masterResEntries.get(resKey); + if (msResData == null) { + // No corresponding master resource entry + changeInfo.category = CAT_SKIPPED_NOT_AVAILABLE; + skipped.put(resKey, changeInfo); + cntNotAvailable++; + continue; + } + + ResSummary msRes = new ResSummary(); + changeInfo.master = msRes; + + msRes.reviewed = msResData.isReviewed(); + msRes.source = msResData.getSourceValue(); + msRes.translation = msResData.getValue(); + + boolean trsMatch = wkRes.translation.equals(msRes.translation); + + if (msRes.reviewed) { + // Master entry is already reviewed + if (!trsMatch && includesReviewed) { + // Overwrites the master value with the updated + // translation, + // forced by the option + ResSummary resolved = new ResSummary(); + changeInfo.resolved = resolved; + resolved.reviewed = true; + resolved.translation = wkRes.translation; + + changeInfo.category = CAT_UPDATED_AS_REVIEWED_OVERWRITTEN; + updated.put(resKey, changeInfo); + cntReviewedOverwritten++; + + // Add change set data to be written in the GP + // instance later + ResourceEntryDataChangeSet changeSet = new ResourceEntryDataChangeSet(); + changeSet.setValue(resolved.translation).setReviewed(Boolean.TRUE); + + langResChanges.put(resKey, changeSet); + continue; + } else { + // Normal case - master entry already marked as + // reviewed remains unchanged. + changeInfo.category = CAT_SKIPPED_ALREADY_REVIEWED; + skipped.put(resKey, changeInfo); + cntAlreadyReviewed++; + continue; + } + } + + boolean exactSrcMatch = wkRes.source.equals(msRes.source); + boolean srcMatch = exactSrcMatch ? true + : ApproximateMatcher.matches(wkRes.source, msRes.source); + + if (!srcMatch && !updateAlways) { + // Significant change in source value, so the + // translation + // in the work copy might be no longer reliable. + changeInfo.category = CAT_SKIPPED_SOURCE_CHANGED; + skipped.put(resKey, changeInfo); + cntSrcChanged++; + continue; + } + + if (trsMatch && !exactSrcMatch) { + // Translation in the master is already identical to + // the work + // copy. The source value in the master does not + // match the source + // value in the work copy exactly, therefore, we + // don't update + // master entry to reviewed:true. + changeInfo.category = CAT_SKIPPED_NO_CHANGES; + skipped.put(resKey, changeInfo); + cntNoChanges++; + continue; + } + + // The master entry will be updated as below + ResSummary resolved = new ResSummary(); + changeInfo.resolved = resolved; + resolved.reviewed = exactSrcMatch; + resolved.translation = wkRes.translation; + if (resolved.reviewed) { + changeInfo.category = CAT_UPDATED_AS_REVIEWED; + cntReviewed++; + } else { + changeInfo.category = CAT_UPDATED_AS_UNREVIEWED; + cntUnreviewed++; + } + updated.put(resKey, changeInfo); + + // Add change set data to be written in the GP instance + // later + ResourceEntryDataChangeSet changeSet = new ResourceEntryDataChangeSet(); + changeSet.setValue(resolved.translation).setReviewed(Boolean.valueOf(resolved.reviewed)); + + langResChanges.put(resKey, changeSet); + } + + System.out.println("Change summary for language: " + trgLang); + System.out.println(" Updated/As reviewed: " + cntReviewed); + System.out.println(" Updated/As reviewed - Overwritten: " + cntReviewedOverwritten); + System.out.println(" Updated/As unreviewed: " + cntUnreviewed); + System.out.println(" Skipped/No changes necessary: " + cntNoChanges); + System.out.println(" Skipped/Source value changed: " + cntSrcChanged); + System.out.println(" Skipped/Already reviewed: " + cntAlreadyReviewed); + System.out.println(" Skipped/Not available: " + cntNotAvailable); + + int numChangeEntries = langResChanges.size(); + + if (numChangeEntries > 0) { + if (isDryRun) { + System.out.println(numChangeEntries + " resource entries will be updated for language (" + + trgLang + ") if not dry run"); + } else { + System.out.println("Updating " + numChangeEntries + " resource entries for language (" + + trgLang + ")"); + masterClient.updateResourceEntries(bundleId, trgLang, langResChanges, false); + } + } else { + System.out.println("Nothing to update for language(" + trgLang + ")"); + } + } + } + + if (changeLogFile != null) { + Gson gson = new Gson(); + try { + JsonWriter jsonLogWriter = new JsonWriter(new FileWriter(changeLogFile)); + jsonLogWriter.setIndent(" "); + gson.toJson(changeLog, ChangeLog.class, jsonLogWriter); + jsonLogWriter.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } else { + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + String jsonChangeLog = gson.toJson(changeLog, ChangeLog.class); + System.out.println(jsonChangeLog); + } + } catch (ServiceException e) { + throw new RuntimeException(e); + } + } + + static class ResSummary { + String source; + String translation; + boolean reviewed; + } + + static class ResChangeInfo { + String category; + ResSummary master; + ResSummary work; + ResSummary resolved; + } + + static class LangChanges { + Map updated; + Map skipped; + } + + static class ChangeLog { + Map> bundles; + } +} \ No newline at end of file