make sure all the Translator transducers are doing multiple lines pro…

…perly (including adding unit tests) and bump the nuget version
silnrsi · Sep 19, 2024 · 4851764 · 4851764
1 parent 7169ef1
commit 4851764
Show file tree

Hide file tree

Showing 6 changed files with 41 additions and 10 deletions.
diff --git a/Package.Release.nuspec b/Package.Release.nuspec
@@ -2,7 +2,7 @@
 <package >
 	<metadata>
 		<id>Encoding-Converters-Core</id>
-		<version>0.9.0</version>
+		<version>0.9.1</version>
 		<authors>Bob Eaton, Jim Kornelson, SIL International</authors>
 		<owners>jnaylor, sillsdev</owners>
 		<license type="expression">MIT</license>

diff --git a/src/EcTranslators/DeepLTranslator/DeepLTranslatorEncConverter.cs b/src/EcTranslators/DeepLTranslator/DeepLTranslatorEncConverter.cs
@@ -281,10 +281,12 @@ private async Task<string> CallDeepLTranslator(string strInput)
 				var options = (FormalityLevel != Formality.Default) ? new TextTranslateOptions { Formality = FormalityLevel } : null;
 				var translatedText = await Task.Run(async delegate
 				{
-					return await DeepLTranslator.TranslateTextAsync(strInput, FromLanguage, ToLanguage, options);
+					var lines = strInput.Split(crlf, StringSplitOptions.RemoveEmptyEntries);
+					var result = await DeepLTranslator.TranslateTextAsync(lines, FromLanguage, ToLanguage, options);
+					return string.Join(Environment.NewLine, result.Select(r => r.Text));
 				}).ConfigureAwait(false);
 
-				return translatedText.Text;
+				return translatedText;
 			}
 			catch (Exception ex)
 			{

diff --git a/src/EcTranslators/GoogleTranslator/GoogleTranslatorEncConverter.cs b/src/EcTranslators/GoogleTranslator/GoogleTranslatorEncConverter.cs
@@ -229,10 +229,13 @@ private async Task<string> CallGoogleTranslator(string strInput)
             {
                 var result = await Task.Run(async delegate
                 {
-                    return await TranslateClient.TranslateTextAsync(strInput, toLanguage, fromLanguage);
-                }).ConfigureAwait(false);
+					var lines = strInput.Split(crlf, StringSplitOptions.RemoveEmptyEntries);
+                    var result = await TranslateClient.TranslateTextAsync(lines, toLanguage, fromLanguage);
+					return String.Join(Environment.NewLine, result.Select(r => r.TranslatedText));
+
+				}).ConfigureAwait(false);
 
-                return result.TranslatedText;
+                return result;
             }
             catch (Exception ex)
             {

diff --git a/src/EcTranslators/TranslatorConverter.cs b/src/EcTranslators/TranslatorConverter.cs
@@ -10,6 +10,8 @@ public abstract class TranslatorConverter : EncConverter
 	{
 		public const int WarnEveryXRequests = 100;
 
+		public readonly char[] crlf = new[] { '\r', '\n' };
+
 		public int RequestCount { get; set; }
 
 		public abstract bool HasUserOverriddenCredentials { get; }

diff --git a/src/EncCnvtrs/ExeEncConverter.cs b/src/EncCnvtrs/ExeEncConverter.cs
@@ -208,6 +208,10 @@ protected string DoExeCall(string sInput)
                 // call a virtual to do this in case the sub-classes have special behavior
                 strOutput = ReadFromExeOutputStream(sr, p.StandardError);
                 strOutput = CleanOutput(sInput, strOutput);
+
+				// some processes do a linux style having converted the \r\n into just a \n, so split on that and rebuild w/ Environment.NewLine
+				var lines = strOutput.Split(charsToTrim, StringSplitOptions.RemoveEmptyEntries);
+				strOutput = string.Join(Environment.NewLine, lines);
             }
             catch (Exception e)
             {

diff --git a/src/TestEncCnvtrs/TestEcTranslators.cs b/src/TestEncCnvtrs/TestEcTranslators.cs
@@ -256,6 +256,10 @@ public void TestNllbTranslatorOverrideKeyExists()
         [TestCase("Hindi;English;bright-coyote-381812;us-central1;google;chat-bison;Translate from Hindi into English.", "यीशु ने यह भी कहा,", "Jesus also said,")]
         [TestCase("Hindi;English;bright-coyote-381812;us-central1;google;chat-bison;Translate from Hindi into English.", "परंतु वह चोगे को छोड़कर वहाँ से भाग गया। ", "But he fled from there, leaving his cloak behind.")]
 		[TestCase("Hindi;English;bright-coyote-381812;us-central1;google;gemini-1.5-flash;Translate from Hindi into English.", "यीशु ने कहा,", "Jesus said, ")]
+		// multiple lines
+		[TestCase("Hindi;English;bright-coyote-381812;us-central1;google;gemini-1.5-flash;Translate from Hindi into English.", @"यीशु ने कहा,
+परमे‍‍श्वर मेरा पिता है।", @"Jesus said, 
+God is my Father. ")]
 		public void TestVertexAiConverter(string converterSpec, string testInput, string testOutput)
         {
             Environment.SetEnvironmentVariable("GOOGLE_APPLICATION_CREDENTIALS", VertexAiCredentials);    // see C:\Users\pete_\source\repos\encoding-converters-core\src\EcTranslators\VertexAi\VertexAiExe\Program.cs
@@ -423,7 +427,11 @@ public void TestAzureOpenAiConverter(string converterSpec, string testInput, str
         [Test]
         [TestCase(ProcessTypeFlags.Translation, "Translate;hi;en", "", "")]
         [TestCase(ProcessTypeFlags.Translation, "Translate;hi;en", "यीशु ने यह भी कहा,", "Jesus also said,")]
-        [TestCase(ProcessTypeFlags.Translation, "Translate;;en", "यीशु ने यह भी कहा,", "Jesus also said,")]
+		// multi-line translations
+		[TestCase(ProcessTypeFlags.Translation, "Translate;hi;en", @"यीशु ने कहा,
+परमे‍‍श्वर मेरा पिता है।", @"Jesus said,
+God is my Father.")]
+		[TestCase(ProcessTypeFlags.Translation, "Translate;;en", "यीशु ने यह भी कहा,", "Jesus also said,")]
         [TestCase(ProcessTypeFlags.Translation, "Translate;en;zh-Hans", "Get to know the beautiful country of Israel.", "了解美丽的以色列国家。")]
         [TestCase(ProcessTypeFlags.Translation | ProcessTypeFlags.Transliteration, "TranslateWithTransliterate;en;ar;;Latn", "God", "allah")]
         [TestCase(ProcessTypeFlags.Translation | ProcessTypeFlags.Transliteration, "TranslateWithTransliterate;;ar;;Latn", "God", "allah")]
@@ -486,7 +494,11 @@ public void TestExtractingLangCode(string menuItem, string languageCodeExpected)
         [TestCase(ProcessTypeFlags.Translation, "Translate;en;zh", "How are you?", "你好吗？")]
         [TestCase(ProcessTypeFlags.Translation, "Translate;en;de;Less", "How are you?", "Wie geht es dir?")]
         [TestCase(ProcessTypeFlags.Translation, "Translate;en;de;More", "How are you?", "Wie geht es Ihnen?")]
-        public void TestDeepLConverter(ProcessTypeFlags processType, string converterSpec, string testInput, string testOutput)
+		// multi-line translations
+		[TestCase(ProcessTypeFlags.Translation, "Translate;en;fr", @"Jesus said,
+God is my father.", @"Jésus a dit,
+Dieu est mon père.")]
+		public void TestDeepLConverter(ProcessTypeFlags processType, string converterSpec, string testInput, string testOutput)
         {
             m_encConverters.AddConversionMap(DeepLTranslatorConverterFriendlyName, converterSpec, ConvType.Unicode_to_Unicode,
                                              EncConverters.strTypeSILDeepLTranslator, "UNICODE", "UNICODE",
@@ -519,7 +531,11 @@ public async Task TestDeeplTranslatorGetCapabilities()
         [TestCase("en;fr", "Hello, world!", "Bonjour le monde!")]
         [TestCase("en;zh", "How are you?", "你好吗？")]
         [TestCase("en;de", "How are you?", "Wie geht es dir?")]
-        public void TestGoogleConverterWithEnvVariable(string converterSpec, string testInput, string testOutput)
+		// multi-line translations
+		[TestCase("hi;en", @"यीशु ने कहा,
+परमे‍‍श्वर मेरा पिता है।", @"Jesus said,
+God is my father.")]
+		public void TestGoogleConverterWithEnvVariable(string converterSpec, string testInput, string testOutput)
         {
             Environment.SetEnvironmentVariable("GOOGLE_APPLICATION_CREDENTIALS", VertexAiCredentials);    // see C:\Users\pete_\source\repos\encoding-converters-core\src\EcTranslators\VertexAi\VertexAiExe\Program.cs
 
@@ -541,7 +557,11 @@ public void TestGoogleConverterWithEnvVariable(string converterSpec, string test
         [TestCase("en;fr", "Hello, world!", "Bonjour le monde!")]
         [TestCase("en;zh", "How are you?", "你好吗？")]
         [TestCase("en;de", "How are you?", "Wie geht es dir?")]
-        public void TestGoogleConverter(string converterSpec, string testInput, string testOutput)
+		// multi-line translations
+		[TestCase("hi;en", @"यीशु ने कहा,
+परमे‍‍श्वर मेरा पिता है।", @"Jesus said,
+God is my father.")]
+		public void TestGoogleConverter(string converterSpec, string testInput, string testOutput)
         {
             m_encConverters.AddConversionMap(GoogleTranslatorConverterFriendlyName, converterSpec, ConvType.Unicode_to_Unicode,
                                              EncConverters.strTypeSILGoogleTranslator, "UNICODE", "UNICODE",