Revamp on internationalization support.

* More consistent usage of the Dec.Config culture. * Change the default culture to InvariantCulture. * Support case-insensitive nan/infinity string parsing.
zorbathut · Jan 9, 2024 · 1e05d62 · 1e05d62
1 parent a31ec4c
commit 1e05d62
Show file tree

Hide file tree

Showing 10 changed files with 201 additions and 105 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,9 +6,14 @@ All notable changes to this project will be documented in this file.
 ### Added
 * Proper support for multidimensional arrays.
 
+### Possibly Breaking
+* Changed default culture from `en-US` to `InvariantCulture`. I don't think this should have any effects given the manual string parsing for Infinity, but I may be missing something; report problems, please.
+
 ### Improved
 * Error messages for database queries interacting with AbstractAttribute.
 * Error message for inappropriately-timed StaticReferences initialization.
+* More consistent usage of the Dec.Config culture.
+* Support case-insensitive nan/infinity float/double parsing.
 
 ### Testing
 * Added proper testing for AbstractAttribute.

diff --git a/src/Config.cs b/src/Config.cs
@@ -39,7 +39,7 @@ public static class Config
         /// </summary>
         /// <remarks>
         /// This should be made unmissably visible to developers and testers, ideally with a popup or a modal dialog.
-        /// 
+        ///
         /// Can be made to throw an exception. If it does, the exception will propagate to the caller. Otherwise, dec will attempt to recover from the error.
         ///
         /// If you're using any multithreading, this must be threadsafe.
@@ -51,7 +51,7 @@ public static class Config
         /// </summary>
         /// <remarks>
         /// This should be made unmissably visible to developers and testers, ideally with a popup or a modal dialog.
-        /// 
+        ///
         /// Can be made to rethrow the exception or throw a new exception. If it does, the exception will propagate to the caller. Otherwise, dec will attempt to recover from the error.
         ///
         /// If you're using any multithreading, this must be threadsafe.
@@ -91,11 +91,13 @@ public enum DefaultExceptionBehavior
         /// The culture to use for parsing and writing values.
         /// </summary>
         /// <remarks>
-        /// This must be set statically, rather than loaded from the user's system, or parsing might break unpredictably. Recommend leaving this set to `en-US` for compatibility with the general Dec ecosystem; other options may have bugs (but report them and I'll fix them.)
+        /// This must be set statically, rather than loaded from the user's system, or parsing might break unpredictably. Recommend leaving this set to InvariantCulture for compatibility with the general Dec ecosystem; other options may have bugs (but report them and I'll fix them!)
+        ///
+        /// Changing this while Dec is running is undefined behavior. Don't do that. Dec may be unable to read files written under a different CultureInfo; if you don't want that to be a problem, well, choose today, and choose wisely.
         ///
-        /// Changing this while Dec is running is undefined behavior. Don't do that. Dec may be unable to read files written under a different CultureInfo.    
+        /// (just leave it set to its default for christ's sake)
         /// </remarks>
-        public static System.Globalization.CultureInfo CultureInfo = new System.Globalization.CultureInfo("en-US"); 
+        public static System.Globalization.CultureInfo CultureInfo = System.Globalization.CultureInfo.InvariantCulture;
 
         /// <summary>
         /// The list of namespaces that dec can access transparently.

diff --git a/src/ReaderXml.cs b/src/ReaderXml.cs
@@ -460,7 +460,7 @@ public override void ParseDictionary(IDictionary dict, Type referencedKeyType, T
                         Dbg.Err($"{elementContext}: Dictionary includes null key, skipping pair");
 
                         // just in case . . .
-                        if (string.Compare(fieldElement.Name.LocalName, "li", true, System.Globalization.CultureInfo.InvariantCulture) == 0)
+                        if (string.Compare(fieldElement.Name.LocalName, "li", true) == 0)
                         {
                             Dbg.Err($"{elementContext}: Did you mean to write `li`? This field is case-sensitive.");
                         }

diff --git a/src/Serialization.cs b/src/Serialization.cs
@@ -1502,38 +1502,73 @@ internal static object ParseString(string text, Type type, object original, Inpu
                 // If we've got text, treat us as an object of appropriate type
                 try
                 {
-                    if ((type == typeof(float) || type == typeof(double)) && text != "NaN" && text.StartsWith("NaNbox"))
+                    if (type == typeof(float))
                     {
-                        // oops, time for nan-boxed values
+                        // first check the various strings, case-insensitive
+                        if (String.Compare(text, "nan", true) == 0)
+                        {
+                            return float.NaN;
+                        }
 
-                        const int expectedFloatSize = 6 + 8;
-                        const int expectedDoubleSize = 6 + 16;
+                        if (String.Compare(text, "infinity", true) == 0)
+                        {
+                            return float.PositiveInfinity;
+                        }
 
-                        if (type == typeof(float) && text.Length != expectedFloatSize)
+                        if (String.Compare(text, "-infinity", true) == 0)
                         {
-                            Dbg.Err($"{context}: Found nanboxed value without the expected number of characters, expected {expectedFloatSize} but got {text.Length}");
-                            return float.NaN;
+                            return float.NegativeInfinity;
+                        }
+
+                        if (text.StartsWith("nanbox", StringComparison.CurrentCultureIgnoreCase))
+                        {
+                            const int expectedFloatSize = 6 + 8;
+
+                            if (type == typeof(float) && text.Length != expectedFloatSize)
+                            {
+                                Dbg.Err($"{context}: Found nanboxed value without the expected number of characters, expected {expectedFloatSize} but got {text.Length}");
+                                return float.NaN;
+                            }
+
+                            int number = Convert.ToInt32(text.Substring(6), 16);
+                            return BitConverter.Int32BitsToSingle(number);
                         }
-                        else if (type == typeof(double) && text.Length != expectedDoubleSize)
+                    }
+
+                    if (type == typeof(double))
+                    {
+                        // first check the various strings, case-insensitive
+                        if (String.Compare(text, "nan", true) == 0)
                         {
-                            Dbg.Err($"{context}: Found nanboxed value without the expected number of characters, expected {expectedDoubleSize} but got {text.Length}");
                             return double.NaN;
                         }
 
-                        if (type == typeof(float))
+                        if (String.Compare(text, "infinity", true) == 0)
                         {
-                            long number = Convert.ToInt64(text.Substring(6), 16);
-                            return BitConverter.Int32BitsToSingle((int)number);
+                            return double.PositiveInfinity;
                         }
-                        else
+
+                        if (String.Compare(text, "-infinity", true) == 0)
                         {
-                            // gotta be double
+                            return double.NegativeInfinity;
+                        }
+
+                        if (text.StartsWith("nanbox", StringComparison.CurrentCultureIgnoreCase))
+                        {
+                            const int expectedDoubleSize = 6 + 16;
+
+                            if (type == typeof(double) && text.Length != expectedDoubleSize)
+                            {
+                                Dbg.Err($"{context}: Found nanboxed value without the expected number of characters, expected {expectedDoubleSize} but got {text.Length}");
+                                return double.NaN;
+                            }
+
                             long number = Convert.ToInt64(text.Substring(6), 16);
                             return BitConverter.Int64BitsToDouble(number);
                         }
                     }
 
-                    return TypeDescriptor.GetConverter(type).ConvertFromInvariantString(text);
+                    return TypeDescriptor.GetConverter(type).ConvertFromString(text);
                 }
                 catch (System.Exception e)  // I would normally not catch System.Exception, but TypeConverter is wrapping FormatException in an Exception for some reason
                 {

diff --git a/src/UtilXml.cs b/src/UtilXml.cs
@@ -34,7 +34,7 @@ internal static XElement ElementNamedWithFallback(this XElement root, string nam
             }
 
             // check to see if we have a case-insensitive match
-            result = root.Elements().Where(child => string.Compare(child.Name.LocalName, name, true, System.Globalization.CultureInfo.InvariantCulture) == 0).FirstOrDefault();
+            result = root.Elements().Where(child => string.Compare(child.Name.LocalName, name, true) == 0).FirstOrDefault();
             if (result != null)
             {
                 Dbg.Err($"{context}: {errorPrefix}; falling back on `{result.Name.LocalName}`, which is not the right case!");

diff --git a/src/WriterXml.cs b/src/WriterXml.cs
@@ -342,15 +342,15 @@ public override void WritePrimitive(object value)
                 if (double.IsNaN(val) && BitConverter.DoubleToInt64Bits(val) != BitConverter.DoubleToInt64Bits(double.NaN))
                 {
                     // oops, all nan boxing!
-                    node.Add(new XText("NaNbox" + BitConverter.DoubleToInt64Bits(val).ToString("X16", System.Globalization.CultureInfo.InvariantCulture)));
+                    node.Add(new XText("NaNbox" + BitConverter.DoubleToInt64Bits(val).ToString("X16")));
                 }
                 else if (Compat.FloatRoundtripBroken)
                 {
-                    node.Add(new XText(val.ToString("G17", System.Globalization.CultureInfo.InvariantCulture)));
+                    node.Add(new XText(val.ToString("G17")));
                 }
                 else
                 {
-                    node.Add(new XText(val.ToString(System.Globalization.CultureInfo.InvariantCulture)));
+                    node.Add(new XText(val.ToString()));
                 }
             }
             else if (value.GetType() == typeof(float))
@@ -359,15 +359,15 @@ public override void WritePrimitive(object value)
                 if (float.IsNaN(val) && BitConverter.SingleToInt32Bits(val) != BitConverter.SingleToInt32Bits(float.NaN))
                 {
                     // oops, all nan boxing!
-                    node.Add(new XText("NaNbox" + BitConverter.SingleToInt32Bits(val).ToString("X8", System.Globalization.CultureInfo.InvariantCulture)));
+                    node.Add(new XText("NaNbox" + BitConverter.SingleToInt32Bits(val).ToString("X8")));
                 }
                 else if (Compat.FloatRoundtripBroken)
                 {
-                    node.Add(new XText(val.ToString("G9", System.Globalization.CultureInfo.InvariantCulture)));
+                    node.Add(new XText(val.ToString("G9")));
                 }
                 else
                 {
-                    node.Add(new XText(val.ToString(System.Globalization.CultureInfo.InvariantCulture)));
+                    node.Add(new XText(val.ToString()));
                 }
             }
             else

diff --git a/test/unit/Base.cs b/test/unit/Base.cs
@@ -15,6 +15,9 @@ public void Clean()
             // Turns out Hebrew is basically the worst-case scenario for parsing of this sort.
             System.Threading.Thread.CurrentThread.CurrentCulture = new System.Globalization.CultureInfo("he-IL");
 
+            // But reset this just in case.
+            Dec.Config.CultureInfo = new System.Globalization.CultureInfo("en-US");
+
             // stop verifying things
             errorValidator = null;
 
@@ -40,7 +43,7 @@ public void Clean()
         private bool handlingErrors = false;
         private bool handledError = false;
         private Func<string, bool> errorValidator = null;
-        
+
         [OneTimeSetUp]
         public void PrepHooks()
         {

diff --git a/test/unit/Culture.cs b/test/unit/Culture.cs
@@ -0,0 +1,91 @@
+namespace DecTest
+{
+    using NUnit.Framework;
+    using System;
+    using System.Collections.Generic;
+    using System.Linq;
+    using System.Xml.Linq;
+
+    [TestFixture]
+    public class Culture : Base
+    {
+
+        [Dec.StaticReferences]
+        public static class StaticRefsDecs
+        {
+            static StaticRefsDecs() { Dec.StaticReferencesAttribute.Initialized(); }
+
+            public static StubDec Brød;
+            public static StubDec ขนมปัง;
+            public static StubDec パン;
+            public static StubDec 餧;
+            public static StubDec 麵包;
+            public static StubDec خبز;
+            public static StubDec 빵;
+            public static StubDec לחם;
+            public static StubDec ပေါင်မုန့်;
+            public static StubDec ബ്രെഡ്;
+        }
+
+        [Test]
+        public void StaticRefs([Values] ParserMode mode)
+        {
+            UpdateTestParameters(new Dec.Config.UnitTestParameters { explicitTypes = new Type[] { typeof(StubDec) }, explicitStaticRefs = new Type[] { typeof(StaticRefsDecs) } });
+
+            var parser = new Dec.Parser();
+            parser.AddString(Dec.Parser.FileType.Xml, @"
+                <Decs>
+                    <StubDec decName=""Brød"" />
+                    <StubDec decName=""ขนมปัง"" />
+                    <StubDec decName=""パン"" />
+                    <StubDec decName=""餧"" />
+                    <StubDec decName=""麵包"" />
+                    <StubDec decName=""خبز"" />
+                    <StubDec decName=""빵"" />
+                    <StubDec decName=""לחם"" />
+                    <StubDec decName=""ပေါင်မုန့်"" />
+                    <StubDec decName=""ബ്രെഡ്"" />
+                </Decs>");
+            parser.Finish();
+
+            DoParserTests(mode);
+
+            Assert.IsNotNull(Dec.Database<StubDec>.Get("Brød"));
+            Assert.IsNotNull(Dec.Database<StubDec>.Get("ขนมปัง"));
+            Assert.IsNotNull(Dec.Database<StubDec>.Get("パン"));
+            Assert.IsNotNull(Dec.Database<StubDec>.Get("餧"));
+            Assert.IsNotNull(Dec.Database<StubDec>.Get("麵包"));
+            Assert.IsNotNull(Dec.Database<StubDec>.Get("خبز"));
+            Assert.IsNotNull(Dec.Database<StubDec>.Get("빵"));
+            Assert.IsNotNull(Dec.Database<StubDec>.Get("לחם"));
+            Assert.IsNotNull(Dec.Database<StubDec>.Get("ပေါင်မုန့်"));
+            Assert.IsNotNull(Dec.Database<StubDec>.Get("ബ്രെഡ്"));
+
+            Assert.AreSame(Dec.Database<StubDec>.Get("Brød"), StaticRefsDecs.Brød);
+            Assert.AreSame(Dec.Database<StubDec>.Get("ขนมปัง"), StaticRefsDecs.ขนมปัง);
+            Assert.AreSame(Dec.Database<StubDec>.Get("パン"), StaticRefsDecs.パン);
+            Assert.AreSame(Dec.Database<StubDec>.Get("餧"), StaticRefsDecs.餧);
+            Assert.AreSame(Dec.Database<StubDec>.Get("麵包"), StaticRefsDecs.麵包);
+            Assert.AreSame(Dec.Database<StubDec>.Get("خبز"), StaticRefsDecs.خبز);
+            Assert.AreSame(Dec.Database<StubDec>.Get("빵"), StaticRefsDecs.빵);
+            Assert.AreSame(Dec.Database<StubDec>.Get("לחם"), StaticRefsDecs.לחם);
+            Assert.AreSame(Dec.Database<StubDec>.Get("ပေါင်မုန့်"), StaticRefsDecs.ပေါင်မုန့်);
+            Assert.AreSame(Dec.Database<StubDec>.Get("ബ്രെഡ്"), StaticRefsDecs.ബ്രെഡ്);
+        }
+
+        [Test]
+        public void Custom()
+        {
+            Dec.Config.CultureInfo = new System.Globalization.CultureInfo("hu-HU");
+
+            var z = @"
+                    <Record>
+                        <recordFormatVersion>1</recordFormatVersion>
+                        <data>1,5</data>
+                    </Record>
+            ";
+            var result = Dec.Recorder.Read<float>(z, "data");
+            Assert.AreEqual(1.5, result);
+        }
+    }
+}