Tighten safe long bounds (#457)

mtth · Mar 30, 2024 · 7cb76a6 · 7cb76a6
1 parent bce35cd
commit 7cb76a6
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 5 deletions.
diff --git a/lib/types.js b/lib/types.js
@@ -2987,10 +2987,11 @@ function readArraySize(tap) {
  *
  * + We are not using the `Number` constants for compatibility with older
  *   browsers.
- * + We must remove one from each bound because of rounding errors.
+ * + We divide the bounds by two to avoid rounding errors during zigzag encoding
+ *   (see https://github.com/mtth/avsc/issues/455).
  */
 function isSafeLong(n) {
-  return n >= -9007199254740990 && n <= 9007199254740990;
+  return n >= -4503599627370496 && n <= 4503599627370496;
 }
 
 /**

diff --git a/test/test_types.js b/test/test_types.js
@@ -131,8 +131,8 @@ suite('types', () => {
 
     let data = [
       {
-        valid: [1, -3, 12314, 9007199254740990, 900719925474090],
-        invalid: [null, 'hi', undefined, 9007199254740991, 1.3, 1e67]
+        valid: [1, -3, 12314, 4503599627370496],
+        invalid: [null, 'hi', undefined, 9007199254740990, 1.3, 1e67]
       }
     ];
 
@@ -145,7 +145,7 @@ suite('types', () => {
     test('resolve long > float', () => {
       let t1 = Type.forSchema('long');
       let t2 = Type.forSchema('float');
-      let n = 9007199254740990; // Number.MAX_SAFE_INTEGER - 1
+      let n = 4503599627370496; // Number.MAX_SAFE_INTEGER / 2
       let buf = t1.toBuffer(n);
       let f = t2.fromBuffer(buf, t2.createResolver(t1));
       assert(Math.abs(f - n) / n < 1e-7);