Daniel/orbit/serialize harden (#228)

* work for now * hardened all serialization code * hardened all serialize code * more hardening * fix wasm
StoneOrbits · Apr 26, 2024 · 3711593 · 3711593
1 parent 3311225
commit 3711593
Show file tree

Hide file tree

Showing 21 changed files with 285 additions and 185 deletions.
diff --git a/VortexEngine/VortexLib/VortexLib.cpp b/VortexEngine/VortexLib/VortexLib.cpp
@@ -166,15 +166,16 @@ EMSCRIPTEN_BINDINGS(Vortex) {
     .function("sanity", &ByteStream::sanity)
     .function("checkCRC", &ByteStream::checkCRC)
     .function("isCRCDirty", &ByteStream::isCRCDirty)
-    .function("serialize", select_overload<bool(uint8_t)>(&ByteStream::serialize))
-    .function("serialize16", select_overload<bool(uint16_t)>(&ByteStream::serialize))
-    .function("serialize32", select_overload<bool(uint32_t)>(&ByteStream::serialize))
+    .function("serialize8", &ByteStream::serialize8)
+    .function("serialize16", &ByteStream::serialize16)
+    .function("serialize32", &ByteStream::serialize32)
     .function("resetUnserializer", &ByteStream::resetUnserializer)
     .function("moveUnserializer", &ByteStream::moveUnserializer)
     .function("unserializerAtEnd", &ByteStream::unserializerAtEnd)
-    .function("unserialize8", &ByteStream::unserialize8)
-    .function("unserialize16", &ByteStream::unserialize16)
-    .function("unserialize32", &ByteStream::unserialize32)
+    // TODO: provide better apis here
+    //.function("unserialize8", &ByteStream::unserialize8)
+    //.function("unserialize16", &ByteStream::unserialize16)
+    //.function("unserialize32", &ByteStream::unserialize32)
     .function("peek8", &ByteStream::peek8)
     .function("peek16", &ByteStream::peek16)
     .function("peek32", &ByteStream::peek32)

diff --git a/VortexEngine/src/Colors/ColorTypes.cpp b/VortexEngine/src/Colors/ColorTypes.cpp
@@ -198,18 +198,32 @@ RGBColor RGBColor::adjustBrightness(uint8_t fadeBy)
   return *this;
 }
 
-void RGBColor::serialize(ByteStream &buffer) const
+bool RGBColor::serialize(ByteStream &buffer) const
 {
-  buffer.serialize(red);
-  buffer.serialize(green);
-  buffer.serialize(blue);
+  if (!buffer.serialize8(red)) {
+    return false;
+  }
+  if (!buffer.serialize8(green)) {
+    return false;
+  }
+  if (!buffer.serialize8(blue)) {
+    return false;
+  }
+  return true;
 }
 
-void RGBColor::unserialize(ByteStream &buffer)
+bool RGBColor::unserialize(ByteStream &buffer)
 {
-  buffer.unserialize(&red);
-  buffer.unserialize(&green);
-  buffer.unserialize(&blue);
+  if (!buffer.unserialize8(&red)) {
+    return false;
+  }
+  if (!buffer.unserialize8(&green)) {
+    return false;
+  }
+  if (!buffer.unserialize8(&blue)) {
+    return false;
+  }
+  return true;
 }
 
 // ========================================================

diff --git a/VortexEngine/src/Colors/ColorTypes.h b/VortexEngine/src/Colors/ColorTypes.h
@@ -79,8 +79,8 @@ class RGBColor
   void clear();
 
   RGBColor adjustBrightness(uint8_t fadeBy);
-  void serialize(ByteStream &buffer) const;
-  void unserialize(ByteStream &buffer);
+  bool serialize(ByteStream &buffer) const;
+  bool unserialize(ByteStream &buffer);
 
   uint32_t raw() const { return ((uint32_t)red << 16) | ((uint32_t)green << 8) | (uint32_t)blue; }
 

diff --git a/VortexEngine/src/Colors/Colorset.cpp b/VortexEngine/src/Colors/Colorset.cpp
@@ -450,52 +450,76 @@ bool Colorset::onEnd() const
   return (m_curIndex == m_numColors - 1);
 }
 
-void Colorset::serialize(ByteStream &buffer) const
+bool Colorset::serialize(ByteStream &buffer) const
 {
-  buffer.serialize(m_numColors);
+  if (!buffer.serialize8(m_numColors)) {
+    return false;
+  }
   // write all the reds/greens/blues together to maximize chance of
   // repeated values to improve RLE compression
   for (uint8_t i = 0; i < m_numColors; ++i) {
-    buffer.serialize(m_palette[i].red);
+    if (!buffer.serialize8(m_palette[i].red)) {
+      return false;
+    }
   }
   for (uint8_t i = 0; i < m_numColors; ++i) {
-    buffer.serialize(m_palette[i].green);
+    if (!buffer.serialize8(m_palette[i].green)) {
+      return false;
+    }
   }
   for (uint8_t i = 0; i < m_numColors; ++i) {
-    buffer.serialize(m_palette[i].blue);
+    if (!buffer.serialize8(m_palette[i].blue)) {
+      return false;
+    }
   }
+  return true;
 }
 
-void Colorset::unserialize(ByteStream &buffer)
+bool Colorset::unserialize(ByteStream &buffer)
 {
-  buffer.unserialize(&m_numColors);
-  initPalette(m_numColors);
+  if (!buffer.unserialize8(&m_numColors)) {
+    return false;
+  }
+  if (m_numColors > MAX_COLOR_SLOTS) {
+    return false;
+  }
+  if (!initPalette(m_numColors)) {
+    return false;
+  }
   for (uint8_t i = 0; i < m_numColors; ++i) {
-    buffer.unserialize(&m_palette[i].red);
+    if (!buffer.unserialize8(&m_palette[i].red)) {
+      return false;
+    }
   }
   for (uint8_t i = 0; i < m_numColors; ++i) {
-    buffer.unserialize(&m_palette[i].green);
+    if (!buffer.unserialize8(&m_palette[i].green)) {
+      return false;
+    }
   }
   for (uint8_t i = 0; i < m_numColors; ++i) {
-    buffer.unserialize(&m_palette[i].blue);
+    if (!buffer.unserialize8(&m_palette[i].blue)) {
+      return false;
+    }
   }
+  return true;
 }
 
-void Colorset::initPalette(uint8_t numColors)
+bool Colorset::initPalette(uint8_t numColors)
 {
   if (m_palette) {
     delete[] m_palette;
     m_palette = nullptr;
   }
   if (!numColors) {
-    return;
+    return true;
   }
   //m_palette = (RGBColor *)vcalloc(numColors, sizeof(RGBColor));
   m_palette = new RGBColor[numColors];
   if (!m_palette) {
     ERROR_OUT_OF_MEMORY();
-    return;
+    return false;
   }
   m_numColors = numColors;
+  return true;
 }
 
diff --git a/VortexEngine/src/Colors/Colorset.h b/VortexEngine/src/Colors/Colorset.h
@@ -143,12 +143,12 @@ class Colorset
   bool onEnd() const;
 
   // serialize the colorset to save/load
-  void serialize(ByteStream &buffer) const;
-  void unserialize(ByteStream &buffer);
+  bool serialize(ByteStream &buffer) const;
+  bool unserialize(ByteStream &buffer);
 
 private:
   // pre-allocate the palette
-  void initPalette(uint8_t numColors);
+  bool initPalette(uint8_t numColors);
 
   // palette of colors
   RGBColor *m_palette;

diff --git a/VortexEngine/src/Menus/MenuList/EditorConnection.cpp b/VortexEngine/src/Menus/MenuList/EditorConnection.cpp
@@ -49,7 +49,9 @@ bool EditorConnection::receiveMessage(const char *message)
     return false;
   }
   for (size_t i = 0; i < len; ++i) {
-    m_receiveBuffer.unserialize(&byte);
+    if (!m_receiveBuffer.unserialize8(&byte)) {
+      return false;
+    }
   }
   // if everything was read out, reset
   if (m_receiveBuffer.unserializerAtEnd()) {
@@ -274,7 +276,9 @@ bool EditorConnection::receiveModes()
     return false;
   }
   // okay unserialize now, first unserialize the size
-  m_receiveBuffer.unserialize(&size);
+  if (!m_receiveBuffer.unserialize32(&size)) {
+    return false;
+  }
   // create a new ByteStream that will hold the full buffer of data
   ByteStream buf(m_receiveBuffer.rawSize());
   // then copy everything from the receive buffer into the rawdata
@@ -304,7 +308,9 @@ bool EditorConnection::receiveDemoMode()
     return false;
   }
   // okay unserialize now, first unserialize the size
-  m_receiveBuffer.unserialize(&size);
+  if (!m_receiveBuffer.unserialize32(&size)) {
+    return false;
+  }
   // create a new ByteStream that will hold the full buffer of data
   ByteStream buf(m_receiveBuffer.rawSize());
   // then copy everything from the receive buffer into the rawdata

diff --git a/VortexEngine/src/Menus/MenuList/Randomizer.cpp b/VortexEngine/src/Menus/MenuList/Randomizer.cpp
@@ -60,7 +60,9 @@ bool Randomizer::init()
     ByteStream ledData;
     Pattern *pat = cur->getPattern(LED_MULTI);
     if (pat) {
-      pat->serialize(ledData);
+      if (!pat->serialize(ledData)) {
+        return false;
+      }
     }
     m_multiRandCtx.seed(ledData.recalcCRC());
   }
@@ -71,7 +73,9 @@ bool Randomizer::init()
     ByteStream ledData;
     Pattern *pat = cur->getPattern(l);
     if (pat) {
-      pat->serialize(ledData);
+      if (!pat->serialize(ledData)) {
+        return false;
+      }
     }
     m_singlesRandCtx[l].seed(ledData.recalcCRC());
   }

diff --git a/VortexEngine/src/Modes/Mode.cpp b/VortexEngine/src/Modes/Mode.cpp
@@ -187,8 +187,12 @@ bool Mode::loadFromBuffer(ByteStream &modeBuffer)
   uint8_t major = 0;
   uint8_t minor = 0;
   // unserialize the vortex version
-  modeBuffer.unserialize(&major);
-  modeBuffer.unserialize(&minor);
+  if (!modeBuffer.unserialize8(&major)) {
+    return false;
+  }
+  if (!modeBuffer.unserialize8(&minor)) {
+    return false;
+  }
   // check the version for incompatibility
   if (!VortexEngine::checkVersion(major, minor)) {
     // incompatible version
@@ -204,35 +208,43 @@ bool Mode::loadFromBuffer(ByteStream &modeBuffer)
   return true;
 }
 
-void Mode::serialize(ByteStream &buffer, uint8_t numLeds) const
+bool Mode::serialize(ByteStream &buffer, uint8_t numLeds) const
 {
   if (!numLeds) {
     numLeds = MODE_LEDCOUNT;
   }
   // serialize the number of leds
-  buffer.serialize(numLeds);
+  if (!buffer.serialize8(numLeds)) {
+    return false;
+  }
   // empty mode?
   if (!numLeds) {
-    return;
+    return true;
   }
   // serialize the flags
   ModeFlags flags = getFlags();
-  buffer.serialize(flags);
+  if (!buffer.serialize8(flags)) {
+    return false;
+  }
 #if VORTEX_SLIM == 0
   // serialiaze the multi led?
   if ((flags & MODE_FLAG_MULTI_LED) && m_multiPat) {
     // serialize the multi led
-    m_multiPat->serialize(buffer);
+    if (!m_multiPat->serialize(buffer)) {
+      return false;
+    }
   }
 #endif
   // if no single leds then just stop here
   if (!(flags & MODE_FLAG_SINGLE_LED)) {
-    return;
+    return true;
   }
   // if there are any sparse singles (spaces) then we need to
   // serialize an led map of which singles are set
   if (flags & MODE_FLAG_SPARSE_SINGLES) {
-    buffer.serialize((uint32_t)getSingleLedMap());
+    if (!buffer.serialize32((uint32_t)getSingleLedMap())) {
+      return false;
+    }
   }
   // then iterate each single led and serialize it
   for (LedPos pos = LED_FIRST; pos < numLeds; ++pos) {
@@ -241,12 +253,15 @@ void Mode::serialize(ByteStream &buffer, uint8_t numLeds) const
       continue;
     }
     // just serialize the pattern then colorset
-    entry->serialize(buffer);
+    if (!entry->serialize(buffer)) {
+      return false;
+    }
     // if they are all same single then only serialize one
     if (flags & MODE_FLAG_ALL_SAME_SINGLE) {
       break;
     }
   }
+  return true;
 }
 
 // this is a hairy function, but a bit of a necessary complexity
@@ -255,7 +270,9 @@ bool Mode::unserialize(ByteStream &buffer)
   clearPattern(LED_ALL);
   uint8_t ledCount = LED_COUNT;
   // unserialize the number of leds
-  buffer.unserialize(&ledCount);
+  if (!buffer.unserialize8(&ledCount)) {
+    return false;
+  }
 #if FIXED_LED_COUNT == 0
   // it's important that we only increase the led count if necessary
   // otherwise we may end up reducing our led count and only rendering
@@ -272,7 +289,9 @@ bool Mode::unserialize(ByteStream &buffer)
   }
   // unserialize the flags value
   ModeFlags flags = 0;
-  buffer.unserialize(&flags);
+  if (!buffer.unserialize8(&flags)) {
+    return false;
+  }
   Pattern *firstPat = nullptr;
   // if there is a multi led pattern then unserialize it
   if (flags & MODE_FLAG_MULTI_LED) {
@@ -291,6 +310,9 @@ bool Mode::unserialize(ByteStream &buffer)
 #else
     // otherwise in normal build actually unserialize it
     m_multiPat = PatternBuilder::unserialize(buffer);
+    if (!m_multiPat) {
+      return false;
+    }
     m_multiPat->init();
 #endif
   }
@@ -301,7 +323,9 @@ bool Mode::unserialize(ByteStream &buffer)
   // is there an led map to unserialize? if not default to all
   LedMap map = (1 << ledCount) - 1;
   if (flags & MODE_FLAG_SPARSE_SINGLES) {
-    buffer.unserialize((uint32_t *)&map);
+    if (!buffer.unserialize32((uint32_t *)&map)) {
+      return false;
+    }
   }
   // unserialize all singleled patterns into their positions
   MAP_FOREACH_LED(map) {
@@ -319,6 +343,10 @@ bool Mode::unserialize(ByteStream &buffer)
       // otherwise unserialize the pattern like normal
       m_singlePats[pos] = PatternBuilder::unserialize(buffer);
     }
+    if (!m_singlePats[pos]) {
+      clearPattern(LED_ALL);
+      return false;
+    }
     m_singlePats[pos]->bind(pos);
   }
   // there is a few different possibilities here:

diff --git a/VortexEngine/src/Modes/Mode.h b/VortexEngine/src/Modes/Mode.h
@@ -67,7 +67,7 @@ class Mode
   bool loadFromBuffer(ByteStream &saveBuffer);
 
   // save the mode to serial
-  void serialize(ByteStream &buffer, uint8_t numLeds = 0) const;
+  bool serialize(ByteStream &buffer, uint8_t numLeds = 0) const;
   // load the mode from serial (optional led count)
   bool unserialize(ByteStream &buffer);