serialization refactoring (#691)

* serialization refactoring

* Update examples/unix/c11/z_get_attachment.c

Co-authored-by: Alexander Bushnev <[email protected]>

* renaming to ze_

* replace _into_ with _to_

* format

* docs update

* fix bytes slice iterator not to rely on serializer

* removed from/to from ze_(de)serialize_ functions

* make writer serializer owned and make them own z_bytes

* docs fix

* docs fix

* test fix

* remove ze_serializer_from_bytes;
add ze_serializer_from/to_writer and ze_deserializer_from/to_reader;

* remove ze_serializer_from_bytes;
add ze_serializer_from/to_writer and ze_deserializer_from/to_reader;

* add missing overloads for c++ functions

* added z_bytes_reader_remaining and ze_deserializer_is_done methods

* move serialization/deserialization under unstable

* ci fix

* fix serialization.h

* report deserialization error when payload size is longer than deserialzie target

* format

* remove serialize/deserialize_sequence_end

* fix missing return value

---------

Co-authored-by: Alexander Bushnev <[email protected]>

.github/workflows/build-check.yaml

@@ -234,7 +234,7 @@ jobs:
       - name: Build project and run test
         run: |
           sudo apt install -y ninja-build
-          CMAKE_GENERATOR=Ninja make
+          Z_FEATURE_UNSTABLE_API=1 CMAKE_GENERATOR=Ninja make
           python3 ./build/tests/attachment.py
         timeout-minutes: 5
 

docs/api.rst

@@ -53,9 +53,8 @@ Data Structures
 .. autoctype:: types.h::zp_send_keep_alive_options_t
 .. autoctype:: types.h::zp_send_join_options_t
 .. autoctype:: types.h::z_bytes_reader_t
-.. autoctype:: types.h::z_bytes_iterator_t
 .. autoctype:: types.h::z_bytes_slice_iterator_t
-.. autoctype:: types.h::z_bytes_writer_t
+.. autoctype:: serialization.h::ze_deserializer_t
 
 
 Owned Types
@@ -71,6 +70,14 @@ See :ref:`owned_types_concept`
   
   Represents an array of bytes container.
 
+.. c:type:: z_owned_bytes_writer_t
+
+  Represents a payload writer.
+
+.. c:type:: ze_owned_serializer_t
+
+  Represents a data serializer (unstable).
+
 .. c:type:: z_owned_string_t
 
   Represents a string without null-terminator.
@@ -140,6 +147,14 @@ See :ref:`loaned_types_concept`
 
   Represents an array of bytes container.
 
+.. c:type:: z_loaned_bytes_writer_t
+
+  Represents a payload writer.
+
+.. c:type:: ze_loaned_serializer_t
+
+  Represents a data serializer (unstable).
+
 .. c:type:: z_loaned_string_t
 
   Represents a string without null-terminator.
@@ -320,54 +335,88 @@ Primitives
 .. autocfunction:: primitives.h::z_slice_len
 .. autocfunction:: primitives.h::z_slice_empty
 .. autocfunction:: primitives.h::z_slice_is_empty
-.. autocfunction:: primitives.h::z_bytes_deserialize_into_int8
-.. autocfunction:: primitives.h::z_bytes_deserialize_into_int16
-.. autocfunction:: primitives.h::z_bytes_deserialize_into_int32
-.. autocfunction:: primitives.h::z_bytes_deserialize_into_int64
-.. autocfunction:: primitives.h::z_bytes_deserialize_into_uint8
-.. autocfunction:: primitives.h::z_bytes_deserialize_into_uint16
-.. autocfunction:: primitives.h::z_bytes_deserialize_into_uint32
-.. autocfunction:: primitives.h::z_bytes_deserialize_into_uint64
-.. autocfunction:: primitives.h::z_bytes_deserialize_into_float
-.. autocfunction:: primitives.h::z_bytes_deserialize_into_double
-.. autocfunction:: primitives.h::z_bytes_deserialize_into_slice
-.. autocfunction:: primitives.h::z_bytes_deserialize_into_string
-.. autocfunction:: primitives.h::z_bytes_serialize_from_int8
-.. autocfunction:: primitives.h::z_bytes_serialize_from_int16
-.. autocfunction:: primitives.h::z_bytes_serialize_from_int32
-.. autocfunction:: primitives.h::z_bytes_serialize_from_int64
-.. autocfunction:: primitives.h::z_bytes_serialize_from_uint8
-.. autocfunction:: primitives.h::z_bytes_serialize_from_uint16
-.. autocfunction:: primitives.h::z_bytes_serialize_from_uint32
-.. autocfunction:: primitives.h::z_bytes_serialize_from_uint64
-.. autocfunction:: primitives.h::z_bytes_serialize_from_float
-.. autocfunction:: primitives.h::z_bytes_serialize_from_double
+.. autocfunction:: serialization.h::ze_deserializer_from_bytes
+.. autocfunction:: serialization.h::ze_deserializer_deserialize_int8
+.. autocfunction:: serialization.h::ze_deserializer_deserialize_int16
+.. autocfunction:: serialization.h::ze_deserializer_deserialize_int32
+.. autocfunction:: serialization.h::ze_deserializer_deserialize_int64
+.. autocfunction:: serialization.h::ze_deserializer_deserialize_uint8
+.. autocfunction:: serialization.h::ze_deserializer_deserialize_uint16
+.. autocfunction:: serialization.h::ze_deserializer_deserialize_uint32
+.. autocfunction:: serialization.h::ze_deserializer_deserialize_uint64
+.. autocfunction:: serialization.h::ze_deserializer_deserialize_float
+.. autocfunction:: serialization.h::ze_deserializer_deserialize_double
+.. autocfunction:: serialization.h::ze_deserializer_deserialize_slice
+.. autocfunction:: serialization.h::ze_deserializer_deserialize_string
+.. autocfunction:: serialization.h::ze_deserializer_deserialize_sequence_length
+.. autocfunction:: serialization.h::ze_serializer_empty
+.. autocfunction:: serialization.h::ze_serializer_finish
+.. autocfunction:: serialization.h::ze_serializer_serialize_int8
+.. autocfunction:: serialization.h::ze_serializer_serialize_int16
+.. autocfunction:: serialization.h::ze_serializer_serialize_int32
+.. autocfunction:: serialization.h::ze_serializer_serialize_int64
+.. autocfunction:: serialization.h::ze_serializer_serialize_uint8
+.. autocfunction:: serialization.h::ze_serializer_serialize_uint16
+.. autocfunction:: serialization.h::ze_serializer_serialize_uint32
+.. autocfunction:: serialization.h::ze_serializer_serialize_uint64
+.. autocfunction:: serialization.h::ze_serializer_serialize_float
+.. autocfunction:: serialization.h::ze_serializer_serialize_double
+.. autocfunction:: serialization.h::ze_serializer_serialize_slice
+.. autocfunction:: serialization.h::ze_serializer_serialize_buf
+.. autocfunction:: serialization.h::ze_serializer_serialize_string
+.. autocfunction:: serialization.h::ze_serializer_serialize_str
+.. autocfunction:: serialization.h::ze_serializer_serialize_sequence_length
+.. autocfunction:: serialization.h::ze_deserialize_int8
+.. autocfunction:: serialization.h::ze_deserialize_int16
+.. autocfunction:: serialization.h::ze_deserialize_int32
+.. autocfunction:: serialization.h::ze_deserialize_int64
+.. autocfunction:: serialization.h::ze_deserialize_uint8
+.. autocfunction:: serialization.h::ze_deserialize_uint16
+.. autocfunction:: serialization.h::ze_deserialize_uint32
+.. autocfunction:: serialization.h::ze_deserialize_uint64
+.. autocfunction:: serialization.h::ze_deserialize_float
+.. autocfunction:: serialization.h::ze_deserialize_double
+.. autocfunction:: serialization.h::ze_deserialize_slice
+.. autocfunction:: serialization.h::ze_deserialize_string
+.. autocfunction:: serialization.h::ze_deserializer_is_done
+.. autocfunction:: serialization.h::ze_serialize_int8
+.. autocfunction:: serialization.h::ze_serialize_int16
+.. autocfunction:: serialization.h::ze_serialize_int32
+.. autocfunction:: serialization.h::ze_serialize_int64
+.. autocfunction:: serialization.h::ze_serialize_uint8
+.. autocfunction:: serialization.h::ze_serialize_uint16
+.. autocfunction:: serialization.h::ze_serialize_uint32
+.. autocfunction:: serialization.h::ze_serialize_uint64
+.. autocfunction:: serialization.h::ze_serialize_float
+.. autocfunction:: serialization.h::ze_serialize_double
+.. autocfunction:: serialization.h::ze_serialize_slice
+.. autocfunction:: serialization.h::ze_serialize_buf
+.. autocfunction:: serialization.h::ze_serialize_string
+.. autocfunction:: serialization.h::ze_serialize_str
 .. autocfunction:: primitives.h::z_bytes_from_slice
-.. autocfunction:: primitives.h::z_bytes_serialize_from_slice
+.. autocfunction:: primitives.h::z_bytes_copy_from_slice
 .. autocfunction:: primitives.h::z_bytes_from_buf
-.. autocfunction:: primitives.h::z_bytes_serialize_from_buf
+.. autocfunction:: primitives.h::z_bytes_copy_from_buf
 .. autocfunction:: primitives.h::z_bytes_from_static_buf
 .. autocfunction:: primitives.h::z_bytes_from_string
-.. autocfunction:: primitives.h::z_bytes_serialize_from_string
+.. autocfunction:: primitives.h::z_bytes_copy_from_string
 .. autocfunction:: primitives.h::z_bytes_from_str
-.. autocfunction:: primitives.h::z_bytes_serialize_from_str
+.. autocfunction:: primitives.h::z_bytes_copy_from_str
 .. autocfunction:: primitives.h::z_bytes_from_static_str
 .. autocfunction:: primitives.h::z_bytes_empty
 .. autocfunction:: primitives.h::z_bytes_len
 .. autocfunction:: primitives.h::z_bytes_is_empty
-.. autocfunction:: primitives.h::z_bytes_get_iterator
-.. autocfunction:: primitives.h::z_bytes_iterator_next
 .. autocfunction:: primitives.h::z_bytes_get_slice_iterator
 .. autocfunction:: primitives.h::z_bytes_slice_iterator_next
 .. autocfunction:: primitives.h::z_bytes_get_reader
 .. autocfunction:: primitives.h::z_bytes_reader_read
-.. autocfunction:: primitives.h::z_bytes_reader_read_bounded
 .. autocfunction:: primitives.h::z_bytes_reader_seek
 .. autocfunction:: primitives.h::z_bytes_reader_tell
-.. autocfunction:: primitives.h::z_bytes_get_writer
+.. autocfunction:: primitives.h::z_bytes_reader_remaining
+.. autocfunction:: primitives.h::z_bytes_writer_empty
+.. autocfunction:: primitives.h::z_bytes_writer_finish
 .. autocfunction:: primitives.h::z_bytes_writer_write_all
 .. autocfunction:: primitives.h::z_bytes_writer_append
-.. autocfunction:: primitives.h::z_bytes_writer_append_bounded
 .. autocfunction:: primitives.h::z_timestamp_check
 .. autocfunction:: primitives.h::z_query_target_default
 .. autocfunction:: primitives.h::z_query_consolidation_auto

docs/concepts.rst

@@ -133,11 +133,11 @@ affect the owned object. However, passing the structure to a function transfers
     z_publisher_put_options_default(&options);
     int64_t metadata = 42;
     z_owned_bytes_t attachment;
-    z_bytes_serialize_from_int64(&attachment, metadata);
+    ze_serialize_int64(&attachment, metadata);
     options.attachment = z_move(attachment); // the data itself is still in the `attachment`
 
     z_owned_bytes_t payload;
-    z_bytes_serialize_from_str(&payload, "Don't panic!");
+    z_bytes_copy_from_str(&payload, "Don't panic!"); 
     z_publisher_put(z_loan(pub), z_move(payload), &options);
     // the `payload` and `attachment` are consumed by the `z_publisher_put` function
 

examples/arduino/z_get.ino

@@ -45,7 +45,7 @@ void reply_handler(z_loaned_reply_t *oreply, void *ctx) {
         z_view_string_t keystr;
         z_keyexpr_as_view_string(z_sample_keyexpr(sample), &keystr);
         z_owned_string_t replystr;
-        z_bytes_deserialize_into_string(z_sample_payload(sample), &replystr);
+        z_bytes_to_string(z_sample_payload(sample), &replystr);
 
         Serial.print(" >> [Get listener] Received (");
         Serial.write(z_string_data(z_view_string_loan(&keystr)), z_string_len(z_view_string_loan(&keystr)));

examples/arduino/z_pub.ino

@@ -109,7 +109,7 @@ void loop() {
 
     // Create payload
     z_owned_bytes_t payload;
-    z_bytes_serialize_from_str(&payload, buf);
+    z_bytes_copy_from_str(&payload, buf);
 
     if (z_publisher_put(z_publisher_loan(&pub), z_bytes_move(&payload), NULL) < 0) {
         Serial.println("Error while publishing data");

examples/arduino/z_pull.ino

@@ -104,7 +104,7 @@ void loop() {
         z_view_string_t keystr;
         z_keyexpr_as_view_string(z_sample_keyexpr(z_sample_loan(&sample)), &keystr);
         z_owned_string_t value;
-        z_bytes_deserialize_into_string(z_sample_payload(z_sample_loan(&sample)), &value);
+        z_bytes_to_string(z_sample_payload(z_sample_loan(&sample)), &value);
         Serial.print(">> [Subscriber] Pulled (");
         Serial.write(z_string_data(z_view_string_loan(&keystr)), z_string_len(z_view_string_loan(&keystr)));
         Serial.print(": ");

examples/arduino/z_queryable.ino

@@ -43,7 +43,7 @@ void query_handler(z_loaned_query_t *query, void *arg) {
 
     // Process value
     z_owned_string_t payload_string;
-    z_bytes_deserialize_into_string(z_query_payload(query), &payload_string);
+    z_bytes_to_string(z_query_payload(query), &payload_string);
     if (z_string_len(z_string_loan(&payload_string)) > 1) {
         Serial.print("     with value '");
         Serial.write(z_string_data(z_string_loan(&payload_string)), z_string_len(z_string_loan(&payload_string)));

examples/arduino/z_sub.ino

@@ -37,7 +37,7 @@ void data_handler(z_loaned_sample_t *sample, void *arg) {
     z_view_string_t keystr;
     z_keyexpr_as_view_string(z_sample_keyexpr(sample), &keystr);
     z_owned_string_t value;
-    z_bytes_deserialize_into_string(z_sample_payload(sample), &value);
+    z_bytes_to_string(z_sample_payload(sample), &value);
 
     Serial.print(" >> [Subscription listener] Received (");
     Serial.write(z_string_data(z_view_string_loan(&keystr)), z_string_len(z_view_string_loan(&keystr)));

examples/espidf/z_get.c

@@ -109,7 +109,7 @@ void reply_handler(z_loaned_reply_t *oreply, void *ctx) {
         z_view_string_t keystr;
         z_keyexpr_as_view_string(z_sample_keyexpr(sample), &keystr);
         z_owned_string_t replystr;
-        z_bytes_deserialize_into_string(z_sample_payload(sample), &replystr);
+        z_bytes_to_string(z_sample_payload(sample), &replystr);
 
         printf(" >> Received ('%.*s': '%.*s')\n", (int)z_string_len(z_loan(keystr)), z_string_data(z_loan(keystr)),
                (int)z_string_len(z_loan(replystr)), z_string_data(z_loan(replystr)));

examples/espidf/z_pub.c

@@ -157,7 +157,7 @@ void app_main() {
 
         // Create payload
         z_owned_bytes_t payload;
-        z_bytes_serialize_from_str(&payload, buf);
+        z_bytes_copy_from_str(&payload, buf);
 
         z_publisher_put(z_loan(pub), z_move(payload), NULL);
     }

examples/espidf/z_pull.c

@@ -164,7 +164,7 @@ void app_main() {
             z_view_string_t keystr;
             z_keyexpr_as_view_string(z_sample_keyexpr(z_loan(sample)), &keystr);
             z_owned_string_t value;
-            z_bytes_deserialize_into_string(z_sample_payload(z_loan(sample)), &value);
+            z_bytes_to_string(z_sample_payload(z_loan(sample)), &value);
             printf(">> [Subscriber] Pulled ('%.*s': '%.*s')\n", (int)z_string_len(z_loan(keystr)),
                    z_string_data(z_loan(keystr)), (int)z_string_len(z_loan(value)), z_string_data(z_loan(value)));
             z_drop(z_move(value));

examples/espidf/z_queryable.c

@@ -111,7 +111,7 @@ void query_handler(z_loaned_query_t *query, void *ctx) {
            z_string_data(z_loan(keystr)), (int)z_string_len(z_loan(params)), z_string_data(z_loan(params)));
     // Process value
     z_owned_string_t payload_string;
-    z_bytes_deserialize_into_string(z_query_payload(query), &payload_string);
+    z_bytes_to_string(z_query_payload(query), &payload_string);
     if (z_string_len(z_loan(payload_string)) > 0) {
         printf("     with value '%.*s'\n", (int)z_string_len(z_loan(payload_string)),
                z_string_data(z_loan(payload_string)));

examples/espidf/z_sub.c

@@ -104,7 +104,7 @@ void data_handler(z_loaned_sample_t* sample, void* arg) {
     z_view_string_t keystr;
     z_keyexpr_as_view_string(z_sample_keyexpr(sample), &keystr);
     z_owned_string_t value;
-    z_bytes_deserialize_into_string(z_sample_payload(sample), &value);
+    z_bytes_to_string(z_sample_payload(sample), &value);
     printf(" >> [Subscriber handler] Received ('%.*s': '%.*s')\n", (int)z_string_len(z_view_string_loan(&keystr)),
            z_string_data(z_view_string_loan(&keystr)), (int)z_string_len(z_string_loan(&value)),
            z_string_data(z_string_loan(&value)));

examples/freertos_plus_tcp/z_get.c

@@ -43,7 +43,7 @@ void reply_handler(z_loaned_reply_t *reply, void *ctx) {
         z_view_string_t keystr;
         z_keyexpr_as_view_string(z_sample_keyexpr(sample), &keystr);
         z_owned_string_t replystr;
-        z_bytes_deserialize_into_string(z_sample_payload(sample), &replystr);
+        z_bytes_to_string(z_sample_payload(sample), &replystr);
 
         printf(">> Received ('%.*s': '%.*s')\n", (int)z_string_len(z_loan(keystr)), z_string_data(z_loan(keystr)),
                (int)z_string_len(z_loan(replystr)), z_string_data(z_loan(replystr)));

examples/freertos_plus_tcp/z_pub.c

@@ -99,7 +99,7 @@ void app_main(void) {
 
         // Create payload
         z_owned_bytes_t payload;
-        z_bytes_serialize_from_str(&payload, buf);
+        z_bytes_copy_from_str(&payload, buf);
 
         z_publisher_put_options_t options;
         z_publisher_put_options_default(&options);

examples/freertos_plus_tcp/z_pub_st.c

@@ -65,7 +65,7 @@ void app_main(void) {
 
             // Create payload
             z_owned_bytes_t payload;
-            z_bytes_serialize_from_str(&payload, buf);
+            z_bytes_copy_from_str(&payload, buf);
 
             z_publisher_put(z_loan(pub), z_move(payload), NULL);
             ++idx;

examples/freertos_plus_tcp/z_pull.c

@@ -73,7 +73,7 @@ void app_main(void) {
             z_view_string_t keystr;
             z_keyexpr_as_view_string(z_sample_keyexpr(z_loan(sample)), &keystr);
             z_owned_string_t value;
-            z_bytes_deserialize_into_string(z_sample_payload(z_loan(sample)), &value);
+            z_bytes_to_string(z_sample_payload(z_loan(sample)), &value);
             printf(">> [Subscriber] Pulled ('%.*s': '%.*s')\n", (int)z_string_len(z_loan(keystr)),
                    z_string_data(z_loan(keystr)), (int)z_string_len(z_loan(value)), z_string_data(z_loan(value)));
             z_drop(z_move(value));

examples/freertos_plus_tcp/z_queryable.c

@@ -39,7 +39,7 @@ void query_handler(z_loaned_query_t *query, void *ctx) {
            z_string_data(z_loan(keystr)), (int)z_string_len(z_loan(params)), z_string_data(z_loan(params)));
     // Process value
     z_owned_string_t payload_string;
-    z_bytes_deserialize_into_string(z_query_payload(query), &payload_string);
+    z_bytes_to_string(z_query_payload(query), &payload_string);
     if (z_string_len(z_loan(payload_string)) > 0) {
         printf("     with value '%.*s'\n", (int)z_string_len(z_loan(payload_string)),
                z_string_data(z_loan(payload_string)));

examples/freertos_plus_tcp/z_sub.c

@@ -33,7 +33,7 @@ void data_handler(z_loaned_sample_t *sample, void *ctx) {
     z_view_string_t keystr;
     z_keyexpr_as_view_string(z_sample_keyexpr(sample), &keystr);
     z_owned_string_t value;
-    z_bytes_deserialize_into_string(z_sample_payload(sample), &value);
+    z_bytes_to_string(z_sample_payload(sample), &value);
     printf(">> [Subscriber] Received ('%.*s': '%.*s')\n", (int)z_string_len(z_loan(keystr)),
            z_string_data(z_loan(keystr)), (int)z_string_len(z_loan(value)), z_string_data(z_loan(value)));
     z_drop(z_move(value));

examples/freertos_plus_tcp/z_sub_st.c

@@ -36,7 +36,7 @@ void data_handler(z_loaned_sample_t *sample, void *ctx) {
     z_view_string_t keystr;
     z_keyexpr_as_view_string(z_sample_keyexpr(sample), &keystr);
     z_owned_string_t value;
-    z_bytes_deserialize_into_string(z_sample_payload(sample), &value);
+    z_bytes_to_string(z_sample_payload(sample), &value);
     printf(">> [Subscriber] Received ('%.*s': '%.*s')\n", (int)z_string_len(z_loan(keystr)),
            z_string_data(z_loan(keystr)), (int)z_string_len(z_loan(value)), z_string_data(z_loan(value)));
     z_drop(z_move(value));

examples/mbed/z_get.cpp

@@ -39,7 +39,7 @@ void reply_handler(z_loaned_reply_t *oreply, void *ctx) {
         z_view_string_t keystr;
         z_keyexpr_as_view_string(z_sample_keyexpr(sample), &keystr);
         z_owned_string_t replystr;
-        z_bytes_deserialize_into_string(z_sample_payload(sample), &replystr);
+        z_bytes_to_string(z_sample_payload(sample), &replystr);
 
         printf(" >> Received ('%.*s': '%.*s')\n", (int)z_string_len(z_view_string_loan(&keystr)),
                z_string_data(z_view_string_loan(&keystr)), (int)z_string_len(z_string_loan(&replystr)),

examples/mbed/z_pub.cpp

@@ -77,7 +77,7 @@ int main(int argc, char **argv) {
 
         // Create payload
         z_owned_bytes_t payload;
-        z_bytes_serialize_from_str(&payload, buf);
+        z_bytes_copy_from_str(&payload, buf);
 
         z_publisher_put(z_publisher_loan(&pub), z_bytes_move(&payload), NULL);
     }

examples/mbed/z_pull.cpp

@@ -86,7 +86,7 @@ int main(int argc, char **argv) {
             z_view_string_t keystr;
             z_keyexpr_as_view_string(z_sample_keyexpr(z_sample_loan(&sample)), &keystr);
             z_owned_string_t value;
-            z_bytes_deserialize_into_string(z_sample_payload(z_sample_loan(&sample)), &value);
+            z_bytes_to_string(z_sample_payload(z_sample_loan(&sample)), &value);
             printf(">> [Subscriber] Pulled ('%.*s': '%.*s')\n", (int)z_string_len(z_view_string_loan(&keystr)),
                    z_string_data(z_view_string_loan(&keystr)), (int)z_string_len(z_string_loan(&value)),
                    z_string_data(z_string_loan(&value)));

examples/mbed/z_queryable.cpp

@@ -42,7 +42,7 @@ void query_handler(z_loaned_query_t *query, void *ctx) {
            z_string_data(z_view_string_loan(&params)));
     // Process value
     z_owned_string_t payload_string;
-    z_bytes_deserialize_into_string(z_query_payload(query), &payload_string);
+    z_bytes_to_string(z_query_payload(query), &payload_string);
     if (z_string_len(z_string_loan(&payload_string)) > 1) {
         printf("     with value '%.*s'\n", (int)z_string_len(z_string_loan(&payload_string)),
                z_string_data(z_string_loan(&payload_string)));