chore: Use msgpack to reduce redundancy

cpp/src/barretenberg/common/serialize.hpp

@@ -30,6 +30,7 @@
 #pragma once
 #include "barretenberg/common/log.hpp"
 #include "barretenberg/common/net.hpp"
+#include "barretenberg/serialize/msgpack_apply.hpp"
 #include <array>
 #include <cassert>
 #include <iostream>
@@ -42,7 +43,14 @@
 __extension__ using uint128_t = unsigned __int128;
 #endif
 
+template <typename T>
+concept IntegralOrEnum = std::integral<T> || std::is_enum_v<T>;
+
 namespace serialize {
+// Forward declare derived msgpack methods
+void read(auto& it, msgpack_concepts::HasMsgPack auto& obj);
+void write(auto& buf, const msgpack_concepts::HasMsgPack auto& obj);
+
 // Basic integer read / write, to / from raw buffers.
 // Pointers to buffers are advanced by length of type.
 inline void read(uint8_t const*& it, uint8_t& value)
@@ -123,34 +131,34 @@ inline void write(uint8_t*& it, uint128_t value)
 #endif
 
 // Reading / writing integer types to / from vectors.
-template <typename T> inline std::enable_if_t<std::is_integral_v<T>> read(std::vector<uint8_t> const& buf, T& value)
+void read(std::vector<uint8_t> const& buf, std::integral auto& value)
 {
     auto ptr = &buf[0];
     read(ptr, value);
 }
 
-template <typename T> inline std::enable_if_t<std::is_integral_v<T>> write(std::vector<uint8_t>& buf, T value)
+void write(std::vector<uint8_t>& buf, const std::integral auto& value)
 {
-    buf.resize(buf.size() + sizeof(T));
-    uint8_t* ptr = &*buf.end() - sizeof(T);
+    buf.resize(buf.size() + sizeof(value));
+    uint8_t* ptr = &*buf.end() - sizeof(value);
     write(ptr, value);
 }
 
 // Reading writing integer types to / from streams.
-template <typename T> inline std::enable_if_t<std::is_integral_v<T>> read(std::istream& is, T& value)
+void read(std::istream& is, std::integral auto& value)
 {
-    std::array<uint8_t, sizeof(T)> buf;
-    is.read((char*)buf.data(), sizeof(T));
+    std::array<uint8_t, sizeof(value)> buf;
+    is.read((char*)buf.data(), sizeof(value));
     uint8_t const* ptr = &buf[0];
     read(ptr, value);
 }
 
-template <typename T> inline std::enable_if_t<std::is_integral_v<T>> write(std::ostream& os, T value)
+void write(std::ostream& os, const std::integral auto& value)
 {
-    std::array<uint8_t, sizeof(T)> buf;
+    std::array<uint8_t, sizeof(value)> buf;
     uint8_t* ptr = &buf[0];
     write(ptr, value);
-    os.write((char*)buf.data(), sizeof(T));
+    os.write((char*)buf.data(), sizeof(value));
 }
 
 // DEBUG_CANARY_READ and DEBUG_CANARY_WRITE write strings during debug testing
@@ -173,19 +181,26 @@ template <typename T> inline std::enable_if_t<std::is_integral_v<T>> write(std::
 } // namespace serialize
 
 namespace std {
-
-// Forwarding functions from std to serialize namespace for integers.
-template <typename B, typename T> inline std::enable_if_t<std::is_integral_v<T>> read(B& buf, T& value)
+inline void read(auto& buf, std::integral auto& value)
 {
     DEBUG_CANARY_READ(buf, value);
     serialize::read(buf, value);
 }
 
-template <typename B, typename T> inline std::enable_if_t<std::is_integral_v<T>> write(B& buf, T value)
+inline void write(auto& buf, std::integral auto value)
 {
     DEBUG_CANARY_WRITE(buf, value);
     serialize::write(buf, value);
 }
+// Forward to the serialize namespace templated methods for HasMsgPack
+inline void read(auto& it, msgpack_concepts::HasMsgPack auto& obj)
+{
+    serialize::read(it, obj);
+}
+inline void write(auto& buf, const msgpack_concepts::HasMsgPack auto& obj)
+{
+    serialize::write(buf, obj);
+}
 
 // Optimised specialisation for reading arrays of bytes from a raw buffer.
 template <size_t N> inline void read(uint8_t const*& it, std::array<uint8_t, N>& value)
@@ -260,6 +275,7 @@ template <size_t N> inline void write(std::ostream& os, std::array<uint8_t, N> c
 // Generic read of array of types from supported buffer types.
 template <typename B, typename T, size_t N> inline void read(B& it, std::array<T, N>& value)
 {
+    using serialize::read;
     DEBUG_CANARY_READ(it, value);
     for (size_t i = 0; i < N; ++i) {
         read(it, value[i]);
@@ -394,15 +410,13 @@ template <typename T, typename B> T from_buffer(B const& buffer, size_t offset =
 
 template <typename T> std::vector<uint8_t> to_buffer(T const& value)
 {
-    using serialize::write;
     std::vector<uint8_t> buf;
     write(buf, value);
     return buf;
 }
 
 template <typename T> uint8_t* to_heap_buffer(T const& value)
 {
-    using serialize::write;
     std::vector<uint8_t> buf;
     write(buf, value);
     auto* ptr = (uint8_t*)aligned_alloc(64, buf.size());
@@ -449,4 +463,65 @@ using out_str_buf = uint8_t**;
 
 // Use these to pass a raw memory pointer.
 using in_ptr = void* const*;
-using out_ptr = void**;
+using out_ptr = void**;
+
+namespace serialize {
+
+/**
+ * @brief Passthrough method that exists for better error reporting.
+ */
+void read_msgpack_field(auto& it, auto& field)
+{
+    using namespace serialize;
+    read(it, field);
+}
+/**
+ * @brief Passthrough method that exists for better error reporting.
+ */
+void read_msgpack_fields(auto& it, auto&... fields)
+{
+    using namespace serialize;
+    (read_msgpack_field(it, fields), ...);
+}
+/**
+ * @brief Automatically derived read for any object that defines .msgpack() (implicitly defined by MSGPACK_FIELDS).
+ * @param it The iterator to read from.
+ * @param func The function to call with each field as an argument.
+ */
+void read(auto& it, msgpack_concepts::HasMsgPack auto& obj)
+{
+    msgpack::msgpack_apply(obj, [&](auto&... obj_fields) {
+        // apply 'read' to each object field
+        read_msgpack_fields(it, obj_fields...);
+    });
+};
+/**
+ * @brief Passthrough method that exists for better error reporting.
+ */
+void write_msgpack_field(auto& it, const auto& field)
+{
+    using namespace serialize;
+    write(it, field);
+}
+// @brief explicit form. Mysteriously has to sometimes be called.
+void write_msgpack(auto& buf, const msgpack_concepts::HasMsgPack auto& obj)
+{
+    msgpack::msgpack_apply(obj, [&](auto&... obj_fields) {
+        // apply 'write' to each object field
+        (write_msgpack_field(buf, obj_fields), ...);
+    });
+};
+/**
+ * @brief Automatically derived :write for any object that defines .msgpack() (implicitly defined by MSGPACK_FIELDS).
+ * @param buf The buffer to write to.
+ * @param func The function to call with each field as an argument.
+ */
+void write(auto& buf, const msgpack_concepts::HasMsgPack auto& obj)
+{
+    using namespace serialize;
+    msgpack::msgpack_apply(obj, [&](auto&... obj_fields) {
+        // apply 'write' to each object field
+        (write_msgpack_field(buf, obj_fields), ...);
+    });
+};
+} // namespace serialize

cpp/src/barretenberg/common/streams.hpp

@@ -19,8 +19,7 @@ inline std::ostream& operator<<(std::ostream& os, std::vector<uint8_t> const& ar
     return os;
 }
 
-template <typename T, std::enable_if_t<std::is_integral<T>::value, bool> = true, typename A>
-inline std::ostream& operator<<(std::ostream& os, std::vector<T, A> const& arr)
+template <std::integral T, typename A> inline std::ostream& operator<<(std::ostream& os, std::vector<T, A> const& arr)
 {
     os << "[";
     for (auto element : arr) {
@@ -30,7 +29,8 @@ inline std::ostream& operator<<(std::ostream& os, std::vector<T, A> const& arr)
     return os;
 }
 
-template <typename T, std::enable_if_t<!std::is_integral<T>::value, bool> = true, typename A>
+template <typename T, typename A>
+    requires(!std::integral<T>)
 inline std::ostream& operator<<(std::ostream& os, std::vector<T, A> const& arr)
 {
     os << "[\n";

cpp/src/barretenberg/crypto/ecdsa/c_bind.cpp

@@ -9,7 +9,7 @@ WASM_EXPORT void ecdsa__compute_public_key(uint8_t const* private_key, uint8_t*
 {
     auto priv_key = from_buffer<secp256k1::fr>(private_key);
     secp256k1::g1::affine_element pub_key = secp256k1::g1::one * priv_key;
-    write(public_key_buf, pub_key);
+    serialize::write(public_key_buf, pub_key);
 }
 
 WASM_EXPORT void ecdsa__construct_signature(uint8_t const* message,
@@ -47,7 +47,7 @@ WASM_EXPORT void ecdsa__recover_public_key_from_signature(uint8_t const* message
     auto recovered_pub_key =
         crypto::ecdsa::recover_public_key<Sha256Hasher, secp256k1::fq, secp256k1::fr, secp256k1::g1>(
             std::string((char*)message, msg_len), sig);
-    write(output_pub_key, recovered_pub_key);
+    serialize::write(output_pub_key, recovered_pub_key);
 }
 
 WASM_EXPORT bool ecdsa__verify_signature(uint8_t const* message,

cpp/src/barretenberg/crypto/pedersen_commitment/c_bind.cpp

@@ -60,7 +60,7 @@ WASM_EXPORT void pedersen__commit(uint8_t const* inputs_buffer, uint8_t* output)
     read(inputs_buffer, to_compress);
     grumpkin::g1::affine_element pedersen_hash = crypto::pedersen_commitment::commit_native(to_compress);
 
-    write(output, pedersen_hash);
+    serialize::write(output, pedersen_hash);
 }
 
 WASM_EXPORT void pedersen_plookup_commit(uint8_t const* inputs_buffer, uint8_t* output)
@@ -69,7 +69,7 @@ WASM_EXPORT void pedersen_plookup_commit(uint8_t const* inputs_buffer, uint8_t*
     read(inputs_buffer, to_compress);
     grumpkin::g1::affine_element pedersen_hash = crypto::pedersen_commitment::lookup::commit_native(to_compress);
 
-    write(output, pedersen_hash);
+    serialize::write(output, pedersen_hash);
 }
 
 WASM_EXPORT void pedersen_plookup_commit_with_hash_index(uint8_t const* inputs_buffer,
@@ -81,7 +81,7 @@ WASM_EXPORT void pedersen_plookup_commit_with_hash_index(uint8_t const* inputs_b
     grumpkin::g1::affine_element pedersen_hash =
         crypto::pedersen_commitment::lookup::commit_native(to_compress, hash_index);
 
-    write(output, pedersen_hash);
+    serialize::write(output, pedersen_hash);
 }
 
 WASM_EXPORT void pedersen__buffer_to_field(uint8_t const* data, size_t length, uint8_t* r)

cpp/src/barretenberg/crypto/pedersen_commitment/c_bind_new.cpp

@@ -60,7 +60,7 @@ WASM_EXPORT void pedersen___commit(fr::vec_in_buf inputs_buffer, fr::out_buf out
     read(inputs_buffer, to_compress);
     grumpkin::g1::affine_element pedersen_hash = crypto::pedersen_commitment::commit_native(to_compress);
 
-    write(output, pedersen_hash);
+    serialize::write(output, pedersen_hash);
 }
 
 WASM_EXPORT void pedersen___plookup_commit(fr::vec_in_buf inputs_buffer, fr::out_buf output)
@@ -69,7 +69,7 @@ WASM_EXPORT void pedersen___plookup_commit(fr::vec_in_buf inputs_buffer, fr::out
     read(inputs_buffer, to_compress);
     grumpkin::g1::affine_element pedersen_hash = crypto::pedersen_commitment::lookup::commit_native(to_compress);
 
-    write(output, pedersen_hash);
+    serialize::write(output, pedersen_hash);
 }
 
 WASM_EXPORT void pedersen___plookup_commit_with_hash_index(fr::vec_in_buf inputs_buffer,
@@ -81,7 +81,7 @@ WASM_EXPORT void pedersen___plookup_commit_with_hash_index(fr::vec_in_buf inputs
     grumpkin::g1::affine_element pedersen_hash =
         crypto::pedersen_commitment::lookup::commit_native(to_compress, ntohl(*hash_index));
 
-    write(output, pedersen_hash);
+    serialize::write(output, pedersen_hash);
 }
 
 WASM_EXPORT void pedersen___buffer_to_field(uint8_t const* data, fr::out_buf r)

cpp/src/barretenberg/crypto/schnorr/c_bind.cpp

@@ -11,15 +11,15 @@ WASM_EXPORT void compute_public_key(uint8_t const* private_key, uint8_t* public_
 {
     auto priv_key = from_buffer<grumpkin::fr>(private_key);
     grumpkin::g1::affine_element pub_key = grumpkin::g1::one * priv_key;
-    write(public_key_buf, pub_key);
+    serialize::write(public_key_buf, pub_key);
 }
 
 WASM_EXPORT void negate_public_key(uint8_t const* public_key_buffer, uint8_t* output)
 {
     // Negate the public key (effectively negating the y-coordinate of the public key) and return the resulting public
     // key.
     auto account_public_key = from_buffer<grumpkin::g1::affine_element>(public_key_buffer);
-    barretenberg::group_elements::write(output, -account_public_key);
+    serialize::write(output, -account_public_key);
 }
 
 WASM_EXPORT void construct_signature(
@@ -56,7 +56,7 @@ WASM_EXPORT void multisig_create_multisig_public_key(uint8_t const* private_key,
 
     auto agg_pubkey = multisig_public_key(key_pair);
 
-    write(multisig_pubkey_buf, agg_pubkey);
+    serialize::write(multisig_pubkey_buf, agg_pubkey);
 }
 
 WASM_EXPORT bool multisig_validate_and_combine_signer_pubkeys(uint8_t const* signer_pubkey_buf,
@@ -67,7 +67,7 @@ WASM_EXPORT bool multisig_validate_and_combine_signer_pubkeys(uint8_t const* sig
         from_buffer<std::vector<multisig::MultiSigPublicKey>>(signer_pubkey_buf);
 
     if (auto combined_key = multisig::validate_and_combine_signer_pubkeys(pubkeys); combined_key) {
-        write(combined_key_buf, *combined_key);
+        serialize::write(combined_key_buf, *combined_key);
         return true;
     } else {
         return false;
@@ -80,8 +80,8 @@ WASM_EXPORT void multisig_construct_signature_round_1(uint8_t* round_one_public_
     using multisig = crypto::schnorr::multisig<grumpkin::g1, KeccakHasher, Blake2sHasher>;
 
     auto [public_output, private_output] = multisig::construct_signature_round_1();
-    write(round_one_public_output_buf, public_output);
-    write(round_one_private_output_buf, private_output);
+    serialize::write_msgpack(round_one_public_output_buf, public_output);
+    serialize::write(round_one_private_output_buf, private_output);
 }
 
 WASM_EXPORT bool multisig_construct_signature_round_2(uint8_t const* message,

cpp/src/barretenberg/crypto/schnorr/c_bind_new.cpp

@@ -13,15 +13,15 @@ WASM_EXPORT void schnorr_compute_public_key(uint8_t const* private_key, uint8_t*
 {
     auto priv_key = from_buffer<grumpkin::fr>(private_key);
     grumpkin::g1::affine_element pub_key = grumpkin::g1::one * priv_key;
-    write(public_key_buf, pub_key);
+    serialize::write(public_key_buf, pub_key);
 }
 
 WASM_EXPORT void schnorr_negate_public_key(uint8_t const* public_key_buffer, uint8_t* output)
 {
     // Negate the public key (effectively negating the y-coordinate of the public key) and return the resulting public
     // key.
     auto account_public_key = from_buffer<grumpkin::g1::affine_element>(public_key_buffer);
-    barretenberg::group_elements::write(output, -account_public_key);
+    serialize::write(output, -account_public_key);
 }
 
 WASM_EXPORT void schnorr_construct_signature(uint8_t const* message_buf,
@@ -62,7 +62,7 @@ WASM_EXPORT void schnorr_multisig_create_multisig_public_key(uint8_t const* priv
 
     auto agg_pubkey = multisig_public_key(key_pair);
 
-    write(multisig_pubkey_buf, agg_pubkey);
+    serialize::write(multisig_pubkey_buf, agg_pubkey);
 }
 
 WASM_EXPORT void schnorr_multisig_validate_and_combine_signer_pubkeys(uint8_t const* signer_pubkey_buf,
@@ -75,10 +75,10 @@ WASM_EXPORT void schnorr_multisig_validate_and_combine_signer_pubkeys(uint8_t co
     auto combined_key = multisig::validate_and_combine_signer_pubkeys(pubkeys);
 
     if (combined_key) {
-        write(combined_key_buf, *combined_key);
+        serialize::write(combined_key_buf, *combined_key);
         *success = true;
     } else {
-        write(combined_key_buf, affine_element::one());
+        serialize::write(combined_key_buf, affine_element::one());
         *success = false;
     }
 }
@@ -89,8 +89,8 @@ WASM_EXPORT void schnorr_multisig_construct_signature_round_1(uint8_t* round_one
     using multisig = crypto::schnorr::multisig<grumpkin::g1, KeccakHasher, Blake2sHasher>;
 
     auto [public_output, private_output] = multisig::construct_signature_round_1();
-    write(round_one_public_output_buf, public_output);
-    write(round_one_private_output_buf, private_output);
+    serialize::write(round_one_public_output_buf, public_output);
+    serialize::write(round_one_private_output_buf, private_output);
 }
 
 WASM_EXPORT void schnorr_multisig_construct_signature_round_2(uint8_t const* message_buf,