Skip to content

Commit

Permalink
chore: Use msgpack to reduce redundancy (AztecProtocol/barretenberg#612)
Browse files Browse the repository at this point in the history 
Co-authored-by: ludamad <[email protected]>
  • Loading branch information
@ludamad @ludamad0
ludamad and ludamad0 authored Jul 19, 2023
1 parent 427ab87 commit e5a2196
Show file tree
Hide file tree
Showing 44 changed files with 462 additions and 725 deletions.
5 changes: 0 additions & 5 deletions circuits/cpp/barretenberg/cpp/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ option(BENCHMARKS "Build benchmarks" ON)
option(FUZZING "Build fuzzing harnesses" OFF)
option(DISABLE_TBB "Intel Thread Building Blocks" ON)
option(COVERAGE "Enable collecting coverage from tests" OFF)
option(SERIALIZE_CANARY "Build with serialize canary" OFF)
option(ENABLE_ASAN "Address sanitizer for debugging tricky memory corruption" OFF)
option(ENABLE_HEAVY_TESTS "Enable heavy tests when collecting coverage" OFF)
option(INSTALL_BARRETENBERG "Enable installation of barretenberg. (Projects embedding barretenberg may want to turn this OFF.)" ON)
Expand All @@ -40,10 +39,6 @@ if(ENABLE_ASAN)
set(DISABLE_ASM ON)
endif()

if(SERIALIZE_CANARY)
add_definitions(-DENABLE_SERIALIZE_CANARY)
endif()

if(FUZZING)
add_definitions(-DFUZZING=1)

Expand Down
154 changes: 100 additions & 54 deletions circuits/cpp/barretenberg/cpp/src/barretenberg/common/serialize.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,12 @@
#pragma once
#include "barretenberg/common/log.hpp"
#include "barretenberg/common/net.hpp"
#include "barretenberg/serialize/msgpack_apply.hpp"
#include <array>
#include <cassert>
#include <iostream>
#include <map>
#include <memory>
#include <optional>
#include <type_traits>
#include <vector>
Expand All @@ -42,7 +44,14 @@
__extension__ using uint128_t = unsigned __int128;
#endif

template <typename T>
concept IntegralOrEnum = std::integral<T> || std::is_enum_v<T>;

namespace serialize {
// Forward declare derived msgpack methods
void read(auto& it, msgpack_concepts::HasMsgPack auto& obj);
void write(auto& buf, const msgpack_concepts::HasMsgPack auto& obj);

// Basic integer read / write, to / from raw buffers.
// Pointers to buffers are advanced by length of type.
inline void read(uint8_t const*& it, uint8_t& value)
Expand Down Expand Up @@ -123,90 +132,65 @@ inline void write(uint8_t*& it, uint128_t value)
#endif

// Reading / writing integer types to / from vectors.
template <typename T> inline std::enable_if_t<std::is_integral_v<T>> read(std::vector<uint8_t> const& buf, T& value)
void read(std::vector<uint8_t> const& buf, std::integral auto& value)
{
auto ptr = &buf[0];
read(ptr, value);
}

template <typename T> inline std::enable_if_t<std::is_integral_v<T>> write(std::vector<uint8_t>& buf, T value)
void write(std::vector<uint8_t>& buf, const std::integral auto& value)
{
buf.resize(buf.size() + sizeof(T));
uint8_t* ptr = &*buf.end() - sizeof(T);
buf.resize(buf.size() + sizeof(value));
uint8_t* ptr = &*buf.end() - sizeof(value);
write(ptr, value);
}

// Reading writing integer types to / from streams.
template <typename T> inline std::enable_if_t<std::is_integral_v<T>> read(std::istream& is, T& value)
void read(std::istream& is, std::integral auto& value)
{
std::array<uint8_t, sizeof(T)> buf;
is.read((char*)buf.data(), sizeof(T));
std::array<uint8_t, sizeof(value)> buf;
is.read((char*)buf.data(), sizeof(value));
uint8_t const* ptr = &buf[0];
read(ptr, value);
}

template <typename T> inline std::enable_if_t<std::is_integral_v<T>> write(std::ostream& os, T value)
void write(std::ostream& os, const std::integral auto& value)
{
std::array<uint8_t, sizeof(T)> buf;
std::array<uint8_t, sizeof(value)> buf;
uint8_t* ptr = &buf[0];
write(ptr, value);
os.write((char*)buf.data(), sizeof(T));
}

// DEBUG_CANARY_READ and DEBUG_CANARY_WRITE write strings during debug testing
// so that we can detect serialization misalignment for more complicated types.
// This is in an awkward location as it must see the above functions, and be seen by the below functions.
#ifndef ENABLE_SERIALIZE_CANARY
#define DEBUG_CANARY_WRITE(buf, x)
#define DEBUG_CANARY_READ(it, x)
#else
#define DEBUG_CANARY_WRITE(buf, x) serialize::write(buf, (uint64_t) typeid(x).hash_code())
#define DEBUG_CANARY_READ(it, x) \
{ \
uint64_t hash_code; \
serialize::read(it, hash_code); \
if (hash_code != (uint64_t) typeid(x).hash_code()) { \
throw std::runtime_error(std::string("Could not read magic string for ") + typeid(x).name()); \
} \
}
#endif
os.write((char*)buf.data(), sizeof(value));
}
} // namespace serialize

namespace std {

// Forwarding functions from std to serialize namespace for integers.
template <typename B, typename T> inline std::enable_if_t<std::is_integral_v<T>> read(B& buf, T& value)
inline void read(auto& buf, std::integral auto& value)
{
DEBUG_CANARY_READ(buf, value);
serialize::read(buf, value);
}

template <typename B, typename T> inline std::enable_if_t<std::is_integral_v<T>> write(B& buf, T value)
inline void write(auto& buf, std::integral auto value)
{
DEBUG_CANARY_WRITE(buf, value);
serialize::write(buf, value);
}

// Optimised specialisation for reading arrays of bytes from a raw buffer.
template <size_t N> inline void read(uint8_t const*& it, std::array<uint8_t, N>& value)
{
DEBUG_CANARY_READ(it, value);
std::copy(it, it + N, value.data());
it += N;
}

// Optimised specialisation for writing arrays of bytes to a raw buffer.
template <size_t N> inline void write(uint8_t*& buf, std::array<uint8_t, N> const& value)
{
DEBUG_CANARY_WRITE(buf, value);
std::copy(value.begin(), value.end(), buf);
buf += N;
}

// Optimised specialisation for reading vectors of bytes from a raw buffer.
inline void read(uint8_t const*& it, std::vector<uint8_t>& value)
{
DEBUG_CANARY_READ(it, value);
uint32_t size;
read(it, size);
value.resize(size);
Expand All @@ -217,7 +201,6 @@ inline void read(uint8_t const*& it, std::vector<uint8_t>& value)
// Optimised specialisation for writing vectors of bytes to a raw buffer.
inline void write(uint8_t*& buf, std::vector<uint8_t> const& value)
{
DEBUG_CANARY_WRITE(buf, value);
write(buf, static_cast<uint32_t>(value.size()));
std::copy(value.begin(), value.end(), buf);
buf += value.size();
Expand All @@ -226,7 +209,6 @@ inline void write(uint8_t*& buf, std::vector<uint8_t> const& value)
// Optimised specialisation for reading vectors of bytes from an input stream.
inline void read(std::istream& is, std::vector<uint8_t>& value)
{
DEBUG_CANARY_READ(is, value);
uint32_t size;
read(is, size);
value.resize(size);
Expand All @@ -236,15 +218,13 @@ inline void read(std::istream& is, std::vector<uint8_t>& value)
// Optimised specialisation for writing vectors of bytes to an output stream.
inline void write(std::ostream& os, std::vector<uint8_t> const& value)
{
DEBUG_CANARY_WRITE(os, value);
write(os, static_cast<uint32_t>(value.size()));
os.write((char*)value.data(), (std::streamsize)value.size());
}

// Optimised specialisation for writing arrays of bytes to a vector.
template <size_t N> inline void write(std::vector<uint8_t>& buf, std::array<uint8_t, N> const& value)
{
DEBUG_CANARY_WRITE(buf, value);
buf.resize(buf.size() + N);
auto ptr = &*buf.end() - N;
write(ptr, value);
Expand All @@ -253,14 +233,13 @@ template <size_t N> inline void write(std::vector<uint8_t>& buf, std::array<uint
// Optimised specialisation for writing arrays of bytes to an output stream.
template <size_t N> inline void write(std::ostream& os, std::array<uint8_t, N> const& value)
{
DEBUG_CANARY_WRITE(os, value);
os.write((char*)value.data(), value.size());
}

// Generic read of array of types from supported buffer types.
template <typename B, typename T, size_t N> inline void read(B& it, std::array<T, N>& value)
{
DEBUG_CANARY_READ(it, value);
using serialize::read;
for (size_t i = 0; i < N; ++i) {
read(it, value[i]);
}
Expand All @@ -269,7 +248,7 @@ template <typename B, typename T, size_t N> inline void read(B& it, std::array<T
// Generic write of array of types to supported buffer types.
template <typename B, typename T, size_t N> inline void write(B& buf, std::array<T, N> const& value)
{
DEBUG_CANARY_WRITE(buf, value);
using serialize::write;
for (size_t i = 0; i < N; ++i) {
write(buf, value[i]);
}
Expand All @@ -278,7 +257,7 @@ template <typename B, typename T, size_t N> inline void write(B& buf, std::array
// Generic read of vector of types from supported buffer types.
template <typename B, typename T, typename A> inline void read(B& it, std::vector<T, A>& value)
{
DEBUG_CANARY_READ(it, value);
using serialize::read;
uint32_t size;
read(it, size);
value.resize(size);
Expand All @@ -290,6 +269,7 @@ template <typename B, typename T, typename A> inline void read(B& it, std::vecto
// Generic write of vector of types to supported buffer types.
template <typename B, typename T, typename A> inline void write(B& buf, std::vector<T, A> const& value)
{
using serialize::write;
write(buf, static_cast<uint32_t>(value.size()));
for (size_t i = 0; i < value.size(); ++i) {
write(buf, value[i]);
Expand All @@ -299,7 +279,7 @@ template <typename B, typename T, typename A> inline void write(B& buf, std::vec
// Read string from supported buffer types.
template <typename B> inline void read(B& it, std::string& value)
{
DEBUG_CANARY_READ(it, value);
using serialize::read;
std::vector<uint8_t> buf;
read(it, buf);
value = std::string(buf.begin(), buf.end());
Expand All @@ -308,29 +288,46 @@ template <typename B> inline void read(B& it, std::string& value)
// Write of strings to supported buffer types.
template <typename B> inline void write(B& buf, std::string const& value)
{
using serialize::write;
write(buf, std::vector<uint8_t>(value.begin(), value.end()));
}

// Read std::pair.
template <typename B, typename T, typename U> inline void read(B& it, std::pair<T, U>& value)
{
DEBUG_CANARY_READ(it, value);
using serialize::read;
read(it, value.first);
read(it, value.second);
}

// Write std::pair.
template <typename B, typename T, typename U> inline void write(B& buf, std::pair<T, U> const& value)
{
DEBUG_CANARY_WRITE(buf, value);
using serialize::write;
write(buf, value.first);
write(buf, value.second);
}

// Read std::shared_ptr.
template <typename B, typename T> inline void read(B& it, std::shared_ptr<T>& value_ptr)
{
using serialize::read;
T value;
read(it, value);
*value_ptr = std::make_shared(value);
}

// Write std::shared_ptr.
template <typename B, typename T> inline void write(B& buf, std::shared_ptr<T> const& value_ptr)
{
using serialize::write;
write(buf, *value_ptr);
}

// Read std::map
template <typename B, typename T, typename U> inline void read(B& it, std::map<T, U>& value)
{
DEBUG_CANARY_READ(it, value);
using serialize::read;
value.clear();
uint32_t size;
read(it, size);
Expand All @@ -344,7 +341,7 @@ template <typename B, typename T, typename U> inline void read(B& it, std::map<T
// Write std::map.
template <typename B, typename T, typename U> inline void write(B& buf, std::map<T, U> const& value)
{
DEBUG_CANARY_WRITE(buf, value);
using serialize::write;
write(buf, static_cast<uint32_t>(value.size()));
for (auto const& kv : value) {
write(buf, kv);
Expand All @@ -354,7 +351,7 @@ template <typename B, typename T, typename U> inline void write(B& buf, std::map
// Read std::optional<T>.
template <typename B, typename T> inline void read(B& it, std::optional<T>& opt_value)
{
DEBUG_CANARY_READ(it, opt_value);
using serialize::read;
bool is_nullopt;
read(it, is_nullopt);
if (is_nullopt) {
Expand All @@ -371,7 +368,7 @@ template <typename B, typename T> inline void read(B& it, std::optional<T>& opt_
// value.
template <typename B, typename T> inline void write(B& buf, std::optional<T> const& opt_value)
{
DEBUG_CANARY_WRITE(buf, opt_value);
using serialize::write;
if (opt_value) {
write(buf, false); // is not nullopt
write(buf, *opt_value);
Expand Down Expand Up @@ -449,4 +446,53 @@ using out_str_buf = uint8_t**;

// Use these to pass a raw memory pointer.
using in_ptr = void* const*;
using out_ptr = void**;
using out_ptr = void**;

namespace serialize {

/**
* @brief Helper method for better error reporting. Clang does not give the best errors for "auto..."
* arguments.
*/
inline void _read_msgpack_field(auto& it, auto& field)
{
using namespace serialize;
read(it, field);
}

/**
* @brief Automatically derived read for any object that defines .msgpack() (implicitly defined by MSGPACK_FIELDS).
* @param it The iterator to read from.
* @param func The function to call with each field as an argument.
*/
inline void read(auto& it, msgpack_concepts::HasMsgPack auto& obj)
{
msgpack::msgpack_apply(obj, [&](auto&... obj_fields) {
// apply 'read' to each object field
(_read_msgpack_field(it, obj_fields), ...);
});
};

/**
* @brief Helper method for better error reporting. Clang does not give the best errors for "auto..."
* arguments.
*/
inline void _write_msgpack_field(auto& it, const auto& field)
{
using namespace serialize;
write(it, field);
}
/**
* @brief Automatically derived write for any object that defines .msgpack() (implicitly defined by MSGPACK_FIELDS).
* @param buf The buffer to write to.
* @param func The function to call with each field as an argument.
*/
inline void write(auto& buf, const msgpack_concepts::HasMsgPack auto& obj)
{
using namespace serialize;
msgpack::msgpack_apply(obj, [&](auto&... obj_fields) {
// apply 'write' to each object field
(_write_msgpack_field(buf, obj_fields), ...);
});
};
} // namespace serialize
6 changes: 3 additions & 3 deletions circuits/cpp/barretenberg/cpp/src/barretenberg/common/streams.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,7 @@ inline std::ostream& operator<<(std::ostream& os, std::vector<uint8_t> const& ar
return os;
}

template <typename T, std::enable_if_t<std::is_integral<T>::value, bool> = true, typename A>
inline std::ostream& operator<<(std::ostream& os, std::vector<T, A> const& arr)
template <std::integral T, typename A> inline std::ostream& operator<<(std::ostream& os, std::vector<T, A> const& arr)
{
os << "[";
for (auto element : arr) {
Expand All @@ -30,7 +29,8 @@ inline std::ostream& operator<<(std::ostream& os, std::vector<T, A> const& arr)
return os;
}

template <typename T, std::enable_if_t<!std::is_integral<T>::value, bool> = true, typename A>
template <typename T, typename A>
requires(!std::integral<T>)
inline std::ostream& operator<<(std::ostream& os, std::vector<T, A> const& arr)
{
os << "[\n";
Expand Down
Loading

0 comments on commit e5a2196

Please sign in to comment.