general update + recompute request timeouts

madMAx43v3r · May 16, 2023 · 6869b46 · 6869b46
1 parent f06414d
commit 6869b46
Show file tree

Hide file tree

Showing 36 changed files with 45 additions and 25 deletions.
diff --git a/mmx-node/linux/x86_64/bin/ProofOfSpace b/mmx-node/linux/x86_64/bin/ProofOfSpace
diff --git a/mmx-node/linux/x86_64/bin/calc_test_rewards b/mmx-node/linux/x86_64/bin/calc_test_rewards
diff --git a/mmx-node/linux/x86_64/bin/chia_recompute_proxy b/mmx-node/linux/x86_64/bin/chia_recompute_proxy
diff --git a/mmx-node/linux/x86_64/bin/chia_recompute_server b/mmx-node/linux/x86_64/bin/chia_recompute_server
diff --git a/mmx-node/linux/x86_64/bin/dump_binary b/mmx-node/linux/x86_64/bin/dump_binary
diff --git a/mmx-node/linux/x86_64/bin/mmx b/mmx-node/linux/x86_64/bin/mmx
diff --git a/mmx-node/linux/x86_64/bin/mmx_compile b/mmx-node/linux/x86_64/bin/mmx_compile
diff --git a/mmx-node/linux/x86_64/bin/mmx_farmer b/mmx-node/linux/x86_64/bin/mmx_farmer
diff --git a/mmx-node/linux/x86_64/bin/mmx_harvester b/mmx-node/linux/x86_64/bin/mmx_harvester
diff --git a/mmx-node/linux/x86_64/bin/mmx_node b/mmx-node/linux/x86_64/bin/mmx_node
diff --git a/mmx-node/linux/x86_64/bin/mmx_timelord b/mmx-node/linux/x86_64/bin/mmx_timelord
diff --git a/mmx-node/linux/x86_64/bin/mmx_wallet b/mmx-node/linux/x86_64/bin/mmx_wallet
diff --git a/mmx-node/linux/x86_64/bin/tx_bench b/mmx-node/linux/x86_64/bin/tx_bench
diff --git a/mmx-node/linux/x86_64/bin/vnxclose b/mmx-node/linux/x86_64/bin/vnxclose
diff --git a/mmx-node/linux/x86_64/bin/vnxdump b/mmx-node/linux/x86_64/bin/vnxdump
diff --git a/mmx-node/linux/x86_64/bin/vnxgraph b/mmx-node/linux/x86_64/bin/vnxgraph
diff --git a/mmx-node/linux/x86_64/bin/vnxpasswd b/mmx-node/linux/x86_64/bin/vnxpasswd
diff --git a/mmx-node/linux/x86_64/bin/vnxplay b/mmx-node/linux/x86_64/bin/vnxplay
diff --git a/mmx-node/linux/x86_64/bin/vnxread b/mmx-node/linux/x86_64/bin/vnxread
diff --git a/mmx-node/linux/x86_64/bin/vnxreadcsv b/mmx-node/linux/x86_64/bin/vnxreadcsv
diff --git a/mmx-node/linux/x86_64/bin/vnxrecord b/mmx-node/linux/x86_64/bin/vnxrecord
diff --git a/mmx-node/linux/x86_64/bin/vnxrouter b/mmx-node/linux/x86_64/bin/vnxrouter
diff --git a/mmx-node/linux/x86_64/bin/vnxservice b/mmx-node/linux/x86_64/bin/vnxservice
diff --git a/mmx-node/linux/x86_64/bin/vnxtimeserver b/mmx-node/linux/x86_64/bin/vnxtimeserver
diff --git a/mmx-node/linux/x86_64/bin/vnxtopic b/mmx-node/linux/x86_64/bin/vnxtopic
diff --git a/mmx-node/linux/x86_64/include/vnx/Input.h b/mmx-node/linux/x86_64/include/vnx/Input.h
@@ -329,8 +329,39 @@ void read(TypeInput& in, std::shared_ptr<T>& value, const TypeCode* type_code, c
 template<typename T>
 void read(TypeInput& in, vnx::optional<T>& value, const TypeCode* type_code, const uint16_t* code);
 
+template<typename T>
+void read_vector_data(TypeInput& in, T& vector, const TypeCode* type_code, const uint16_t* value_code, const uint32_t size) {
+	if(size > in.max_list_size) {
+		throw std::logic_error("vector size > max_list_size: " + std::to_string(size));
+	}
+	if(!in.safe_read) {
+		vector.resize(size);
+	}
+	if(is_equivalent<typename T::value_type>{}(value_code, type_code)) {
+		if(in.safe_read) {
+			for(uint32_t i = 0; i < size; i += VNX_BUFFER_SIZE) {
+				const auto chunk_size = std::min<uint32_t>(size - i, VNX_BUFFER_SIZE);
+				vector.resize(i + chunk_size);
+				in.read((char*)&vector[i], chunk_size);
+			}
+		} else {
+			in.read((char*)vector.data(), size * sizeof(typename T::value_type));
+		}
+	} else {
+		for(uint32_t i = 0; i < size; ++i) {
+			if(in.safe_read) {
+				typename T::value_type tmp;
+				vnx::type<typename T::value_type>().read(in, tmp, type_code, value_code);
+				vector.push_back(tmp);
+			} else {
+				vnx::type<typename T::value_type>().read(in, vector[i], type_code, value_code);
+			}
+		}
+	}
+}
+
 /// Returns dimension array and total size of a matrix (CODE_MATRIX)
-size_t get_matrix_size(std::vector<size_t>& dims, const uint16_t* code);
+size_t read_matrix_size(std::vector<size_t>& dims, const uint16_t* code);
 
 /** \brief Reads a dynamically allocated N-dimensional matrix from the input stream.
  *
@@ -341,15 +372,9 @@ size_t get_matrix_size(std::vector<size_t>& dims, const uint16_t* code);
  */
 template<typename T>
 void read_matrix(TypeInput& in, std::vector<T>& data, std::vector<size_t>& dims, const uint16_t* code) {
-	const auto total_size = get_matrix_size(dims, code);
-	if(total_size > in.max_list_size) {
-		throw std::logic_error("matrix size > max_list_size: " + std::to_string(total_size));
-	}
-	data.resize(total_size);
+	const auto total_size = read_matrix_size(dims, code);
 	const uint16_t* value_code = code + 2 + dims.size();
-	for(size_t i = 0; i < total_size; ++i) {
-		vnx::type<T>().read(in, data[i], nullptr, value_code);
-	}
+	read_vector_data(in, data, nullptr, value_code, total_size);
 }
 
 /**
@@ -389,7 +414,7 @@ void read_array(TypeInput& in, T& array, const TypeCode* type_code, const uint16
 		case CODE_MATRIX:
 		case CODE_ALT_MATRIX: {
 			std::vector<size_t> dims;
-			size = get_matrix_size(dims, code);
+			size = read_matrix_size(dims, code);
 			value_code = code + 2 + dims.size();
 			break;
 		}
@@ -457,7 +482,7 @@ void read_vector(TypeInput& in, T& vector, const TypeCode* type_code, const uint
 		case CODE_MATRIX:
 		case CODE_ALT_MATRIX: {
 			std::vector<size_t> dims;
-			size = get_matrix_size(dims, code);
+			size = read_matrix_size(dims, code);
 			value_code = code + 2 + dims.size();
 			break;
 		}
@@ -470,17 +495,7 @@ void read_vector(TypeInput& in, T& vector, const TypeCode* type_code, const uint
 			size = 1;
 			value_code = code;
 	}
-	if(size > in.max_list_size) {
-		throw std::logic_error("vector size > max_list_size: " + std::to_string(size));
-	}
-	vector.resize(size);
-	if(is_equivalent<typename T::value_type>{}(value_code, type_code)) {
-		in.read((char*)vector.data(), size * sizeof(typename T::value_type));
-	} else {
-		for(uint32_t i = 0; i < size; ++i) {
-			vnx::type<typename T::value_type>().read(in, vector[i], type_code, value_code);
-		}
-	}
+	read_vector_data(in, vector, type_code, value_code, size);
 }
 
 /// Same as read_vector<T>() with T = std::vector<T>

diff --git a/mmx-node/linux/x86_64/include/vnx/InputStream.h b/mmx-node/linux/x86_64/include/vnx/InputStream.h
@@ -339,6 +339,8 @@ class TypeInput : public InputBuffer {
 	 */
 	void clear();
 
+	bool safe_read = false;											///< to avoid pre-allocation of large buffers
+
 	size_t max_list_size = VNX_MAX_SIZE;							///< to limit maximum list size
 
 	std::unordered_map<Hash64, const TypeCode*> type_code_map;		///< for faster lock-free lookup

diff --git a/mmx-node/linux/x86_64/include/vnx/ThreadPool.h b/mmx-node/linux/x86_64/include/vnx/ThreadPool.h
@@ -49,6 +49,9 @@ class ThreadPool {
 	/// Returns number of active non-idle threads (ie. which are doing some work)
 	size_t get_num_running() const;
 
+	/// Returns get_num_pending() + get_num_running()
+	size_t get_num_pending_total() const;
+
 	/// Trigger all threads to exit. (thread-safe)
 	void exit();
 

diff --git a/mmx-node/linux/x86_64/include/vnx/Type.h b/mmx-node/linux/x86_64/include/vnx/Type.h
@@ -70,13 +70,13 @@ typedef SSIZE_T ssize_t;
  * Applies to CODE_DYNAMIC.
  * Absolute maximum is 255 to detect different byte order.
  */
-#define VNX_MAX_BYTE_CODE_SIZE 32
+#define VNX_MAX_BYTE_CODE_SIZE 32u
 
 /** \brief The buffer size for InputBuffer and OutputBuffer.
  * 
  * This is also the maximum combined size of all primitive fields in a struct or class.
  */
-#define VNX_BUFFER_SIZE 16384
+#define VNX_BUFFER_SIZE 16384u
 
 
 /// VNX namespace

diff --git a/mmx-node/linux/x86_64/lib/libmmx_db.so b/mmx-node/linux/x86_64/lib/libmmx_db.so
diff --git a/mmx-node/linux/x86_64/lib/libmmx_iface.so b/mmx-node/linux/x86_64/lib/libmmx_iface.so
diff --git a/mmx-node/linux/x86_64/lib/libmmx_modules.so b/mmx-node/linux/x86_64/lib/libmmx_modules.so
diff --git a/mmx-node/linux/x86_64/lib/libmmx_vm.so b/mmx-node/linux/x86_64/lib/libmmx_vm.so
diff --git a/mmx-node/linux/x86_64/lib/libvnx_addons.so b/mmx-node/linux/x86_64/lib/libvnx_addons.so
diff --git a/mmx-node/linux/x86_64/lib/libvnx_base.so b/mmx-node/linux/x86_64/lib/libvnx_base.so
diff --git a/mmx-node/linux/x86_64/www/web-gui/public/GIT_COMMIT_HASH.json b/mmx-node/linux/x86_64/www/web-gui/public/GIT_COMMIT_HASH.json
@@ -1 +1 @@
-{ "GIT_COMMIT_HASH": "1bfa52a036bc9c52d801ad4085a21be6e9bfc636" }
+{ "GIT_COMMIT_HASH": "48d2be445f3895d40fbe6b0db5345d494b88ea1f" }
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		{ "GIT_COMMIT_HASH": "1bfa52a036bc9c52d801ad4085a21be6e9bfc636" }
		{ "GIT_COMMIT_HASH": "48d2be445f3895d40fbe6b0db5345d494b88ea1f" }