Merge pull request #25 from mochi-hpc/carns/dev-bedrock-service-query

[WIP] bedrock service query update

README.md

@@ -3,6 +3,10 @@
 mochi-quintain is a microservice (i.e., a Mochi provider) that responds to
 parameterized RPCs to generate synthetic concurrent server load.
 
+NOTE: As of June 2024, Quintain requires that bedrock be built with SSG
+support (`+ssg` in Spack).  In the future it will be updated to support
+Flock.
+
 ## Provider
 The provider portion of mochi-quintain can be started with bedrock.
 
@@ -21,3 +25,20 @@ Or none at all:
 The primary client is an MPI program that generates a workload for the
 provider and measures it's performance.
 
+## Example executioa by hand
+
+Note that this example assumes that you are running from within the build
+tree and thus need to set an explicit library path for bedrock to be able to
+find the provider libraries for quintain.
+
+In one terminal (for a server):
+```
+export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/home/carns/working/src/mochi/mochi-quintain/build/src/.libs"
+
+bedrock --jx9 -c ~/working/src/mochi/mochi-quintain/tests/mochi-quintain-provider.jx9 --jx9-context "num_rpc_xstreams=5" na+sm://
+```
+
+In another terminal (for a client):
+```
+src/quintain-benchmark -g quintain.ssg -j ../tests/quintain-benchmark-example.json  -o foo
+```

configure.ac

@@ -85,26 +85,18 @@ LIBS="$BEDROCK_LIBS $LIBS"
 CPPFLAGS="$BEDROCK_CFLAGS $CPPFLAGS"
 CFLAGS="$BEDROCK_CFLAGS $CFLAGS"
 
-# see if bedrock has a C API for retrieving configuration
-AC_MSG_CHECKING(for bedrock_service_query_config in Bedrock)
-AC_TRY_COMPILE([
-#include <bedrock/service-handle.h>
-], [
-char* foo = bedrock_service_query_config(NULL, "");
-],
-AC_MSG_RESULT(yes),
-AC_MSG_RESULT(no)
-AC_MSG_ERROR([This version of Bedrock does not support bedrock_service_query_config])
-)
-
 PKG_CHECK_MODULES([SSG], [ssg >= 0.5],[],
    [AC_MSG_ERROR([Could not find working SSG installationi v0.5 or higher])])
 LIBS="$SSG_LIBS $LIBS"
 CPPFLAGS="$SSG_CFLAGS $CPPFLAGS"
 CFLAGS="$SSG_CFLAGS $CFLAGS"
 
 # need zlib output file
-CHECK_ZLIB
+PKG_CHECK_MODULES([ZLIB], [zlib],[],
+   [AC_MSG_ERROR([Could not find working zlib installation!])])
+LIBS="$ZLIB_LIBS $LIBS"
+CPPFLAGS="$ZLIB_CFLAGS $CPPFLAGS"
+CFLAGS="$ZLIB_CFLAGS $CFLAGS"
 
 # optional mpi; this will cause the benchmark program to be built as well,
 # in addition to the provider piece

src/Makefile.subdir

@@ -1,11 +1,15 @@
 src_libquintain_bedrock_la_SOURCES += src/quintain-bedrock-module.c
-src_libquintain_bedrock_la_LIBADD = src/libquintain-server.la src/libquintain-client.la
+src_libquintain_bedrock_la_LIBADD = src/libquintain-server.la src/libquintain-client.la -lbedrock-client
 
 src_libquintain_client_la_SOURCES += src/quintain-client.c \
-                                     src/quintain-rpc.h
+                                     src/quintain-rpc.h \
+				     src/bedrock-c-wrapper.cpp \
+				     bedrock-c-wrapper.h
 
 src_libquintain_server_la_SOURCES += src/quintain-server.c \
-                                     src/quintain-rpc.h
+                                     src/quintain-rpc.h \
+				     src/bedrock-c-wrapper.cpp \
+				     bedrock-c-wrapper.h
 
 dist_bin_SCRIPTS += src/quintain-benchmark-parse.sh
 

src/bedrock-c-wrapper.cpp

@@ -0,0 +1,48 @@
+#include "bedrock-c-wrapper.h"
+#include <bedrock/Client.hpp>
+#include <bedrock/ServiceHandle.hpp>
+
+struct bedrock_client {
+    bedrock::Client inner;
+};
+
+struct bedrock_service {
+    bedrock::ServiceHandle inner;
+};
+
+extern "C" int bedrock_client_init(margo_instance_id mid,
+                                   bedrock_client_t* client)
+{
+    *client = new bedrock_client{bedrock::Client{mid}};
+    return BEDROCK_SUCCESS;
+}
+
+extern "C" int bedrock_client_finalize(bedrock_client_t client)
+{
+    delete client;
+    return BEDROCK_SUCCESS;
+}
+
+extern "C" int bedrock_service_handle_create(bedrock_client_t   client,
+                                             const char*        address,
+                                             uint16_t           provider_id,
+                                             bedrock_service_t* sh)
+{
+    *sh = new bedrock_service{
+        client->inner.makeServiceHandle(address, provider_id)};
+    return BEDROCK_SUCCESS;
+}
+
+extern "C" int bedrock_service_handle_destroy(bedrock_service_t sh)
+{
+    delete sh;
+    return BEDROCK_SUCCESS;
+}
+
+extern "C" char* bedrock_service_query_config(bedrock_service_t sh,
+                                              const char*       script)
+{
+    std::string config;
+    sh->inner.queryConfig(script, &config);
+    return strdup(config.c_str());
+}

src/bedrock-c-wrapper.h

@@ -0,0 +1,33 @@
+#ifndef BEDROCK_C_WRAPPER
+#define BEDROCK_C_WRAPPER
+
+#include <margo.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define BEDROCK_SUCCESS 0
+
+typedef struct bedrock_client*  bedrock_client_t;
+typedef struct bedrock_service* bedrock_service_t;
+
+int bedrock_client_init(margo_instance_id mid, bedrock_client_t* client);
+
+int bedrock_client_finalize(bedrock_client_t client);
+
+int bedrock_service_handle_create(bedrock_client_t,
+                                  const char*        address,
+                                  uint16_t           provider_id,
+                                  bedrock_service_t* sh);
+
+int bedrock_service_handle_destroy(bedrock_service_t sh);
+
+char* bedrock_service_query_config(bedrock_service_t sh, const char* script);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/quintain-benchmark.c

@@ -24,9 +24,9 @@
 #include <abt.h>
 #include <ssg.h>
 #include <quintain-client.h>
-#include <bedrock/service-handle.h>
 
 #include "quintain-macros.h"
+#include "bedrock-c-wrapper.h"
 
 /* record up to 32 million (power of 2) samples.  This will take 256 MiB of RAM
  * per rank */

src/quintain-server.c

@@ -242,7 +242,12 @@ static void qtn_work_ult(hg_handle_t handle)
 finish:
     margo_respond(handle, &out);
     margo_free_input(handle, &in);
-    if (bulk_handle != HG_BULK_NULL) margo_bulk_free(bulk_handle);
+    if (bulk_handle != HG_BULK_NULL) {
+        if (in.flags & QTN_WORK_USE_SERVER_POOLSET)
+            margo_bulk_poolset_release(provider->poolset, bulk_handle);
+        else
+            margo_bulk_free(bulk_handle);
+    }
     if (bulk_buffer != NULL) free(bulk_buffer);
     if (out.resp_buffer) free(out.resp_buffer);
     margo_destroy(handle);