From e2d31c7ba6b19beca3dde8d71b8d14af4efa69db Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Sun, 10 Jul 2022 15:19:45 +0000 Subject: [PATCH] Make allgather signature consistent for server_connect_fn() pmix_server_connect_fn() call grpcomm.allgather(), which uses a signature to identify the action. pmix_server_connect_fn() uses "procs" (which is an array of pmix_proc_t) as signature. procs was provided by client when it called PMIx_connect(). The issue is that different client may have procs in different order (when inter MPI communicator trascation is used). As a result, pmix servers may end up with different signature for the same allgather operation. This patch addressed the issue by sorting "procs" by pmix_proc_t before passing it to grpcomm.allgather. Signed-off-by: Wei Zhang --- src/prted/pmix/pmix_server_dyn.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/prted/pmix/pmix_server_dyn.c b/src/prted/pmix/pmix_server_dyn.c index 80198b3711..999d359f8f 100644 --- a/src/prted/pmix/pmix_server_dyn.c +++ b/src/prted/pmix/pmix_server_dyn.c @@ -1009,6 +1009,18 @@ static void connect_release(int status, pmix_data_buffer_t *buf, void *cbdata) PMIX_RELEASE(md); } +static int compare_pmix_proc(const void *a, const void *b) +{ + const pmix_proc_t *proc_a = (pmix_proc_t *)a; + const pmix_proc_t *proc_b = (pmix_proc_t *)b; + + int nspace_dif = strncmp(proc_a->nspace, proc_b->nspace, PMIX_MAX_NSLEN); + if (nspace_dif != 0) + return nspace_dif; + + return proc_a->rank - proc_b->rank; +} + static void _cnct(int sd, short args, void *cbdata) { prte_pmix_server_op_caddy_t *cd = (prte_pmix_server_op_caddy_t *) cbdata; @@ -1142,6 +1154,9 @@ static void _cnct(int sd, short args, void *cbdata) md->sig->sz = cd->nprocs; md->sig->signature = (pmix_proc_t *) malloc(md->sig->sz * sizeof(pmix_proc_t)); memcpy(md->sig->signature, cd->procs, md->sig->sz * sizeof(pmix_proc_t)); + /* different process may have the procs in different order, so sort md->sig->signature + * by pmix_proc_t to make it consistent accross processes */ + qsort(md->sig->signature, md->sig->sz, sizeof(pmix_proc_t), compare_pmix_proc); md->opcbfunc = cd->cbfunc; md->cbdata = cd->cbdata;