From e31ffc513f478d56e03bf148b02f3f2d8468ac77 Mon Sep 17 00:00:00 2001 From: Aurelien Bouteiller Date: Mon, 22 Jan 2024 15:55:51 -0500 Subject: [PATCH] bugfix: cuda handles were not consistently initialized when gpus where initialized/not initialized differently than the -g param Signed-off-by: Aurelien Bouteiller --- src/dplasmaaux_cuda.c | 9 +++++++++ src/dplasmaaux_cuda.h | 1 + src/dtd_wrappers/zgemm.c | 1 + tests/common.c | 20 +++++--------------- tests/testing_zgemm_dtd.c | 15 --------------- tests/testing_zpotrf_dtd.c | 4 ---- 6 files changed, 16 insertions(+), 34 deletions(-) diff --git a/src/dplasmaaux_cuda.c b/src/dplasmaaux_cuda.c index e90cf82a..e4c0ceb8 100644 --- a/src/dplasmaaux_cuda.c +++ b/src/dplasmaaux_cuda.c @@ -97,3 +97,12 @@ void *dplasma_create_cuda_handles(void *obj, void *_n) return new; } + +void dplasma_destroy_cuda_handles(void *_h, void *_n) +{ + dplasma_cuda_handles_t *handles = (dplasma_cuda_handles_t*)_h; + (void)_n; + cublasDestroy_v2(handles->cublas_handle); + cusolverDnDestroy(handles->cusolverDn_handle); + free(handles); +} diff --git a/src/dplasmaaux_cuda.h b/src/dplasmaaux_cuda.h index f7e57ed2..aa1360ff 100644 --- a/src/dplasmaaux_cuda.h +++ b/src/dplasmaaux_cuda.h @@ -66,6 +66,7 @@ typedef struct { } dplasma_cuda_handles_t; void *dplasma_create_cuda_handles(void *obj, void *user); +void dplasma_destroy_cuda_handles(void *_h, void *_n); #define DPLASMA_CUBLAS_CHECK_STATUS( STR, STATUS, CODE ) \ do { \ diff --git a/src/dtd_wrappers/zgemm.c b/src/dtd_wrappers/zgemm.c index 80361892..9b39354d 100644 --- a/src/dtd_wrappers/zgemm.c +++ b/src/dtd_wrappers/zgemm.c @@ -81,6 +81,7 @@ parsec_core_zgemm_cuda(parsec_device_gpu_module_t* gpu_device, #endif /* defined(PARSEC_DEBUG_NOISIER) */ handles = parsec_info_get(&gpu_stream->infos, CuHI); + assert(NULL != handles); parsec_cuda_exec_stream_t* cuda_stream = (parsec_cuda_exec_stream_t*)gpu_stream; cublasSetStream( handles->cublas_handle, cuda_stream->cuda_stream ); diff --git a/tests/common.c b/tests/common.c index 78a3d865..3b7c0fa9 100644 --- a/tests/common.c +++ b/tests/common.c @@ -626,17 +626,6 @@ char cwd[1024]; int unix_timestamp; #endif -#if defined(DPLASMA_HAVE_CUDA) -static void destroy_cuda_handles(void *_h, void *_n) -{ - dplasma_cuda_handles_t *handles = (dplasma_cuda_handles_t*)_h; - (void)_n; - cublasDestroy_v2(handles->cublas_handle); - cusolverDnDestroy(handles->cusolverDn_handle); - free(handles); -} -#endif - parsec_context_t* setup_parsec(int argc, char **argv, int *iparam) { #ifdef PARSEC_PROF_TRACE @@ -716,10 +705,11 @@ parsec_context_t* setup_parsec(int argc, char **argv, int *iparam) } } if( nbgpu > 0 ) { - parsec_info_register(&parsec_per_stream_infos, "DPLASMA::CUDA::HANDLES", - destroy_cuda_handles, NULL, - dplasma_create_cuda_handles, NULL, - NULL); + CuHI = parsec_info_register(&parsec_per_stream_infos, "DPLASMA::CUDA::HANDLES", + dplasma_destroy_cuda_handles, NULL, + dplasma_create_cuda_handles, NULL, + NULL); + assert(-1 != CuHI); } #endif diff --git a/tests/testing_zgemm_dtd.c b/tests/testing_zgemm_dtd.c index fb11a847..3a02a216 100644 --- a/tests/testing_zgemm_dtd.c +++ b/tests/testing_zgemm_dtd.c @@ -123,13 +123,6 @@ int main(int argc, char ** argv) parsec_context_start(parsec); parsec_task_class_t *zgemm_tc = parsec_dtd_create_zgemm_task_class(dtd_tp, TILE_FULL, PARSEC_DEV_ALL); -#if defined(DPLASMA_HAVE_CUDA) - if( gpus > 0 ){ - CuHI = parsec_info_lookup(&parsec_per_stream_infos, "DPLASMA::CUDA::HANDLES", NULL); - assert( CuHI != -1); - } -#endif /* defined(DPLASMA_HAVE_CUDA) */ - for( m = 0; m < dcC.super.mt; m++ ) { tempmm = m == dcC.super.mt-1 ? dcC.super.m-m*dcC.super.mb : dcC.super.mb; ldcm = BLKLDD(&dcC.super, m); @@ -370,14 +363,6 @@ int main(int argc, char ** argv) parsec_context_start(parsec); parsec_task_class_t *zgemm_tc = parsec_dtd_create_zgemm_task_class(dtd_tp, TILE_FULL, PARSEC_DEV_ALL); -#if defined(DPLASMA_HAVE_CUDA) - if( gpus > 0 ){ - CuHI = parsec_info_lookup(&parsec_per_stream_infos, "DPLASMA::CUDA::HANDLES", NULL); - assert( CuHI != -1); - } -#endif - - for( m = 0; m < dcC.super.mt; m++ ) { tempmm = m == dcC.super.mt-1 ? dcC.super.m-m*dcC.super.mb : dcC.super.mb; ldcm = BLKLDD(&dcC.super, m); diff --git a/tests/testing_zpotrf_dtd.c b/tests/testing_zpotrf_dtd.c index 423f1cf8..a94325ef 100644 --- a/tests/testing_zpotrf_dtd.c +++ b/tests/testing_zpotrf_dtd.c @@ -96,10 +96,6 @@ int main(int argc, char **argv) #if defined(DPLASMA_HAVE_CUDA) zpotrf_dtd_workspace_info_t* infos = (zpotrf_dtd_workspace_info_t*) malloc(sizeof(zpotrf_dtd_workspace_info_t)); - if( gpus > 0 ){ - CuHI = parsec_info_lookup(&parsec_per_stream_infos, "DPLASMA::CUDA::HANDLES", NULL); - assert(CuHI != -1); - } #endif if( dplasmaLower == uplo ) {