Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

improve ULT stack size configuration #340

Merged
merged 3 commits into from
Jun 29, 2021
Merged

improve ULT stack size configuration #340

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3196bbf
configure: make the default ULT stack size configurable
shintaro-iwasaki Jun 29, 2021
bb26724
info: make more information available before ABT_init
shintaro-iwasaki Jun 29, 2021
e44af9a
test/basic: add a new test to check ABT_info_query_config
shintaro-iwasaki Jun 29, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 14 additions & 8 deletions configure.ac
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,13 @@ AC_ARG_ENABLE([static-cacheline-size],
<value> - assume [value] bytes (e.g., <value> = 64)
],,[enable_static_cacheline_size=auto])

# --enable-default-stacksize
AC_ARG_ENABLE([default-stacksize],
[ --enable-default-stacksize@<:@=SIZE@:>@
set a default ULT stack size in bytes (default: 16384)
ABT_THREAD_STACKSIZE=SIZE at run-time supersedes this setting.
],,[enable_default_stacksize=16384])

# --enable-ver20-api
AC_ARG_ENABLE([ver20-api],
AS_HELP_STRING([--enable-ver20-api],
Expand All @@ -247,14 +254,6 @@ AC_ARG_WITH([libunwind],
AS_HELP_STRING([--with-libunwind=PATH],
[specify path where libunwind include directory and lib directory can be found]))

# --enable-stack-unwind
AC_ARG_ENABLE([stack-unwind],
[ --enable-stack-unwind@<:@=OPTS@:>@ enable stack unwinding, which is disabled by default.
yes|verbose - enable stack unwinding. Dump the raw stack information too
unwind-only - enable stack unwinding. Do not dump the raw stack information
no|none - disable stack unwinding
],,[enable_stack_unwind=no])

# --with-papi
AC_ARG_WITH([papi],
AS_HELP_STRING([--with-papi=PATH],
Expand Down Expand Up @@ -816,6 +815,13 @@ fi
AC_DEFINE_UNQUOTED(ABT_CONFIG_STATIC_CACHELINE_SIZE, [$static_cacheline_size],
[Define to use static cache-line size])

# --enable-default-stacksize
if test x"`echo $enable_default_stacksize | sed -e 's/[[0-9]]//g'`" != x"" ; then
# enable_default_stacksize is not numeric
AC_MSG_WARN([Unknown value $enable_default_stacksize for --enable-default-stacksize])
enable_default_stacksize=16384
fi
AC_DEFINE_UNQUOTED(ABT_CONFIG_DEFAULT_THREAD_STACKSIZE, [$enable_default_stacksize], [Define to set the default ULT stack size])

# --with-lts
if test "x$with_lts" != "x"; then
Expand Down
262 changes: 164 additions & 98 deletions src/arch/abtd_env.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
#include <strings.h>

#define ABTD_KEY_TABLE_DEFAULT_SIZE 4
#define ABTD_THREAD_DEFAULT_STACKSIZE 16384
#define ABTD_SCHED_DEFAULT_STACKSIZE (4 * 1024 * 1024)
#define ABTD_SCHED_EVENT_FREQ 50
#define ABTD_SCHED_SLEEP_NSEC 100
Expand Down Expand Up @@ -60,107 +59,36 @@ void ABTD_env_init(ABTI_global *p_global)
ABTD_affinity_init(p_global, env);
}

#ifdef ABT_CONFIG_USE_DEBUG_LOG_PRINT
/* If the debug log printing is set in configure, logging is turned on by
* default. */
const ABT_bool default_use_logging = ABT_TRUE;
const ABT_bool default_use_debug = ABT_TRUE;
#else
/* Otherwise, logging is not turned on by default. */
const ABT_bool default_use_logging = ABT_FALSE;
const ABT_bool default_use_debug = ABT_FALSE;
#endif
/* ABT_USE_LOG, ABT_ENV_USE_LOG */
p_global->use_logging = load_env_bool("USE_LOG", default_use_logging);

/* ABT_USE_DEBUG, ABT_ENV_USE_DEBUG */
p_global->use_debug = load_env_bool("USE_DEBUG", default_use_debug);

/* ABT_MAX_NUM_XSTREAMS, ABT_ENV_MAX_NUM_XSTREAMS
* Maximum size of the internal ES array */
p_global->max_xstreams =
load_env_int("MAX_NUM_XSTREAMS", p_global->num_cores, 1,
ABTD_ENV_INT_MAX);

/* ABT_KEY_TABLE_SIZE, ABT_ENV_KEY_TABLE_SIZE
* Default key table size */
p_global->key_table_size = roundup_pow2_uint32(
load_env_uint32("KEY_TABLE_SIZE", ABTD_KEY_TABLE_DEFAULT_SIZE, 1,
ABTD_ENV_UINT32_MAX));

/* ABT_STACK_OVERFLOW_CHECK, ABT_ENV_STACK_OVERFLOW_CHECK */
env = get_abt_env("STACK_OVERFLOW_CHECK");
if (env) {
if (strcasecmp(env, "mprotect_strict") == 0) {
/* Log setting */
p_global->use_logging = ABTD_env_get_use_logging();
/* Debug setting (unused) */
p_global->use_debug = ABTD_env_get_use_debug();
/* Maximum size of the internal ES array */
p_global->max_xstreams = ABTD_env_get_max_xstreams();
/* Default key table size */
p_global->key_table_size = ABTD_env_key_table_size();
/* mprotect-based stack guard setting */
ABT_bool is_strict;
if (ABTD_env_get_stack_guard_mprotect(&is_strict)) {
if (is_strict) {
p_global->stack_guard_kind = ABTI_STACK_GUARD_MPROTECT_STRICT;
} else if (strcasecmp(env, "mprotect") == 0) {
p_global->stack_guard_kind = ABTI_STACK_GUARD_MPROTECT;
} else {
/* Otherwise, disable mprotect-based stack guard. */
p_global->stack_guard_kind = ABTI_STACK_GUARD_NONE;
p_global->stack_guard_kind = ABTI_STACK_GUARD_MPROTECT;
}
} else {
/* Set the default mode. */
#if ABT_CONFIG_STACK_CHECK_TYPE == ABTI_STACK_CHECK_TYPE_MPROTECT
p_global->stack_guard_kind = ABTI_STACK_GUARD_MPROTECT;
#elif ABT_CONFIG_STACK_CHECK_TYPE == ABTI_STACK_CHECK_TYPE_MPROTECT_STRICT
p_global->stack_guard_kind = ABTI_STACK_GUARD_MPROTECT_STRICT;
#else
/* Stack canary is compile-time setting. */
p_global->stack_guard_kind = ABTI_STACK_GUARD_NONE;
#endif
}

/* ABT_SYS_PAGE_SIZE, ABT_ENV_SYS_PAGE_SIZE
* System page size. It must be 2^N. */
size_t sys_page_size = ABTD_SYS_PAGE_SIZE;
#if HAVE_GETPAGESIZE
sys_page_size = getpagesize();
#endif
p_global->sys_page_size = roundup_pow2_size(
load_env_size("SYS_PAGE_SIZE", sys_page_size, 64, ABTD_ENV_SIZE_MAX));

/* ABT_THREAD_STACKSIZE, ABT_ENV_THREAD_STACKSIZE
* Default stack size for ULT */
size_t default_thread_stacksize = ABTD_THREAD_DEFAULT_STACKSIZE;
if (p_global->stack_guard_kind == ABTI_STACK_GUARD_MPROTECT ||
p_global->stack_guard_kind == ABTI_STACK_GUARD_MPROTECT_STRICT) {
/* Maximum 2 pages are used for mprotect(), so let's increase the
* default stack size. */
default_thread_stacksize += p_global->sys_page_size * 2;
}
p_global->thread_stacksize =
ABTU_roundup_size(load_env_size("THREAD_STACKSIZE",
default_thread_stacksize, 512,
ABTD_ENV_SIZE_MAX),
ABT_CONFIG_STATIC_CACHELINE_SIZE);

/* ABT_SCHED_STACKSIZE, ABT_ENV_SCHED_STACKSIZE
* Default stack size for scheduler */
size_t default_sched_stacksize = ABTD_SCHED_DEFAULT_STACKSIZE;
if (p_global->stack_guard_kind == ABTI_STACK_GUARD_MPROTECT ||
p_global->stack_guard_kind == ABTI_STACK_GUARD_MPROTECT_STRICT) {
/* Maximum 2 pages are used for mprotect(), so let's increase the
* default stack size. */
default_sched_stacksize += p_global->sys_page_size * 2;
}
p_global->sched_stacksize =
ABTU_roundup_size(load_env_size("SCHED_STACKSIZE",
default_sched_stacksize, 512,
ABTD_ENV_SIZE_MAX),
ABT_CONFIG_STATIC_CACHELINE_SIZE);

/* ABT_SCHED_EVENT_FREQ, ABT_ENV_SCHED_EVENT_FREQ
* Default frequency for event checking by the scheduler */
p_global->sched_event_freq =
load_env_uint32("SCHED_EVENT_FREQ", ABTD_SCHED_EVENT_FREQ, 1,
ABTD_ENV_UINT32_MAX);

/* ABT_SCHED_SLEEP_NSEC, ABT_ENV_SCHED_SLEEP_NSEC
* Default nanoseconds for scheduler sleep */
p_global->sched_sleep_nsec =
load_env_uint64("SCHED_SLEEP_NSEC", ABTD_SCHED_SLEEP_NSEC, 0,
ABTD_ENV_UINT64_MAX);
/* System page size. */
p_global->sys_page_size = ABTD_env_get_sys_pagesize();
/* Default stack size for ULT */
p_global->thread_stacksize = ABTD_env_get_thread_stacksize();
/* Default stack size for scheduler */
p_global->sched_stacksize = ABTD_env_get_sched_stacksize();
/* Default frequency for event checking by the scheduler */
p_global->sched_event_freq = ABTD_env_get_sched_event_freq();
/* Default nanoseconds for scheduler sleep */
p_global->sched_sleep_nsec = ABTD_env_get_sched_sleep_nsec();

/* ABT_MUTEX_MAX_HANDOVERS, ABT_ENV_MUTEX_MAX_HANDOVERS
* Default maximum number of mutex handover */
Expand Down Expand Up @@ -292,14 +220,152 @@ void ABTD_env_init(ABTI_global *p_global)
}
#endif

/* ABT_PRINT_CONFIG, ABT_ENV_PRINT_CONFIG
* Whether to print the configuration on ABT_init() */
p_global->print_config = load_env_bool("PRINT_CONFIG", ABT_FALSE);
/* Whether to print the configuration on ABT_init() */
p_global->print_config = ABTD_env_get_print_config();

/* Init timer */
ABTD_time_init();
}

ABT_bool ABTD_env_get_use_debug(void)
{
#ifdef ABT_CONFIG_USE_DEBUG_LOG_PRINT
const ABT_bool default_use_debug = ABT_TRUE;
#else
const ABT_bool default_use_debug = ABT_FALSE;
#endif
/* ABT_USE_DEBUG, ABT_ENV_USE_DEBUG */
return load_env_bool("USE_DEBUG", default_use_debug);
}

ABT_bool ABTD_env_get_use_logging(void)
{
#ifdef ABT_CONFIG_USE_DEBUG_LOG_PRINT
/* If the debug log printing is set in configure, logging is turned on by
* default. */
const ABT_bool default_use_logging = ABT_TRUE;
#else
/* Otherwise, logging is not turned on by default. */
const ABT_bool default_use_logging = ABT_FALSE;
#endif
/* ABT_USE_LOG, ABT_ENV_USE_LOG */
return load_env_bool("USE_LOG", default_use_logging);
}

ABT_bool ABTD_env_get_print_config(void)
{
/* ABT_PRINT_CONFIG, ABT_ENV_PRINT_CONFIG */
return load_env_bool("PRINT_CONFIG", ABT_FALSE);
}

int ABTD_env_get_max_xstreams(void)
{
const int num_cores = sysconf(_SC_NPROCESSORS_ONLN);
/* ABT_MAX_NUM_XSTREAMS, ABT_ENV_MAX_NUM_XSTREAMS */
return load_env_int("MAX_NUM_XSTREAMS", num_cores, 1, ABTD_ENV_INT_MAX);
}

uint32_t ABTD_env_key_table_size(void)
{
/* ABT_KEY_TABLE_SIZE, ABT_ENV_KEY_TABLE_SIZE */
return roundup_pow2_uint32(load_env_uint32("KEY_TABLE_SIZE",
ABTD_KEY_TABLE_DEFAULT_SIZE, 1,
ABTD_ENV_UINT32_MAX));
}

size_t ABTD_env_get_sys_pagesize(void)
{
/* ABT_SYS_PAGE_SIZE, ABT_ENV_SYS_PAGE_SIZE
* System page size. It must be 2^N. */
size_t sys_page_size = ABTD_SYS_PAGE_SIZE;
#if HAVE_GETPAGESIZE
sys_page_size = getpagesize();
#endif
return roundup_pow2_size(
load_env_size("SYS_PAGE_SIZE", sys_page_size, 64, ABTD_ENV_SIZE_MAX));
}

size_t ABTD_env_get_thread_stacksize(void)
{
size_t default_thread_stacksize = ABT_CONFIG_DEFAULT_THREAD_STACKSIZE;
if (ABTD_env_get_stack_guard_mprotect(NULL)) {
/* Maximum 2 pages are used for mprotect(), so let's increase the
* default stack size. */
const size_t sys_page_size = ABTD_env_get_sys_pagesize();
default_thread_stacksize += sys_page_size * 2;
}
/* ABT_THREAD_STACKSIZE, ABT_ENV_THREAD_STACKSIZE */
return ABTU_roundup_size(load_env_size("THREAD_STACKSIZE",
default_thread_stacksize, 512,
ABTD_ENV_SIZE_MAX),
ABT_CONFIG_STATIC_CACHELINE_SIZE);
}

size_t ABTD_env_get_sched_stacksize(void)
{
size_t default_sched_stacksize = ABTD_SCHED_DEFAULT_STACKSIZE;
if (ABTD_env_get_stack_guard_mprotect(NULL)) {
/* Maximum 2 pages are used for mprotect(), so let's increase the
* default stack size. */
const size_t sys_page_size = ABTD_env_get_sys_pagesize();
default_sched_stacksize += sys_page_size * 2;
}
/* ABT_SCHED_STACKSIZE, ABT_ENV_SCHED_STACKSIZE */
return ABTU_roundup_size(load_env_size("SCHED_STACKSIZE",
default_sched_stacksize, 512,
ABTD_ENV_SIZE_MAX),
ABT_CONFIG_STATIC_CACHELINE_SIZE);
}

uint32_t ABTD_env_get_sched_event_freq(void)
{
/* ABT_SCHED_EVENT_FREQ, ABT_ENV_SCHED_EVENT_FREQ */
return load_env_uint32("SCHED_EVENT_FREQ", ABTD_SCHED_EVENT_FREQ, 1,
ABTD_ENV_UINT32_MAX);
}

uint64_t ABTD_env_get_sched_sleep_nsec(void)
{
/* ABT_SCHED_SLEEP_NSEC, ABT_ENV_SCHED_SLEEP_NSEC */
return load_env_uint64("SCHED_SLEEP_NSEC", ABTD_SCHED_SLEEP_NSEC, 0,
ABTD_ENV_UINT64_MAX);
}

ABT_bool ABTD_env_get_stack_guard_mprotect(ABT_bool *is_strict)
{
/* ABT_STACK_OVERFLOW_CHECK, ABT_ENV_STACK_OVERFLOW_CHECK */
const char *env = get_abt_env("STACK_OVERFLOW_CHECK");
ABT_bool strict_val, mprotect_val;
if (env) {
if (strcasecmp(env, "mprotect_strict") == 0) {
strict_val = ABT_TRUE;
mprotect_val = ABT_TRUE;
} else if (strcasecmp(env, "mprotect") == 0) {
strict_val = ABT_FALSE;
mprotect_val = ABT_TRUE;
} else {
/* Otherwise, disable mprotect-based stack guard. */
strict_val = ABT_FALSE;
mprotect_val = ABT_FALSE;
}
} else {
/* Set the default mode. */
#if ABT_CONFIG_STACK_CHECK_TYPE == ABTI_STACK_CHECK_TYPE_MPROTECT
strict_val = ABT_FALSE;
mprotect_val = ABT_TRUE;
#elif ABT_CONFIG_STACK_CHECK_TYPE == ABTI_STACK_CHECK_TYPE_MPROTECT_STRICT
strict_val = ABT_TRUE;
mprotect_val = ABT_TRUE;
#else
strict_val = ABT_FALSE;
mprotect_val = ABT_FALSE;
#endif
}
if (is_strict)
*is_strict = strict_val;
return mprotect_val;
}

/*****************************************************************************/
/* Internal static functions */
/*****************************************************************************/
Expand Down
12 changes: 12 additions & 0 deletions src/include/abtd.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,18 @@ typedef struct ABTD_affinity_cpuset {

/* Environment */
void ABTD_env_init(ABTI_global *p_global);
/* Following does not need p_global. */
ABT_bool ABTD_env_get_use_debug(void);
ABT_bool ABTD_env_get_use_logging(void);
ABT_bool ABTD_env_get_print_config(void);
int ABTD_env_get_max_xstreams(void);
uint32_t ABTD_env_key_table_size(void);
size_t ABTD_env_get_sys_pagesize(void);
size_t ABTD_env_get_thread_stacksize(void);
size_t ABTD_env_get_sched_stacksize(void);
uint32_t ABTD_env_get_sched_event_freq(void);
uint64_t ABTD_env_get_sched_sleep_nsec(void);
ABT_bool ABTD_env_get_stack_guard_mprotect(ABT_bool *is_strict);

/* ES Context */
ABTU_ret_err int ABTD_xstream_context_create(void *(*f_xstream)(void *),
Expand Down
Loading