From 7851789f9835198b0dcc2550f5f1b6713146c6fc Mon Sep 17 00:00:00 2001 From: cindytsai Date: Wed, 21 Aug 2024 11:13:43 -0500 Subject: [PATCH 01/10] Too much SET_TIMER, and it makes time profile too large. --- src/append_grid.cpp | 6 ------ src/get_dtype_property.cpp | 14 -------------- 2 files changed, 20 deletions(-) diff --git a/src/append_grid.cpp b/src/append_grid.cpp index 41d0cfe5..67fdcce7 100644 --- a/src/append_grid.cpp +++ b/src/append_grid.cpp @@ -23,8 +23,6 @@ static int set_particle_data(yt_grid* grid); // Return : YT_SUCCESS or YT_FAIL //------------------------------------------------------------------------------------------------------- int append_grid(yt_grid* grid) { - SET_TIMER(__PRETTY_FUNCTION__); - // export grid info to libyt.hierarchy PyArrayObject* py_array_obj; @@ -76,8 +74,6 @@ int append_grid(yt_grid* grid) { // Return : YT_SUCCESS or YT_FAIL //------------------------------------------------------------------------------------------------------- static int set_field_data(yt_grid* grid) { - SET_TIMER(__PRETTY_FUNCTION__); - yt_field* field_list = LibytProcessControl::Get().field_list; PyObject *py_grid_id, *py_field_labels, *py_field_data; @@ -179,8 +175,6 @@ static int set_field_data(yt_grid* grid) { // Return : YT_SUCCESS or YT_FAIL //------------------------------------------------------------------------------------------------------- static int set_particle_data(yt_grid* grid) { - SET_TIMER(__PRETTY_FUNCTION__); - yt_particle* particle_list = LibytProcessControl::Get().particle_list; PyObject *py_grid_id, *py_ptype_labels, *py_attributes, *py_data; diff --git a/src/get_dtype_property.cpp b/src/get_dtype_property.cpp index a5735f7f..2fa6cf19 100644 --- a/src/get_dtype_property.cpp +++ b/src/get_dtype_property.cpp @@ -25,8 +25,6 @@ // Return : YT_SUCCESS or YT_FAIL //------------------------------------------------------------------------------------------------------- int get_npy_dtype(yt_dtype data_type, int* npy_dtype) { - SET_TIMER(__PRETTY_FUNCTION__); - switch (data_type) { case YT_FLOAT: *npy_dtype = NPY_FLOAT; @@ -103,8 +101,6 @@ int get_npy_dtype(yt_dtype data_type, int* npy_dtype) { // Return : YT_SUCCESS or YT_FAIL //------------------------------------------------------------------------------------------------------- int get_yt_dtype_from_npy(int npy_dtype, yt_dtype* data_dtype) { - SET_TIMER(__PRETTY_FUNCTION__); - switch (npy_dtype) { case NPY_FLOAT: *data_dtype = YT_FLOAT; @@ -173,8 +169,6 @@ int get_yt_dtype_from_npy(int npy_dtype, yt_dtype* data_dtype) { // Return : YT_SUCCESS or YT_FAIL //------------------------------------------------------------------------------------------------------- int get_mpi_dtype(yt_dtype data_type, MPI_Datatype* mpi_dtype) { - SET_TIMER(__PRETTY_FUNCTION__); - switch (data_type) { case YT_FLOAT: *mpi_dtype = MPI_FLOAT; @@ -258,8 +252,6 @@ int get_mpi_dtype(yt_dtype data_type, MPI_Datatype* mpi_dtype) { // Return : YT_SUCCESS or YT_FAIL //------------------------------------------------------------------------------------------------------- int get_dtype_size(yt_dtype data_type, int* dtype_size) { - SET_TIMER(__PRETTY_FUNCTION__); - switch (data_type) { case YT_FLOAT: *dtype_size = sizeof(float); @@ -343,8 +335,6 @@ int get_dtype_size(yt_dtype data_type, int* dtype_size) { // Return : YT_SUCCESS or YT_FAIL //------------------------------------------------------------------------------------------------------- int get_dtype_typeid(yt_dtype data_type, const std::type_info** dtype_id) { - SET_TIMER(__PRETTY_FUNCTION__); - switch (data_type) { case YT_FLOAT: *dtype_id = &typeid(float); @@ -430,8 +420,6 @@ int get_dtype_typeid(yt_dtype data_type, const std::type_info** dtype_id) { // Return : YT_SUCCESS or YT_FAIL //------------------------------------------------------------------------------------------------------- int get_dtype_allocation(yt_dtype data_type, unsigned long length, void** data_ptr) { - SET_TIMER(__PRETTY_FUNCTION__); - switch (data_type) { case YT_FLOAT: *data_ptr = malloc(length * sizeof(float)); @@ -532,8 +520,6 @@ int get_dtype_allocation(yt_dtype data_type, unsigned long length, void** data_p //------------------------------------------------------------------------------------------------------- int big_MPI_Get_dtype(void* recv_buff, long data_len, yt_dtype* data_dtype, MPI_Datatype* mpi_dtype, int get_rank, MPI_Aint base_address, MPI_Win* window) { - SET_TIMER(__PRETTY_FUNCTION__); - switch (*data_dtype) { case YT_FLOAT: return big_MPI_Get(recv_buff, data_len, mpi_dtype, get_rank, base_address, window); From ad2620fd079ceb15eaba402f0710ec21df48a4fb Mon Sep 17 00:00:00 2001 From: cindytsai Date: Fri, 23 Aug 2024 12:40:20 -0500 Subject: [PATCH 02/10] Add `opt_unit` and `mu` for upcoming yt-4.4 Adding these two user param won't affect yt-4.2. --- example/amr-example/example.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/example/amr-example/example.cpp b/example/amr-example/example.cpp index cc48e7ea..8d352a93 100644 --- a/example/amr-example/example.cpp +++ b/example/amr-example/example.cpp @@ -208,6 +208,10 @@ int main(int argc, char* argv[]) { yt_set_UserParameterInt("mhd", 1, &mhd); const int srhd = 0; yt_set_UserParameterInt("srhd", 1, &srhd); + const int opt_unit = 0; + yt_set_UserParameterInt("opt_unit", 1, &opt_unit); + const float mu = 0.6; + yt_set_UserParameterFloat("mu", 1, &mu); // demo of some other parameters we can add const int user_int = 1; From 9bac05b39753e6bfeae7b73424e224d9e1e66b91 Mon Sep 17 00:00:00 2001 From: cindytsai Date: Mon, 26 Aug 2024 12:10:35 -0500 Subject: [PATCH 03/10] Printing jedi info in the same line. --- src/libyt_kernel.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libyt_kernel.cpp b/src/libyt_kernel.cpp index 7fc41e27..194ced28 100644 --- a/src/libyt_kernel.cpp +++ b/src/libyt_kernel.cpp @@ -32,8 +32,8 @@ void LibytKernel::configure_impl() { PyObject* py_module_jedi = PyImport_ImportModule("jedi"); if (py_module_jedi == NULL) { - log_info("Unable to import jedi, jedi auto-completion library is disabled\n"); - log_info("See https://jedi.readthedocs.io/ \n"); + log_info( + "Unable to import jedi, jedi auto-completion library is disabled (See https://jedi.readthedocs.io/)\n"); m_py_jedi_interpreter = NULL; } else { m_py_jedi_interpreter = PyObject_GetAttrString(py_module_jedi, "Interpreter"); @@ -187,8 +187,8 @@ nl::json LibytKernel::complete_request_impl(const std::string& code, int cursor_ // Check if jedi has successfully import if (m_py_jedi_interpreter == NULL) { - log_info("Unable to import jedi, jedi auto-completion library is disabled\n"); - log_info("See https://jedi.readthedocs.io/ \n"); + log_info( + "Unable to import jedi, jedi auto-completion library is disabled (See https://jedi.readthedocs.io/)\n"); return xeus::create_complete_reply({}, cursor_pos, cursor_pos); } From 02635398719e8013e58688f68fda828923df9494 Mon Sep 17 00:00:00 2001 From: cindytsai Date: Mon, 26 Aug 2024 12:20:57 -0500 Subject: [PATCH 04/10] Clear the error buffer before redirecting stderr. I just noticed this bug if the env doesn't have jedi installed. --- src/libyt_python_shell.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/libyt_python_shell.cpp b/src/libyt_python_shell.cpp index d3ed426b..d5e03e74 100644 --- a/src/libyt_python_shell.cpp +++ b/src/libyt_python_shell.cpp @@ -469,8 +469,12 @@ CodeValidity LibytPythonShell::check_code_validity(const std::string& code, bool CodeValidity code_validity; - PyRun_SimpleString("import sys, io\n"); - PyRun_SimpleString("sys.OUTPUT_STDERR=''\nstderr_buf=io.StringIO()\nsys.stderr=stderr_buf\n"); + // clear error buffer before redirecting stderr + PyErr_Clear(); + PyRun_SimpleString("import sys, io"); + PyRun_SimpleString("sys.OUTPUT_STDERR=''"); + PyRun_SimpleString("stderr_buf=io.StringIO()"); + PyRun_SimpleString("sys.stderr=stderr_buf"); PyObject* py_test_compile; if (prompt_env) { From a6ebb3969d123855fb7ee198cc3b1501c4cc80a5 Mon Sep 17 00:00:00 2001 From: cindytsai Date: Mon, 26 Aug 2024 14:57:59 -0500 Subject: [PATCH 05/10] Update doc link for libyt kernel. --- src/libyt_kernel.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libyt_kernel.cpp b/src/libyt_kernel.cpp index 194ced28..e3320885 100644 --- a/src/libyt_kernel.cpp +++ b/src/libyt_kernel.cpp @@ -325,7 +325,7 @@ nl::json LibytKernel::kernel_info_request_impl() { // helper libyt_kernel_info["help_links"] = nl::json::array(); libyt_kernel_info["help_links"][0] = - nl::json::object({{"text", "libyt Kernel Documents"}, {"url", "https://yt-project.github.io/libyt/"}}); + nl::json::object({{"text", "libyt Kernel Documents"}, {"url", "https://libyt.readthedocs.io/en/latest/"}}); // status libyt_kernel_info["status"] = "ok"; From 6484a857f7c4729c109a8b9902893c5bc6e13c45 Mon Sep 17 00:00:00 2001 From: cindytsai Date: Mon, 26 Aug 2024 14:58:22 -0500 Subject: [PATCH 06/10] Clear error buffer before redirecting the error. --- src/libyt_python_shell.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libyt_python_shell.cpp b/src/libyt_python_shell.cpp index d5e03e74..9c8be378 100644 --- a/src/libyt_python_shell.cpp +++ b/src/libyt_python_shell.cpp @@ -565,6 +565,7 @@ std::array LibytPythonShell::execute_cell(const std: #endif // Clear the template buffer and redirect stdout, stderr + PyErr_Clear(); PyRun_SimpleString("import sys, io\n"); PyRun_SimpleString("sys.OUTPUT_STDOUT=''\nstdout_buf=io.StringIO()\nsys.stdout=stdout_buf\n"); PyRun_SimpleString("sys.OUTPUT_STDERR=''\nstderr_buf=io.StringIO()\nsys.stderr=stderr_buf\n"); From 0b45f692879533448f6ba313ee952405a933fe74 Mon Sep 17 00:00:00 2001 From: cindytsai Date: Mon, 26 Aug 2024 16:21:37 -0500 Subject: [PATCH 07/10] Change command from jupyter-lab to jupyter lab --- .../jupyter-notebook/jupyter-notebook-access.md | 2 +- doc/in-situ-python-analysis/jupyter-notebook/remote-cluster.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/in-situ-python-analysis/jupyter-notebook/jupyter-notebook-access.md b/doc/in-situ-python-analysis/jupyter-notebook/jupyter-notebook-access.md index 23311ba5..5b464559 100644 --- a/doc/in-situ-python-analysis/jupyter-notebook/jupyter-notebook-access.md +++ b/doc/in-situ-python-analysis/jupyter-notebook/jupyter-notebook-access.md @@ -39,7 +39,7 @@ We need another process to start Jupyter Notebook/JupyterLab and connect to liby 4. Launch Jupyter Notebook/JupyterLab ```bash jupyter notebook # launch Jupyter Notebook - jupyter-lab # launch JupyterLab + jupyter lab # launch JupyterLab ``` 5. Click `Libyt` to connect to libyt kernel once the simulation is running and libyt kernel is activated. diff --git a/doc/in-situ-python-analysis/jupyter-notebook/remote-cluster.md b/doc/in-situ-python-analysis/jupyter-notebook/remote-cluster.md index 27c11372..bd9924b0 100644 --- a/doc/in-situ-python-analysis/jupyter-notebook/remote-cluster.md +++ b/doc/in-situ-python-analysis/jupyter-notebook/remote-cluster.md @@ -47,7 +47,7 @@ Please go through [Login Node](#login-node) before going through [Local Laptop]( 2. Launch Jupyter Notebook / JupyterLab in no-browser mode with port `XXXX` and allowing access from other IP address (`--ip="0.0.0.0"`): ```bash jupyter notebook --no-browser --port=XXXX --ip="0.0.0.0" - jupyter-lab --no-browser --port=XXXX --ip="0.0.0.0" + jupyter lab --no-browser --port=XXXX --ip="0.0.0.0" ``` Change `XXXX` to some unused port on login node, this is just a placeholder. 3. You should see something like: From cb6a3f0f6bb8877f83e64f20a19ec250eaf0bad7 Mon Sep 17 00:00:00 2001 From: cindytsai Date: Tue, 27 Aug 2024 11:43:25 -0500 Subject: [PATCH 08/10] No need to change this. --- src/libyt_python_shell.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/libyt_python_shell.cpp b/src/libyt_python_shell.cpp index 9c8be378..73fc84b7 100644 --- a/src/libyt_python_shell.cpp +++ b/src/libyt_python_shell.cpp @@ -471,10 +471,8 @@ CodeValidity LibytPythonShell::check_code_validity(const std::string& code, bool // clear error buffer before redirecting stderr PyErr_Clear(); - PyRun_SimpleString("import sys, io"); - PyRun_SimpleString("sys.OUTPUT_STDERR=''"); - PyRun_SimpleString("stderr_buf=io.StringIO()"); - PyRun_SimpleString("sys.stderr=stderr_buf"); + PyRun_SimpleString("import sys, io\n"); + PyRun_SimpleString("sys.OUTPUT_STDERR=''\nstderr_buf=io.StringIO()\nsys.stderr=stderr_buf\n"); PyObject* py_test_compile; if (prompt_env) { From f4bb41453772930e4dd1b1099e1e8b854776fa01 Mon Sep 17 00:00:00 2001 From: cindytsai Date: Tue, 27 Aug 2024 16:59:57 -0500 Subject: [PATCH 09/10] This avoids parsing the wrong file to display the error. Py_CompileString(..., ..., Py_file_input) parses the file to display the traceback error. I don't know why if it cannot get the file, then it uses the inline script import. Thus, it is displaying the wrong traceback, because it is parsing the wrong file. To avoid that, I set the file name to , so when executing codes in Jupyter, it is displaying the error correctly. --- src/libyt_kernel.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libyt_kernel.cpp b/src/libyt_kernel.cpp index e3320885..96a507ce 100644 --- a/src/libyt_kernel.cpp +++ b/src/libyt_kernel.cpp @@ -68,7 +68,7 @@ nl::json LibytKernel::execute_request_impl(int execution_counter, const std::str bool store_history, nl::json user_expressions, bool allow_stdin) { SET_TIMER(__PRETTY_FUNCTION__); - std::string cell_name = std::string("In [") + std::to_string(execution_counter) + std::string("]"); + std::string cell_name = std::string(""); // Find if '%' is the first non-space character, if so, redirect jobs to define command std::size_t found = code.find_first_not_of("\t\n\v\f\r "); From b3ef39a33db4d5884bfd4554d623f86ea4e1217b Mon Sep 17 00:00:00 2001 From: cindytsai Date: Wed, 28 Aug 2024 16:21:37 -0500 Subject: [PATCH 10/10] Reset the state We need to free it because it resets the state of the Python functions tracked by libyt. The design of how API gets called is a bit weird. Will fix this in future update. --- example/quick-start/quick-start.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/example/quick-start/quick-start.cpp b/example/quick-start/quick-start.cpp index 81a7fd6d..05c77cba 100644 --- a/example/quick-start/quick-start.cpp +++ b/example/quick-start/quick-start.cpp @@ -43,15 +43,18 @@ int main(int argc, char* argv[]) { // Execute Python functions and activate Python entry points // ========================================================== if (yt_run_Function("print_hello_world") != YT_SUCCESS) { + fprintf(stderr, "ERROR: yt_run_Function failed!\n"); exit(EXIT_FAILURE); } if (yt_run_FunctionArguments("print_args", 3, "\'1\'", "2", "3.0") != YT_SUCCESS) { + fprintf(stderr, "ERROR: yt_run_FunctionArguments failed!\n"); exit(EXIT_FAILURE); } // Activate Python prompt if (yt_run_InteractiveMode("LIBYT_STOP") != YT_SUCCESS) { + fprintf(stderr, "ERROR: yt_run_InteractiveMode failed!\n"); exit(EXIT_FAILURE); } @@ -67,6 +70,12 @@ int main(int argc, char* argv[]) { exit(EXIT_FAILURE); } + // Free and reset the state + if (yt_free() != YT_SUCCESS) { + fprintf(stderr, "ERROR: yt_free() failed!\n"); + exit(EXIT_FAILURE); + } + // ======================================= // Finalize libyt // =======================================