Improve tutorials and CI/CD stability (#5005)

Closes #5004, closes #5006, closes #4998, closes #4997, closes #4996 Description of changes: - fix CI/CD pipeline issues - fix flaky Drude sample test (sample time step was too large) - select mpiexec flags based on the MPI vendor and Docker container - upgrade CPU and GPU resource allocation based on newly available hardware - fix regression in tools used to process tutorials during the summer school - solution Markdown cells can now be reversed back to solution code cells - improve tutorials based on summer school feedback
espressomd · Oct 18, 2024 · f73f0f5 · f73f0f5
2 parents 03033c4 + 386dfb4
commit f73f0f5
Show file tree

Hide file tree

Showing 17 changed files with 112 additions and 147 deletions.
diff --git a/.github/workflows/push_pull.yml b/.github/workflows/push_pull.yml
@@ -39,7 +39,7 @@ jobs:
       credentials:
          username: ${{ github.actor }}
          password: ${{ secrets.github_token }}
-      options: --cpus 2
+      options: --cpus 4
     steps:
       - name: Checkout
         uses: actions/checkout@main
@@ -55,8 +55,8 @@ jobs:
       - name: Build and check
         uses: ./.github/actions/build_and_check
         env:
-          build_procs: 2
-          check_procs: 2
+          build_procs: 4
+          check_procs: 4
           myconfig: 'maxset'
           with_ccache: 'true'
           with_cuda: 'false'
@@ -78,7 +78,7 @@ jobs:
       credentials:
          username: ${{ github.actor }}
          password: ${{ secrets.github_token }}
-      options: --cpus 2
+      options: --cpus 4
     steps:
       - name: Checkout
         uses: actions/checkout@main
@@ -94,8 +94,8 @@ jobs:
       - name: Build and check
         uses: ./.github/actions/build_and_check
         env:
-          build_procs: 2
-          check_procs: 2
+          build_procs: 4
+          check_procs: 4
           myconfig: 'maxset'
           with_ccache: 'true'
           with_cuda: 'false'

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -372,13 +372,16 @@ else()
 endif()
 
 # OpenMPI checks the number of processes against the number of CPUs
-set(ESPRESSO_MPIEXEC_OVERSUBSCRIBE "")
+set(ESPRESSO_MPIEXEC_PREFLAGS "")
 # Open MPI 4.x has a bug on NUMA archs that prevents running in singleton mode
 set(ESPRESSO_MPIEXEC_GUARD_SINGLETON_NUMA OFF)
 set(ESPRESSO_CPU_MODEL_NAME_OMPI_SINGLETON_NUMA_PATTERN "AMD (EPYC|Ryzen)")
 
 if("${ESPRESSO_MPIEXEC_VENDOR}" STREQUAL "OpenMPI")
-  set(ESPRESSO_MPIEXEC_OVERSUBSCRIBE "-oversubscribe")
+  list(APPEND ESPRESSO_MPIEXEC_PREFLAGS "--oversubscribe")
+  if(ESPRESSO_INSIDE_DOCKER)
+    list(APPEND ESPRESSO_MPIEXEC_PREFLAGS "--bind-to" "none")
+  endif()
   if(${ESPRESSO_MPIEXEC_VERSION} VERSION_LESS 5.0)
     if(NOT DEFINED ESPRESSO_CPU_MODEL_NAME)
       if(CMAKE_SYSTEM_NAME STREQUAL Linux)

diff --git a/Readme.md b/Readme.md
@@ -1,9 +1,3 @@
-# Invitation to the ESPResSo Summer School 2024
-
-[![CECAM Flagship School registration link](https://img.shields.io/badge/CECAM%20Flagship%20School-Register%20Now-blue?style=for-the-badge)](https://www.cecam.org/workshop-details/1324)
-
-The summer school "Simulating soft matter across scales" will take place on October 7-11, 2024, in Stuttgart. Registration is now open on [CECAM](https://www.cecam.org/workshop-details/1324).
-
 # ESPResSo
 
 [![GitLab CI](https://gitlab.icp.uni-stuttgart.de/espressomd/espresso/badges/python/pipeline.svg)](https://gitlab.icp.uni-stuttgart.de/espressomd/espresso/-/commits/python)

diff --git a/cmake/espresso_unit_test.cmake b/cmake/espresso_unit_test.cmake
@@ -52,7 +52,7 @@ function(ESPRESSO_UNIT_TEST)
       set(TEST_NUM_PROC ${ESPRESSO_TEST_NP})
     endif()
     espresso_set_mpiexec_tmpdir(${TEST_NAME})
-    add_test(${TEST_NAME} ${MPIEXEC} ${ESPRESSO_MPIEXEC_OVERSUBSCRIBE}
+    add_test(${TEST_NAME} ${MPIEXEC} ${ESPRESSO_MPIEXEC_PREFLAGS}
              ${MPIEXEC_NUMPROC_FLAG} ${TEST_NUM_PROC} ${MPIEXEC_PREFLAGS}
              ${ESPRESSO_MPIEXEC_TMPDIR} ${CMAKE_CURRENT_BINARY_DIR}/${TEST_NAME}
              ${MPIEXEC_POSTFLAGS})

diff --git a/doc/bibliography.bib b/doc/bibliography.bib
@@ -688,13 +688,13 @@ @Book{kruger17a
   address   = {Cham},
 }
 
-@Book{kundu01a,
+@Book{kundu02a,
 author    = {Kundu, Pijush K. and Cohen, Ira M. and Hu, Howard},
 title     = {Fluid Mechanics},
-year      = {2001},
+year      = {2002},
 edition   = {2nd},
 publisher = {Elsevier Science},
-isbn      = {9780080545585},
+isbn      = {978-0-12-178251-1},
 doi       = {10.1016/C2009-0-20716-1},
 }
 

diff --git a/doc/tutorials/Readme.md b/doc/tutorials/Readme.md
@@ -70,7 +70,7 @@ physical systems.
   Measuring the excess chemical potential of a salt solution using the Widom particle insertion method.  
   [Guide](widom_insertion/widom_insertion.ipynb)
 * **Grand-Canonical Monte Carlo**
-  Simulating a polyelectrolyte solution coupled to a reservoir of salt.
+  Simulating a polyelectrolyte solution coupled to a reservoir of salt.  
   [Guide](grand_canonical_monte_carlo/grand_canonical_monte_carlo.ipynb)
 
 [comment]: # (End of tutorials landing page)

diff --git a/doc/tutorials/convert.py b/doc/tutorials/convert.py
@@ -75,7 +75,7 @@ def add_cell_from_script(nb, filepath):
 
 
 def remove_empty_cells(nb):
-    for i in range(len(nb['cells']) - 1, 0, -1):
+    for i in range(len(nb["cells"]))[::-1]:
         cell = nb['cells'][i]
         if cell['source'].strip() == '':
             nb['cells'].pop(i)
@@ -90,11 +90,19 @@ def parse_solution_cell(cell):
 
 
 def convert_exercise2_to_code(nb):
-    for i in range(len(nb["cells"]) - 1, 0, -1):
+    for i in range(len(nb["cells"]))[::-1]:
         cell = nb["cells"][i]
-        solution = parse_solution_cell(cell)
-        if solution is not None:
-            cell["source"] = solution
+        if cell["cell_type"] != "markdown":
+            continue
+        source = cell["source"]
+        if source.startswith("<details") and source.endswith("</details>"):
+            m = re.search("```python\n(.+)\n```", source, flags=re.DOTALL)
+            solution = "# SOLUTION CELL\n" + m.group(1)
+            nb["cells"][i] = nbformat.v4.new_code_cell(source=solution)
+            if (i + 1) != len(nb["cells"]) and \
+                    nb["cells"][i + 1]["cell_type"] == "code" and \
+                    not nb["cells"][i + 1]["source"]:
+                del nb["cells"][i + 1]
 
 
 def disable_plot_interactivity(nb):

diff --git a/doc/tutorials/electrokinetics/electrokinetics.ipynb b/doc/tutorials/electrokinetics/electrokinetics.ipynb
@@ -52,7 +52,7 @@
    "id": "192c4793",
    "metadata": {},
    "source": [
-    "The first system that is simulated in this tutorial is the simple advection-diffusion of a drop of uncharged chemical species in a constant velocity field. To keep the computation time small, we restrict ourselves to a 2D problem, but the algorithm is also capable of solving the 3D advection-diffusion equation. Furthermore, we can also skip solving the electrostatic Poisson equation, since there are is no charged species present. The equations we solve thus reduce to\n",
+    "The first system that is simulated in this tutorial is the simple advection-diffusion of a drop of uncharged chemical species in a constant velocity field. To keep the computation time small, we restrict ourselves to a 2D problem, but the algorithm is also capable of solving the 3D advection-diffusion equation. Furthermore, we can also skip solving the electrostatic Poisson equation, since there are no charged species present. The equations we solve thus reduce to\n",
     "\n",
     "\\begin{equation}\n",
     "\\partial_{t} n = D \\Delta n - \\vec{\\nabla} \\cdot (\\vec{v} n).\n",
@@ -187,7 +187,7 @@
    "metadata": {},
    "source": [
     "# Exercise:\n",
-    "- Create an instance of the [`espressomd.electrokinetics.EKSpecies`]() and add it to the system with [`system.ekcontainer.add()`](https://espressomd.github.io/doc/espressomd.html#espressomd.electrokinetics.EKContainer.add). \n",
+    "- Create an instance of the [`espressomd.electrokinetics.EKSpecies`](https://espressomd.github.io/doc/espressomd.html#espressomd.electrokinetics.EKSpecies) and add it to the system with [`system.ekcontainer.add()`](https://espressomd.github.io/doc/espressomd.html#espressomd.electrokinetics.EKContainer.add). \n",
     "\n",
     "# Hint:\n",
     "- Use the variables `DIFFUSION_COEFFICIENT`, `KT` and `TAU` defined above.\n",

diff --git a/doc/tutorials/lattice_boltzmann/lattice_boltzmann_theory.ipynb b/doc/tutorials/lattice_boltzmann/lattice_boltzmann_theory.ipynb
@@ -68,7 +68,7 @@
     "The LBM essentially consists of solving a fully discretized version of the linearized\n",
     "Boltzmann equation. The Boltzmann equation describes the time evolution of the one\n",
     "particle distribution function $f \\left(\\vec{x}, \\vec{p}, t\\right)$, which is the probability of finding a molecule in\n",
-    "a phase space volume $d^3\\vec{x}\\,d^3\\vec{p}$ at time $t$.The function $f$ is normalized so that the integral\n",
+    "a phase space volume $d^3\\vec{x}\\,d^3\\vec{p}$ at time $t$. The function $f$ is normalized so that the integral\n",
     "over the whole phase space is the total mass of the particles:\n",
     "\n",
     "$$\\int \\int f \\left(\\vec{x}, \\vec{p}, t\\right) \\,d^3\\vec{x}\\,d^3\\vec{p} = N,$$\n",
@@ -121,8 +121,8 @@
     "cell in every LBM step.\n",
     "\n",
     "<figure>\n",
-    "<img src='figures/latticeboltzmann-grid.png', style=\"width: 300px;\"/>\n",
     "<center>\n",
+    "<img src='figures/latticeboltzmann-grid.png', style=\"width: 300px;\"/>\n",
     "<figcaption>The 19 velocity vectors $\\vec{c_i}$ for a D3Q19 lattice. From the central grid point, the velocity vectors point towards all 18 nearest neighbours marked by filled circles. The 19th velocity vector is the rest mode (zero velocity).</figcaption>\n",
     "</center>\n",
     "</figure>"
@@ -137,7 +137,7 @@
     "The second step is the collision part, where the actual physics happens. For the LBM it is assumed that the collision process linearly relaxes the populations to the local equilibrium, thus that it is a linear (=matrix) operator \n",
     "acting on the populations in each LB cell. It should conserve \n",
     "the particle number and the momentum. At this point it is clear\n",
-    "why the mode space is helpful. A 19 dimensional matrix that\n",
+    "why the mode space is helpful. A 19-dimensional matrix that\n",
     "conserves the first 4 modes (with the eigenvalue 1) is diagonal in the\n",
     "first four rows and columns.\n",
     "By symmetry consideration one finds that only four independent\n",
@@ -155,22 +155,22 @@
     "and has the form\n",
     "\n",
     "\\begin{align*}\n",
-    "  m^\\star_i &= \\gamma_i \\left( m_i - m_i^\\text{eq} \\right) + m_i ^\\text{eq}.\n",
+    "  m^\\star_i &= \\omega_i \\left( m_i - m_i^\\text{eq} \\right) + m_i ^\\text{eq}.\n",
     "\\end{align*}\n",
     "\n",
     "Here $m^\\star_i$ is the $i$th mode after the collision.\n",
     "In other words: each mode is relaxed towards\n",
-    "its equilibrium value with a relaxation rate $\\gamma_i$.\n",
+    "its equilibrium value with a relaxation rate $\\omega_i$.\n",
     "The conserved modes are not relaxed, i.e. their relaxation rate is 1.\n",
     "We summarize them here:\n",
     "\n",
     "\\begin{align*}\n",
-    "  m^\\star_i &= \\gamma_i m_i  \\\\\n",
-    "  \\gamma_1=\\dots=\\gamma_4&=1 \\\\\n",
-    "  \\gamma_5&=\\gamma_\\text{b} \\\\\n",
-    "  \\gamma_6=\\dots=\\gamma_{10}&=\\gamma_\\text{s} \\\\\n",
-    "  \\gamma_{11}=\\dots=\\gamma_{16}&=\\gamma_\\text{odd} \\\\\n",
-    "  \\gamma_{17}=\\dots = \\gamma_{19}&=\\gamma_\\text{even} \\\\\n",
+    "  m^\\star_i &= \\omega_i m_i  \\\\\n",
+    "  \\omega_1=\\dots=\\omega_4&=1 \\\\\n",
+    "  \\omega_5&=\\omega_\\text{b} \\\\\n",
+    "  \\omega_6=\\dots=\\omega_{10}&=\\omega_\\text{s} \\\\\n",
+    "  \\omega_{11}=\\dots=\\omega_{16}&=\\omega_\\text{odd} \\\\\n",
+    "  \\omega_{17}=\\dots = \\omega_{19}&=\\omega_\\text{even} \\\\\n",
     "\\end{align*}\n",
     "\n",
     "To include hydrodynamic fluctuations of the fluid, \n",
@@ -202,8 +202,8 @@
     "to the fluctuation dissipation theorem. \n",
     "\n",
     "<figure>\n",
-    "<img src='figures/latticeboltzmann-momentumexchange.png', style=\"width: 300px;\"/>\n",
     "<center>\n",
+    "<img src='figures/latticeboltzmann-momentumexchange.png', style=\"width: 300px;\"/>\n",
     "<figcaption>The coupling scheme between fluid and particles is based on the interpolation of the fluid velocity $\\vec{u}$ from the grid nodes. This is done by trilinear interpolation. The difference between the particle velocity $\\vec{v}(t)$ and the interpolated velocity $\\vec{u}(\\vec{r},t)$ is used in the momentum exchange of the equation $\\vec{F}_H$ above.</figcaption>\n",
     "</center>\n",
     "</figure>"
@@ -221,33 +221,44 @@
     "[<tt>LBFluidWalberlaGPU</tt>](https://espressomd.github.io/doc/espressomd.html#espressomd.lb.LBFluidWalberlaGPU) in the module\n",
     "[<tt>espressomd.lb</tt>](https://espressomd.github.io/doc/espressomd.html#module-espressomd.lb).\n",
     "\n",
-    "The LB lattice is a cubic lattice, with a lattice constant <tt>agrid</tt> that\n",
+    "The LB lattice is a cubic lattice, with a lattice constant <tt>agrid</tt> in MD units that\n",
     "is the same in all spatial directions. The chosen box length must be an integer multiple\n",
     "of <tt>agrid</tt>. The LB lattice is shifted by 0.5 <tt>agrid</tt> in all directions: the node\n",
     "with integer coordinates $\\left(0,0,0\\right)$ is located at\n",
     "$\\left(0.5a,0.5a,0.5a\\right)$.\n",
     "The LB scheme and the MD scheme are not synchronized: in one\n",
     "LB time step, several MD steps may be performed. This allows to speed\n",
-    "up the simulations and is adjusted with the parameter <tt>tau</tt>.\n",
+    "up the simulations and is adjusted with the parameter <tt>tau</tt> in MD units.\n",
     "The LB parameter <tt>tau</tt> must be an integer multiple of the MD timestep.\n",
     "\n",
-    "Even if MD features are not used, the System parameters <tt>cell_system.skin</tt> and <tt>time_step</tt> must be set. LB steps are performed \n",
+    "Even if MD features are not used, the system parameters <tt>cell_system.skin</tt> and <tt>time_step</tt> must be set. LB steps are performed \n",
     "in regular intervals, such that the timestep $\\tau$ for LB is recovered.\n",
     "\n",
-    "Important note: all commands of the LB interface use\n",
-    "MD units. This is convenient, as e.g. a particular \n",
-    "viscosity can be set and the LB time step can be changed without\n",
-    "altering the viscosity. On the other hand this is a source\n",
+    "Important note: all commands of the LB interface use MD units.\n",
+    "This is convenient, as e.g. a particular viscosity can be set\n",
+    "and the LB time step can be changed without altering the viscosity.\n",
+    "In LB units, both <tt>agrid</tt> and <tt>tau</tt> are unity.\n",
+    "On the other hand this is a source\n",
     "of a plethora of mistakes: the LBM is only reliable in a certain \n",
     "range of parameters (in LB units) and the unit conversion\n",
     "may take some of them far out of this range. So remember to always\n",
     "make sure you are not messing with that!\n",
     "\n",
     "One brief example: a certain velocity may be 10 in MD units.\n",
     "If the LB time step is 0.1 in MD units, and the lattice constant\n",
-    "is 1, then it corresponds to a velocity of $1\\ \\frac{a}{\\tau}$ in LB units.\n",
-    "This is the maximum velocity of the discrete velocity set and therefore\n",
-    "causes numerical instabilities like negative populations."
+    "is 1.0 in MD units, then it corresponds to a velocity of 1 in LB units.\n",
+    "More specifically, given a velocity $v = 10\\ \\ell^\\star\\cdot t^{\\star-1}$ with\n",
+    "$\\ell^\\star$ the reduced MD length unit and $t^\\star$ the reduced MD time unit,\n",
+    "and LB parameters $a = 1\\ \\ell^\\star$ and $\\tau = 0.1\\ t^\\star$ in MD units,\n",
+    "then the corresponding velocity in LB units is given by\n",
+    "$u = v \\cdot \\tau / a = 10\\ \\ell^\\star\\cdot t^{\\star-1} \\cdot 0.1\\ t^\\star / 1\\ \\ell^{\\star} = 1$.\n",
+    "\n",
+    "In practice, you would like the fluid velocity to be much smaller than the fluid\n",
+    "speed of sound $c_s = \\frac{1}{\\sqrt{3}}\\frac{a}{\\tau}$ in MD units, aka Mach 1,\n",
+    "to avoid numerical instabilities due to the breakdown of the incompressibility assumption.\n",
+    "It is usually good practice to avoid exceeding Mach 0.2.\n",
+    "At Mach 0.35 the relative error in the calculated fluid density is ~6%,\n",
+    "which can ultimately lead to negative populations."
    ]
   },
   {