From efb27bf568366fd220412ebabe1818eabc6ec0ff Mon Sep 17 00:00:00 2001
From: Simone Bacchio <sbacchio@gmail.com>
Date: Mon, 18 Mar 2024 14:48:20 +0100
Subject: [PATCH 001/108] Adding new macro for declaring enum classes with
 useful utils

---
 include/declare_enum.h | 60 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 60 insertions(+)
 create mode 100644 include/declare_enum.h

diff --git a/include/declare_enum.h b/include/declare_enum.h
new file mode 100644
index 0000000000..175c47722e
--- /dev/null
+++ b/include/declare_enum.h
@@ -0,0 +1,60 @@
+/*
+ * A macro that declares an `enum class` as well as a `to_string` function for the enums.
+ * The enum has also a default value `size` that measures the size of the enum.
+ *
+ * Credit: https://stackoverflow.com/a/71375077/12084612
+ * -------
+ * License: CC BY-SA 4.0
+ * --------
+ * Usage:
+ * ------
+ *
+ * DECLARE_ENUM(WeekEnum, Mon, Tue, Wed, Thu, Fri, Sat, Sun,);
+ *
+ * int main()
+ * {
+ *     WeekEnum weekDay = WeekEnum::Wed;
+ *     std::cout << to_string(weekDay) << std::endl; // prints Wed
+ *     std::cout << to_string(WeekEnum::Sat) << std::endl; // prints Sat
+ *     std::cout << to_string((int) WeekEnum::size) << std::endl; // prints 7
+ *     return 0;
+ * }
+ *
+ */
+
+#pragma once
+#include <iostream>
+#include <string>
+#include <vector>
+#include <algorithm>
+#include <unordered_map>
+
+// Add the definition of this method into a cpp file. (only the declaration in the header)
+static inline const std::vector<std::string> &get_enum_names(const std::string &en_key, const std::string &en_str)
+{
+  static std::unordered_map<std::string, std::vector<std::string>> en_names_map;
+  const auto it = en_names_map.find(en_key);
+  if (it != en_names_map.end()) return it->second;
+
+  constexpr auto delim(',');
+  std::vector<std::string> en_names;
+  std::size_t start {};
+  auto end = en_str.find(delim);
+  while (end != std::string::npos) {
+    while (en_str[start] == ' ') ++start;
+    en_names.push_back(en_str.substr(start, end - start));
+    start = end + 1;
+    end = en_str.find(delim, start);
+  }
+  while (en_str[start] == ' ') ++start;
+  en_names.push_back(en_str.substr(start));
+  return en_names_map.emplace(en_key, std::move(en_names)).first->second;
+}
+
+#define DECLARE_ENUM(ENUM_NAME, ...)                                                                                   \
+  enum class ENUM_NAME : unsigned int { __VA_ARGS__ size };                                                            \
+  inline std::string to_string(ENUM_NAME en)                                                                           \
+  {                                                                                                                    \
+    const auto &names = get_enum_names(#ENUM_NAME #__VA_ARGS__, #__VA_ARGS__);                                         \
+    return names[static_cast<std::size_t>(en)];                                                                        \
+  }

From 2514ca715c3d8f977317ee1c33d57fcdfef409d3 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <sbacchio@gmail.com>
Date: Mon, 18 Mar 2024 14:50:54 +0100
Subject: [PATCH 002/108] Adding DDParam structure used as property for any
 lattice_field

---
 include/domain_decomposition.h | 153 +++++++++++++++++++++++++++++++++
 include/lattice_field.h        |  35 ++++++++
 2 files changed, 188 insertions(+)
 create mode 100644 include/domain_decomposition.h

diff --git a/include/domain_decomposition.h b/include/domain_decomposition.h
new file mode 100644
index 0000000000..464cd3b2c0
--- /dev/null
+++ b/include/domain_decomposition.h
@@ -0,0 +1,153 @@
+#pragma once
+
+#include <fast_intdiv.h>
+#include "declare_enum.h"
+
+namespace quda
+{
+
+  // using namespace quda;
+
+  DECLARE_ENUM(DD,     // name of the enum class
+               in_use, // whether DD is in use
+
+               mode_red_black, // red-black DD, e.g. for SAP
+               red_active,     // if red blocks are active
+               black_active,   // if black blocks are active
+               rb_hopping,     // if hopping between red and black is allowed
+               first_black,    // the color of the first block of the local lattice
+  );
+
+  // Params for domain decompation
+  // we template over Int such that int_fastdiv can be used in kernels
+  template <typename Int = int> class DDParam
+  {
+
+  public:
+    bool flags[(int)DD::size];  // the default value of all flags is 0
+    Int blockDim[QUDA_MAX_DIM]; // the size of the block per direction
+
+    // Default constructor
+    DDParam() : flags {0}, blockDim {0} { }
+
+    // Default copy
+    DDParam(const DDParam &dd) = default;
+
+    // Default copy
+    template <typename T> DDParam(const DDParam<T> &dd)
+    {
+#pragma unroll
+      for (auto i = 0u; i < (int)DD::size; i++) flags[i] = dd.flags[i];
+#pragma unroll
+      for (auto i = 0u; i < QUDA_MAX_DIM; i++) blockDim[i] = dd.blockDim[i];
+    }
+
+    // sets the given flag to true
+    constexpr inline DDParam &operator&=(const DD &flag)
+    {
+      flags[(int)flag] = true;
+      return *this;
+    }
+
+    // returns false if in use
+    constexpr inline bool operator!() const { return not flags[(int)DD::in_use]; }
+
+    // returns value of given flag
+    constexpr inline bool operator&&(const DD &flag) const { return flags[(int)flag]; }
+
+    constexpr inline bool is(const DD &flag) const { return flags[(int)flag]; }
+
+    // Pretty print the args struct
+    void print()
+    {
+      if (not *this) {
+        printfQuda("DDParam not in use\n");
+        return;
+      }
+      printfQuda("Printing DDParam\n");
+      for (int i = 0; i < (int)DD::size; i++)
+        printfQuda("flags[DD::%s] = %s\n", to_string((DD)i).c_str(), flags[i] ? "true" : "false");
+      for (int i = 0; i < QUDA_MAX_DIM; i++) printfQuda("blockDim[%d] = %d\n", i, static_cast<int>(blockDim[i]));
+    }
+
+    // template <bool debug = false, typename T> // if true, prints debug information
+    inline bool match(const DDParam &dd) const
+    {
+      bool debug = true;
+      // if one of the two is not in use we return true, i.e. one of the two is a full field
+      if (not *this or not dd) return true;
+
+      // false if only one is red_black
+      if (is(DD::mode_red_black) ^ dd.is(DD::mode_red_black)) {
+        if (debug) printfQuda("Only one of the two is red_black\n");
+        return false;
+      }
+
+      if (is(DD::mode_red_black) and dd.is(DD::mode_red_black))
+        for (int i = 0; i < QUDA_MAX_DIM; i++)
+          if (blockDim[i] != dd.blockDim[i]) {
+            if (debug)
+              printfQuda("blockDim[%d] = %d != %d \n", i, static_cast<int>(blockDim[i]),
+                         static_cast<int>(dd.blockDim[i]));
+            return false;
+          }
+
+      return true;
+    }
+
+    /* if a field is zero at given coordinates */
+    template <typename Coord, typename Arg> constexpr inline bool isZero(const Coord &x, const Arg &arg) const
+    {
+
+      // if DD not in use we return immidiatelly
+      if (not is(DD::in_use)) return false;
+
+      if (is(DD::mode_red_black)) {
+
+        // Computing block_parity: 0 = red, 1 = black
+        int block_parity = is(DD::first_black);
+        for (int i = 0; i < x.size(); i++) { block_parity += x[i] / blockDim[i]; }
+        block_parity %= 2;
+
+        // Checking if my parity is active
+        if (not is(DD::red_active) and block_parity == 0) return true;
+        if (not is(DD::black_active) and block_parity == 1) return true;
+      }
+
+      return false;
+    }
+
+    /* if hopping is allowed */
+    template <typename Coord, typename Arg>
+    constexpr inline bool doHopping(const Coord &x, const int &mu, const int &dir, const Arg &arg) const
+    {
+
+      // if DD not in use we return immidiatelly
+      if (not is(DD::in_use)) return true;
+
+      if (is(DD::mode_red_black)) {
+
+        // Checking if we are on the border
+        bool on_border = (dir > 0) ? ((x[mu] + 1) % blockDim[mu] == 0) : (x[mu] % blockDim[mu] == 0);
+
+        // If all the follwing are true then it is a full operator
+        if (is(DD::red_active) and is(DD::black_active)) {
+          if (not on_border || is(DD::rb_hopping)) return true;
+          return false;
+        }
+
+        // Computing block_parity: 0 = red, 1 = black
+        int block_parity = is(DD::first_black) + on_border;
+        for (int i = 0; i < x.size(); i++) { block_parity += x[i] / blockDim[i]; }
+        block_parity %= 2;
+
+        // Checking if my parity is active
+        if (is(DD::red_active) and block_parity == 0) return true;
+        if (is(DD::black_active) and block_parity == 1) return true;
+      }
+
+      return false;
+    }
+  };
+
+} // namespace quda
diff --git a/include/lattice_field.h b/include/lattice_field.h
index 462add3bde..0e044fe330 100644
--- a/include/lattice_field.h
+++ b/include/lattice_field.h
@@ -7,6 +7,7 @@
 #include <object.h>
 #include <quda_api.h>
 #include <reference_wrapper_helper.h>
+#include <domain_decomposition.h>
 
 /**
  * @file lattice_field.h
@@ -478,6 +479,9 @@ namespace quda {
      */
     inline static int bufferIndex = 0;
 
+    /** Domain decomposition options */
+    DDParam<int> dd {};
+
     /**
        @brief Default constructor
     */
@@ -928,6 +932,37 @@ namespace quda {
 
 #define checkNative(...) Native_(__func__, __FILE__, __LINE__, __VA_ARGS__)
 
+  /**
+     @brief Helper function for determining if the domain decomposition of the fields is the same.
+     @param[in] a Input field
+     @param[in] b Input field
+     @return true if all fields match
+   */
+  template <typename T1, typename T2>
+  inline bool DD_(const char *func, const char *file, int line, const T1 &a_, const T2 &b_)
+  {
+    const unwrap_t<T1> &a(a_);
+    const unwrap_t<T2> &b(b_);
+    if (!a.dd.match(b.dd)) errorQuda("DD not match (%s:%d in %s())", file, line, func);
+    return true;
+  }
+
+  /**
+     @brief Helper function for determining if the domain decomposition of the fields is the same.
+     @param[in] a Input field
+     @param[in] b Input field
+     @param[in] args List of additional fields to check domain decomposition on
+     @return true if all fields match
+   */
+  template <typename T1, typename T2, typename... Args>
+  inline bool DD_(const char *func, const char *file, int line, const T1 &a, const T2 &b, const Args &...args)
+  {
+    // checking all possible pairs
+    return (DD_(func, file, line, a, b) & DD_(func, file, line, a, args...) & DD_(func, file, line, b, args...));
+  }
+
+#define checkDD(...) DD_(__func__, __FILE__, __LINE__, __VA_ARGS__)
+
   /**
      @brief Return whether data is reordered on the CPU or GPU.  This can set
      at QUDA initialization using the environment variable

From 1c89d4ff9b31c7bfb93cb4ab6475d6c417b6c80a Mon Sep 17 00:00:00 2001
From: Simone Bacchio <sbacchio@gmail.com>
Date: Mon, 18 Mar 2024 14:54:13 +0100
Subject: [PATCH 003/108] Adding method size to Coord

---
 include/index_helper.cuh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/include/index_helper.cuh b/include/index_helper.cuh
index 990ff949e7..eae33e1e8d 100644
--- a/include/index_helper.cuh
+++ b/include/index_helper.cuh
@@ -236,6 +236,7 @@ namespace quda {
     int X;       // full lattice site index
     constexpr const int& operator[](int i) const { return x[i]; }
     constexpr int& operator[](int i) { return x[i]; }
+    constexpr int size() const { return nDim; }
   };
 
   /**

From 82e3d6ca3c60e77e44e530e84461e6d9a22af225 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <sbacchio@gmail.com>
Date: Mon, 18 Mar 2024 14:59:57 +0100
Subject: [PATCH 004/108] First implementation of DDaware dslash operator

---
 include/dslash_helper.cuh         |  9 +++++++++
 include/kernels/dslash_wilson.cuh | 30 +++++++++++++++++++++++-------
 2 files changed, 32 insertions(+), 7 deletions(-)

diff --git a/include/dslash_helper.cuh b/include/dslash_helper.cuh
index e67582b682..bcd7bc6dce 100644
--- a/include/dslash_helper.cuh
+++ b/include/dslash_helper.cuh
@@ -11,6 +11,7 @@
 #include <shmem_pack_helper.cuh>
 #include <kernel_helper.h>
 #include <tune_quda.h>
+#include <domain_decomposition.h>
 
 #if defined(_NVHPC_CUDA)
 #include <constant_kernel_arg.h>
@@ -286,6 +287,10 @@ namespace quda
     int exterior_dims; // dimension to run in the exterior Dslash
     int exterior_blocks;
 
+    DDParam<int_fastdiv> dd_out;
+    DDParam<int_fastdiv> dd_in;
+    DDParam<int_fastdiv> dd_x;
+
     // for shmem ...
     static constexpr bool packkernel = false;
     void *packBuffer[4 * QUDA_MAX_DIM];
@@ -337,6 +342,9 @@ namespace quda
       pack_blocks(0),
       exterior_dims(0),
       exterior_blocks(0),
+      dd_out(out.dd),
+      dd_in(in.dd),
+      dd_x(x.dd),
 #ifndef NVSHMEM_COMMS
       counter(0)
 #else
@@ -353,6 +361,7 @@ namespace quda
       checkOrder(out, in, x);        // check all orders match
       checkPrecision(out, in, x, U); // check all precisions match
       checkLocation(out, in, x, U);  // check all locations match
+      checkDD(out, in, x);           // check all DD match
       if (!in.isNative() || !U.isNative())
         errorQuda("Unsupported field order colorspinor=%d gauge=%d combination\n", in.FieldOrder(), U.FieldOrder());
 
diff --git a/include/kernels/dslash_wilson.cuh b/include/kernels/dslash_wilson.cuh
index f87e8f9865..c5a5b03303 100644
--- a/include/kernels/dslash_wilson.cuh
+++ b/include/kernels/dslash_wilson.cuh
@@ -73,7 +73,8 @@ namespace quda
 
 #pragma unroll
     for (int d = 0; d < 4; d++) { // loop over dimension - 4 and not nDim since this is used for DWF as well
-      {                           // Forward gather - compute fwd offset for vector fetch
+      // Forward gather - compute fwd offset for vector fetch
+      if (arg.dd_in.doHopping(coord, d, +1, arg.dc)) {
         const int fwd_idx = getNeighborIndexCB(coord, d, +1, arg.dc);
         const int gauge_idx = (Arg::nDim == 5 ? coord.x_cb % arg.dc.volume_4d_cb : coord.x_cb);
         constexpr int proj_dir = dagger ? +1 : -1;
@@ -99,7 +100,8 @@ namespace quda
         }
       }
 
-      { // Backward gather - compute back offset for spinor and gauge fetch
+      // Backward gather - compute back offset for spinor and gauge fetch
+      if (arg.dd_in.doHopping(coord, d, -1, arg.dc)) {
         const int back_idx = getNeighborIndexCB(coord, d, -1, arg.dc);
         const int gauge_idx = (Arg::nDim == 5 ? back_idx % arg.dc.volume_4d_cb : back_idx);
         constexpr int proj_dir = dagger ? -1 : +1;
@@ -148,15 +150,29 @@ namespace quda
 
       const int my_spinor_parity = nParity == 2 ? parity : 0;
       Vector out;
-      applyWilson<nParity, dagger, mykernel_type>(out, arg, coord, parity, idx, thread_dim, active);
 
       int xs = coord.x_cb + coord.s * arg.dc.volume_4d_cb;
+      if (arg.dd_out.isZero(coord, arg.dc)) {
+        if (mykernel_type != EXTERIOR_KERNEL_ALL || active) arg.out(xs, my_spinor_parity) = out;
+        return;
+      }
+
+      applyWilson<nParity, dagger, mykernel_type>(out, arg, coord, parity, idx, thread_dim, active);
+
       if (xpay && mykernel_type == INTERIOR_KERNEL) {
-        Vector x = arg.x(xs, my_spinor_parity);
-        out = x + arg.a * out;
+        if (arg.dd_x.isZero(coord, arg.dc)) {
+          out = arg.a * out;
+        } else {
+          Vector x = arg.x(xs, my_spinor_parity);
+          out = x + arg.a * out;
+        }
       } else if (mykernel_type != INTERIOR_KERNEL && active) {
-        Vector x = arg.out(xs, my_spinor_parity);
-        out = x + (xpay ? arg.a * out : out);
+        if (arg.dd_x.isZero(coord, arg.dc)) {
+          out = (xpay ? arg.a * out : out);
+        } else {
+          Vector x = arg.out(xs, my_spinor_parity);
+          out = x + (xpay ? arg.a * out : out);
+        }
       }
 
       if (mykernel_type != EXTERIOR_KERNEL_ALL || active) arg.out(xs, my_spinor_parity) = out;

From 2ac39adde2f6bdcc48d1a8abf617972392928bab Mon Sep 17 00:00:00 2001
From: Simone Bacchio <sbacchio@gmail.com>
Date: Tue, 19 Mar 2024 08:58:10 +0100
Subject: [PATCH 005/108] Added dd options for tests

---
 tests/utils/command_line_params.cpp | 20 ++++++++++++++++----
 tests/utils/command_line_params.h   |  4 ++++
 2 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/tests/utils/command_line_params.cpp b/tests/utils/command_line_params.cpp
index d9cf5227d1..8cfb639c2f 100644
--- a/tests/utils/command_line_params.cpp
+++ b/tests/utils/command_line_params.cpp
@@ -74,6 +74,9 @@ QudaInverterType precon_type = QUDA_INVALID_INVERTER;
 QudaSchwarzType precon_schwarz_type = QUDA_INVALID_SCHWARZ;
 QudaAcceleratorType precon_accelerator_type = QUDA_INVALID_ACCELERATOR;
 
+std::array<int, 4> dd_block_size = {4, 4, 4, 4};
+bool dd_red_black = false;
+
 double madwf_diagonal_suppressor = 0.0;
 int madwf_ls = 4;
 int madwf_null_miniter = niter;
@@ -1029,7 +1032,7 @@ void add_multigrid_option_group(std::shared_ptr<QUDAApp> quda_app)
 
 void add_eofa_option_group(std::shared_ptr<QUDAApp> quda_app)
 {
-  auto opgroup = quda_app->add_option_group("EOFA", "Options controlling EOFA parameteres");
+  auto opgroup = quda_app->add_option_group("EOFA", "Options controlling EOFA parameters");
 
   CLI::TransformPairs<int> eofa_pm_map {{"plus", 1}, {"minus", 0}};
   opgroup->add_option("--eofa-pm", eofa_pm, "Set to evalute \"plus\" or \"minus\" EOFA operator (default plus)")
@@ -1040,9 +1043,19 @@ void add_eofa_option_group(std::shared_ptr<QUDAApp> quda_app)
   opgroup->add_option("--eofa-mq3", eofa_mq1, "Set mq3 for EOFA operator (default 1.0)");
 }
 
+void add_dd_option_group(std::shared_ptr<QUDAApp> quda_app)
+{
+  auto opgroup = quda_app->add_option_group("DD", "Options controlling DD parameters");
+  opgroup
+    ->add_option("--dd-block-size", dd_block_size,
+                 "Set the domain decomposition block size in all four dimension (default 4 4 4 4)")
+    ->expected(4);
+  opgroup->add_option("--dd-red-black", dd_red_black, "Enable red-black domain decomposition (default false)");
+}
+
 void add_madwf_option_group(std::shared_ptr<QUDAApp> quda_app)
 {
-  auto opgroup = quda_app->add_option_group("MADWF", "Options controlling MADWF parameteres");
+  auto opgroup = quda_app->add_option_group("MADWF", "Options controlling MADWF parameters");
 
   opgroup->add_option("--madwf-diagonal-suppressor", madwf_diagonal_suppressor,
                       "Set the digonal suppressor for MADWF (default 0)");
@@ -1103,8 +1116,7 @@ void add_gaugefix_option_group(std::shared_ptr<QUDAApp> quda_app)
 
 void add_comms_option_group(std::shared_ptr<QUDAApp> quda_app)
 {
-  auto opgroup
-    = quda_app->add_option_group("Communication", "Options controlling communication (split grid) parameteres");
+  auto opgroup = quda_app->add_option_group("Communication", "Options controlling communication (split grid) parameters");
   opgroup->add_option("--grid-partition", grid_partition, "Set the grid partition (default 1 1 1 1)")->expected(4);
 }
 
diff --git a/tests/utils/command_line_params.h b/tests/utils/command_line_params.h
index fb52e5764d..4e88d9313e 100644
--- a/tests/utils/command_line_params.h
+++ b/tests/utils/command_line_params.h
@@ -133,6 +133,7 @@ void add_eigen_option_group(std::shared_ptr<QUDAApp> quda_app);
 void add_deflation_option_group(std::shared_ptr<QUDAApp> quda_app);
 void add_multigrid_option_group(std::shared_ptr<QUDAApp> quda_app);
 void add_eofa_option_group(std::shared_ptr<QUDAApp> quda_app);
+void add_dd_option_group(std::shared_ptr<QUDAApp> quda_app);
 void add_madwf_option_group(std::shared_ptr<QUDAApp> quda_app);
 void add_su3_option_group(std::shared_ptr<QUDAApp> quda_app);
 void add_heatbath_option_group(std::shared_ptr<QUDAApp> quda_app);
@@ -211,6 +212,9 @@ extern QudaInverterType precon_type;
 extern QudaSchwarzType precon_schwarz_type;
 extern QudaAcceleratorType precon_accelerator_type;
 
+extern std::array<int, 4> dd_block_size;
+extern bool dd_red_black;
+
 extern double madwf_diagonal_suppressor;
 extern int madwf_ls;
 extern int madwf_null_miniter;

From 3945185153b8de44df95e9d6970ec25fd5ec36ca Mon Sep 17 00:00:00 2001
From: Simone Bacchio <sbacchio@gmail.com>
Date: Tue, 19 Mar 2024 08:58:30 +0100
Subject: [PATCH 006/108] Added set and reset functions

---
 include/domain_decomposition.h | 30 ++++++++++++++++++++++--------
 1 file changed, 22 insertions(+), 8 deletions(-)

diff --git a/include/domain_decomposition.h b/include/domain_decomposition.h
index 464cd3b2c0..c9b2ca039b 100644
--- a/include/domain_decomposition.h
+++ b/include/domain_decomposition.h
@@ -42,21 +42,35 @@ namespace quda
       for (auto i = 0u; i < QUDA_MAX_DIM; i++) blockDim[i] = dd.blockDim[i];
     }
 
-    // sets the given flag to true
-    constexpr inline DDParam &operator&=(const DD &flag)
-    {
-      flags[(int)flag] = true;
-      return *this;
-    }
-
     // returns false if in use
     constexpr inline bool operator!() const { return not flags[(int)DD::in_use]; }
 
     // returns value of given flag
-    constexpr inline bool operator&&(const DD &flag) const { return flags[(int)flag]; }
+    constexpr inline bool operator&&(const DD &flag) const { return flags[(int)DD::in_use] && flags[(int)flag]; }
 
+    // returns value of given flag
     constexpr inline bool is(const DD &flag) const { return flags[(int)flag]; }
 
+    constexpr inline void set(const DD &flag) { flags[(int)flag] = true; }
+
+    template <typename... Args> constexpr inline void set(const DD &flag, Args... args)
+    {
+      set(flag);
+      set(args...);
+    }
+
+    constexpr inline void reset()
+    {
+#pragma unroll
+      for (auto i = 0u; i < (int)DD::size; i++) flags[i] = 0;
+    }
+
+    template <typename... Args> constexpr inline void reset(Args... args)
+    {
+      reset();
+      set(args...);
+    }
+
     // Pretty print the args struct
     void print()
     {

From c116ff84f0ea1ac1c60e5a4ca38d814bc5fa1192 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <sbacchio@gmail.com>
Date: Tue, 19 Mar 2024 08:59:26 +0100
Subject: [PATCH 007/108] Addind test routines for domain decomposition

---
 tests/dslash_test.cpp     |  1 +
 tests/dslash_test_utils.h | 40 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 41 insertions(+)

diff --git a/tests/dslash_test.cpp b/tests/dslash_test.cpp
index 11f8265e7f..107d2ccba1 100644
--- a/tests/dslash_test.cpp
+++ b/tests/dslash_test.cpp
@@ -89,6 +89,7 @@ int main(int argc, char **argv)
   auto app = make_app();
   app->add_option("--test", dtest_type, "Test method")->transform(CLI::CheckedTransformer(dtest_type_map));
   add_eofa_option_group(app);
+  add_dd_option_group(app);
   add_comms_option_group(app);
 
   try {
diff --git a/tests/dslash_test_utils.h b/tests/dslash_test_utils.h
index 3d13d11478..d1871248ef 100644
--- a/tests/dslash_test_utils.h
+++ b/tests/dslash_test_utils.h
@@ -62,6 +62,7 @@ struct DslashTestWrapper {
   // CUDA color spinor fields
   ColorSpinorField cudaSpinor;
   ColorSpinorField cudaSpinorOut;
+  ColorSpinorField cudaSpinorTmp;
 
   // Dirac pointers
   quda::Dirac *dirac = nullptr;
@@ -81,6 +82,7 @@ struct DslashTestWrapper {
   QudaParity parity = QUDA_EVEN_PARITY;
   static inline dslash_test_type dtest_type = dslash_test_type::Dslash;
   static inline bool test_split_grid = false;
+  static inline bool test_domain_decomposition = false;
   int num_src = 1;
 
   const bool transfer = false;
@@ -144,6 +146,8 @@ struct DslashTestWrapper {
 
     if (inv_param.cpu_prec != gauge_param.cpu_prec) errorQuda("Gauge and spinor CPU precisions must match");
 
+    test_domain_decomposition = dd_red_black;
+
     for (int i = 0; i < 4; i++) inv_param.split_grid[i] = grid_partition[i];
     num_src = grid_partition[0] * grid_partition[1] * grid_partition[2] * grid_partition[3];
     test_split_grid = num_src > 1;
@@ -291,6 +295,7 @@ struct DslashTestWrapper {
       cudaSpinor = ColorSpinorField(csParam);
       printfQuda("Creating cudaSpinorOut with nParity = %d\n", csParam.siteSubset);
       cudaSpinorOut = ColorSpinorField(csParam);
+      if (test_domain_decomposition) { cudaSpinorTmp = ColorSpinorField(csParam); }
 
       printfQuda("Sending spinor field to GPU\n");
       cudaSpinor = spinor;
@@ -790,6 +795,41 @@ struct DslashTestWrapper {
         dslashMultiSrcQuda(_hp_x.data(), _hp_b.data(), &inv_param, parity, hostGauge, &gauge_param);
       }
 
+    } else if (test_domain_decomposition) {
+
+      if (dd_red_black) {
+        for (auto i = 0u; i < 4; i++) {
+          cudaSpinor.dd.blockDim[i] = dd_block_size[i];
+          cudaSpinorTmp.dd.blockDim[i] = dd_block_size[i];
+        }
+
+        blas::zero(cudaSpinorOut);
+
+        // We test it applying all 4 possible operators Drr, Drb, Dbr, Dbb
+        for (int col = 0; col < 4; col++) {
+
+          cudaSpinor.dd.reset(col % 2 == 0 ? DD::red_active : DD::black_active);
+          cudaSpinorTmp.dd.reset(col / 2 == 0 ? DD::red_active : DD::black_active);
+
+          switch (dtest_type) {
+          case dslash_test_type::Dslash: dirac->Dslash(cudaSpinorTmp, cudaSpinor, parity); break;
+          case dslash_test_type::MatPC:
+          case dslash_test_type::Mat: dirac->M(cudaSpinorTmp, cudaSpinor); break;
+          case dslash_test_type::MatPCDagMatPC:
+          case dslash_test_type::MatDagMat: dirac->MdagM(cudaSpinorTmp, cudaSpinor); break;
+          default:
+            errorQuda("Test type %s not support for current Dslash", get_string(dtest_type_map, dtest_type).c_str());
+          }
+
+          cudaSpinor.dd.reset();
+          cudaSpinorTmp.dd.reset();
+
+          blas::xpy(cudaSpinorTmp, cudaSpinorOut);
+        }
+      } else {
+        errorQuda("Test dd type not supported");
+      }
+
     } else {
 
       for (int i = 0; i < niter; i++) {

From 9ef487173f431e22f2e27c1b858821eab76b2d9c Mon Sep 17 00:00:00 2001
From: Simone Bacchio <sbacchio@gmail.com>
Date: Tue, 19 Mar 2024 09:13:44 +0100
Subject: [PATCH 008/108] Enabling DD::in_use by default

---
 include/domain_decomposition.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/include/domain_decomposition.h b/include/domain_decomposition.h
index c9b2ca039b..a8d4cc3258 100644
--- a/include/domain_decomposition.h
+++ b/include/domain_decomposition.h
@@ -68,7 +68,7 @@ namespace quda
     template <typename... Args> constexpr inline void reset(Args... args)
     {
       reset();
-      set(args...);
+      set(DD::in_use, args...);
     }
 
     // Pretty print the args struct

From 5435b960d2d901f15abf7583e8d85b1b880beaa3 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <sbacchio@gmail.com>
Date: Tue, 19 Mar 2024 09:13:57 +0100
Subject: [PATCH 009/108] Adding missing DD::mode

---
 tests/dslash_test_utils.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/dslash_test_utils.h b/tests/dslash_test_utils.h
index d1871248ef..50b866fa75 100644
--- a/tests/dslash_test_utils.h
+++ b/tests/dslash_test_utils.h
@@ -808,8 +808,8 @@ struct DslashTestWrapper {
         // We test it applying all 4 possible operators Drr, Drb, Dbr, Dbb
         for (int col = 0; col < 4; col++) {
 
-          cudaSpinor.dd.reset(col % 2 == 0 ? DD::red_active : DD::black_active);
-          cudaSpinorTmp.dd.reset(col / 2 == 0 ? DD::red_active : DD::black_active);
+          cudaSpinor.dd.reset(DD::mode_red_black, col % 2 == 0 ? DD::red_active : DD::black_active);
+          cudaSpinorTmp.dd.reset(DD::mode_red_black, col / 2 == 0 ? DD::red_active : DD::black_active);
 
           switch (dtest_type) {
           case dslash_test_type::Dslash: dirac->Dslash(cudaSpinorTmp, cudaSpinor, parity); break;

From 66c7ae6be897b82c07d1a45e2bc14c5cc7d33229 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <sbacchio@gmail.com>
Date: Tue, 19 Mar 2024 09:14:29 +0100
Subject: [PATCH 010/108] Example of usage of DD in SAP with MR solver

---
 include/invert_quda.h | 12 ++++++++++++
 lib/inv_mr_quda.cpp   | 22 +++++++++++++++++++++-
 2 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/include/invert_quda.h b/include/invert_quda.h
index 8b46148c96..5dc83ce7bf 100644
--- a/include/invert_quda.h
+++ b/include/invert_quda.h
@@ -235,6 +235,10 @@ namespace quda {
     /** Whether to use additive or multiplicative Schwarz preconditioning */
     QudaSchwarzType schwarz_type = QUDA_INVALID_SCHWARZ;
 
+    /** The size of a block per direction in the Schwarz procedure
+        (default = 0, i.e. local volume, old implementation) */
+    int schwarz_block[QUDA_MAX_DIM] = {0};
+
     /** The type of accelerator type to use for preconditioner */
     QudaAcceleratorType accelerator_type_precondition = QUDA_INVALID_ACCELERATOR;
 
@@ -400,6 +404,14 @@ namespace quda {
 
     // for incremental eigCG:
     void updateRhsIndex(QudaInvertParam &param) { rhs_idx = param.rhs_idx; }
+
+    constexpr inline bool do_block_schwarz() const
+    {
+      if (schwarz_type == QUDA_INVALID_SCHWARZ) return false;
+      for (int i = 0; i < QUDA_MAX_DIM; i++)
+        if (schwarz_block[i] > 0) return true;
+      return false;
+    }
   };
 
   class Solver {
diff --git a/lib/inv_mr_quda.cpp b/lib/inv_mr_quda.cpp
index df8f0f2dbc..3e744fa2c6 100644
--- a/lib/inv_mr_quda.cpp
+++ b/lib/inv_mr_quda.cpp
@@ -88,12 +88,25 @@ namespace quda
 
       int k = 0;
       double scale = 1.0;
-      if ((node_parity + step) % 2 == 0 && param.schwarz_type == QUDA_MULTIPLICATIVE_SCHWARZ) {
+      if (!param.do_block_schwarz() && param.schwarz_type == QUDA_MULTIPLICATIVE_SCHWARZ
+          && (node_parity + step) % 2 == 0) {
         // for multiplicative Schwarz we alternate updates depending on node parity
       } else {
 
         commGlobalReductionPush(param.global_reduction); // use local reductions for DD solver
 
+        if (param.do_block_schwarz()) {
+          if (param.schwarz_type == QUDA_MULTIPLICATIVE_SCHWARZ) {
+            // Red or black active
+            Ar.dd.reset(DD::mode_red_black, step % 2 == 0 ? DD::red_active : DD::black_active);
+            r_sloppy.dd.reset(DD::mode_red_black, step % 2 == 0 ? DD::red_active : DD::black_active);
+          } else {
+            // Both red and black active but no hopping
+            Ar.dd.reset(DD::mode_red_black, DD::red_active, DD::black_active);
+            r_sloppy.dd.reset(DD::mode_red_black, DD::red_active, DD::black_active);
+          }
+        }
+
         blas::zero(x_sloppy); // can get rid of this for a special first update kernel
         double c2 = param.global_reduction == QUDA_BOOLEAN_TRUE ? r2 : blas::norm2(r); // c2 holds the initial r2
         scale = c2 > 0.0 ? sqrt(c2) : 1.0;
@@ -119,6 +132,7 @@ namespace quda
           } else {
             // doing local reductions so can make it asynchronous
             commAsyncReductionSet(true);
+            // TODO: make these block aware
             blas::cDotProductNormA(Ar, r_sloppy);
 
             // omega*alpha is done in the kernel
@@ -132,6 +146,12 @@ namespace quda
 
         blas::axpy(scale, x_sloppy, x); // Scale and sum to accumulator
 
+        if (param.do_block_schwarz()) {
+          // Disable domain decomposition
+          Ar.dd.reset();
+          r_sloppy.dd.reset();
+        }
+
         commGlobalReductionPop(); // renable global reductions for outer solver
       }
 

From 20032120b8dd4f56e0b4adf6975deca532cb25b0 Mon Sep 17 00:00:00 2001
From: Ferenc Pittler <pittler@hiskp.uni-bonn.de>
Date: Thu, 4 Apr 2024 13:51:28 +0200
Subject: [PATCH 011/108] implementing projection of ColorSpinorField to a
 domain determined by ddParam

---
 include/color_spinor_field.h | 10 +++++++
 lib/color_spinor_field.cpp   | 23 +++++++++++++++
 lib/color_spinor_util.in.cu  | 54 ++++++++++++++++++++++++++++++++++++
 3 files changed, 87 insertions(+)

diff --git a/include/color_spinor_field.h b/include/color_spinor_field.h
index 453a5e8df4..cbc41eae7e 100644
--- a/include/color_spinor_field.h
+++ b/include/color_spinor_field.h
@@ -149,6 +149,7 @@ namespace quda
     int composite_dim = 0; // e.g., number of eigenvectors in the set
     bool is_component = false;
     int component_id = 0; // eigenvector index
+    DDParam<int> dd {};
 
     /**
        If using CUDA native fields, this function will ensure that the
@@ -438,6 +439,13 @@ namespace quda
      */
     void copy(const ColorSpinorField &src);
 
+    /**
+       @brief Project the field to a domain determined by
+       ddParam
+     */
+
+    void projectDD();
+
     /**
        @brief Zero all elements of this field
      */
@@ -946,6 +954,8 @@ namespace quda
                               void *Dst = nullptr, const void *Src = nullptr);
 
   void genericSource(ColorSpinorField &a, QudaSourceType sourceType, int x, int s, int c);
+
+  void genericProjectDD(ColorSpinorField &a);
   int genericCompare(const ColorSpinorField &a, const ColorSpinorField &b, int tol);
 
   /**
diff --git a/lib/color_spinor_field.cpp b/lib/color_spinor_field.cpp
index fba73f7b62..d0f558f533 100644
--- a/lib/color_spinor_field.cpp
+++ b/lib/color_spinor_field.cpp
@@ -98,6 +98,7 @@ namespace quda
     nSpin = param.nSpin;
     nVec = param.nVec;
     twistFlavor = param.twistFlavor;
+    dd = param.dd;
 
     if (param.pc_type != QUDA_5D_PC && param.pc_type != QUDA_4D_PC) errorQuda("Unexpected pc_type %d", param.pc_type);
     pc_type = param.pc_type;
@@ -527,6 +528,7 @@ namespace quda
     param.pc_type = pc_type;
     param.suggested_parity = suggested_parity;
     param.create = QUDA_NULL_FIELD_CREATE;
+    param.dd = dd;
   }
 
   void ColorSpinorField::exchange(void **ghost, void **sendbuf, int nFace) const
@@ -1532,6 +1534,27 @@ namespace quda
     genericPrintVector(*this, parity, x_cb, rank);
   }
 
+  void ColorSpinorField::projectDD()
+  {
+    if (Location() == QUDA_CPU_FIELD_LOCATION) {
+      genericProjectDD(*this);
+    } else {
+      ColorSpinorParam param(*this);
+      param.fieldOrder = QUDA_SPACE_SPIN_COLOR_FIELD_ORDER;
+      param.location = QUDA_CPU_FIELD_LOCATION;
+      param.setPrecision((param.Precision() == QUDA_HALF_PRECISION || param.Precision() == QUDA_QUARTER_PRECISION) ?
+                           QUDA_SINGLE_PRECISION :
+                           param.Precision());
+      param.create = QUDA_COPY_FIELD_CREATE;
+      // since CPU fields cannot be low precision, use single precision instead
+      if (precision < QUDA_SINGLE_PRECISION) param.setPrecision(QUDA_SINGLE_PRECISION, QUDA_INVALID_PRECISION, false);
+
+      ColorSpinorField tmp(param);
+      tmp.projectDD();
+      *this = tmp;
+    }
+  }
+
   int ColorSpinorField::Compare(const ColorSpinorField &a, const ColorSpinorField &b, const int tol)
   {
     if (checkLocation(a, b) == QUDA_CUDA_FIELD_LOCATION) errorQuda("device field not implemented");
diff --git a/lib/color_spinor_util.in.cu b/lib/color_spinor_util.in.cu
index 0b9355d4d1..9dfbad1fb7 100644
--- a/lib/color_spinor_util.in.cu
+++ b/lib/color_spinor_util.in.cu
@@ -423,4 +423,58 @@ namespace quda {
     resize(v, new_size, param);
   }
 
+  template <class P> void projectDD(P &p, const ColorSpinorField &meta)
+  {
+    Coord<4> coord;
+    int X[4] = {meta.X(0), meta.X(1), meta.X(2), meta.X(3)};
+    X[0] *= (p.Nparity() == 1) ? 2 : 1; // need full lattice dims
+
+    for (int parity = 0; parity < p.Nparity(); parity++) {
+      for (int x_cb = 0; x_cb < p.VolumeCB(); x_cb++) {
+        getCoords(coord, x_cb, X, parity);
+
+        if (meta.dd.isZero(coord, meta.getDslashConstant())) {
+          //	  printfQuda("Should be zero %d\n",x_cb);
+          for (int s = 0; s < p.Nspin(); s++)
+            for (int c = 0; c < p.Ncolor(); c++) p(parity, x_cb, s, c) = 0;
+        } else {
+          //	  printfQuda("Should not be zero %d\n",x_cb);
+        }
+      }
+    }
+  }
+
+  template <typename Float, int nSpin, int nColor, QudaFieldOrder order> void genericProjectDD(ColorSpinorField &a)
+  {
+    FieldOrderCB<Float, nSpin, nColor, 1, order> A(a);
+    projectDD(A, a);
+  }
+
+  template <typename Float> void genericProjectDD(ColorSpinorField &a)
+  {
+
+    switch (a.Nspin()) {
+    case (1):
+      if constexpr (is_enabled_spin(1)) genericProjectDD<Float, 1, 3, QUDA_SPACE_SPIN_COLOR_FIELD_ORDER>(a);
+      break;
+
+    case (2):
+      if constexpr (is_enabled_spin(2)) genericProjectDD<Float, 2, 3, QUDA_SPACE_SPIN_COLOR_FIELD_ORDER>(a);
+      break;
+    case (4):
+      if constexpr (is_enabled_spin(4)) genericProjectDD<Float, 4, 3, QUDA_SPACE_SPIN_COLOR_FIELD_ORDER>(a);
+      break;
+    default: errorQuda("Nspin %d not implemented", a.Nspin());
+    }
+  }
+
+  void genericProjectDD(ColorSpinorField &a)
+  {
+    switch (a.Precision()) {
+    case QUDA_DOUBLE_PRECISION: genericProjectDD<double>(a); break;
+    case QUDA_SINGLE_PRECISION: genericProjectDD<float>(a); break;
+    default: errorQuda("Precision %d not implemented", a.Precision());
+    }
+  }
+
 } // namespace quda

From f76859b14f71fffb76d5bf9f521d673fb1398388 Mon Sep 17 00:00:00 2001
From: Ferenc Pittler <pittler@hiskp.uni-bonn.de>
Date: Thu, 4 Apr 2024 13:52:30 +0200
Subject: [PATCH 012/108] implementing tests for projection on domain
 determined by ddParam

---
 tests/dslash_test_utils.h | 41 ++++++++++++++++++++++++++++++++++++++-
 1 file changed, 40 insertions(+), 1 deletion(-)

diff --git a/tests/dslash_test_utils.h b/tests/dslash_test_utils.h
index 50b866fa75..d26f8ca662 100644
--- a/tests/dslash_test_utils.h
+++ b/tests/dslash_test_utils.h
@@ -63,6 +63,8 @@ struct DslashTestWrapper {
   ColorSpinorField cudaSpinor;
   ColorSpinorField cudaSpinorOut;
   ColorSpinorField cudaSpinorTmp;
+  ColorSpinorField cudaSpinorTmp2;
+  ColorSpinorField cudaSpinorTmp3;
 
   // Dirac pointers
   quda::Dirac *dirac = nullptr;
@@ -295,7 +297,11 @@ struct DslashTestWrapper {
       cudaSpinor = ColorSpinorField(csParam);
       printfQuda("Creating cudaSpinorOut with nParity = %d\n", csParam.siteSubset);
       cudaSpinorOut = ColorSpinorField(csParam);
-      if (test_domain_decomposition) { cudaSpinorTmp = ColorSpinorField(csParam); }
+      if (test_domain_decomposition) {
+        cudaSpinorTmp = ColorSpinorField(csParam);
+        cudaSpinorTmp2 = ColorSpinorField(csParam);
+        cudaSpinorTmp3 = ColorSpinorField(csParam);
+      }
 
       printfQuda("Sending spinor field to GPU\n");
       cudaSpinor = spinor;
@@ -801,6 +807,8 @@ struct DslashTestWrapper {
         for (auto i = 0u; i < 4; i++) {
           cudaSpinor.dd.blockDim[i] = dd_block_size[i];
           cudaSpinorTmp.dd.blockDim[i] = dd_block_size[i];
+          cudaSpinorTmp2.dd.blockDim[i] = dd_block_size[i];
+          cudaSpinorTmp3.dd.blockDim[i] = dd_block_size[i];
         }
 
         blas::zero(cudaSpinorOut);
@@ -824,6 +832,33 @@ struct DslashTestWrapper {
           cudaSpinor.dd.reset();
           cudaSpinorTmp.dd.reset();
 
+          cudaSpinorTmp2 = spinor;
+          cudaSpinorTmp2.dd.reset(DD::mode_red_black, col % 2 == 0 ? DD::red_active : DD::black_active);
+          cudaSpinorTmp2.projectDD();
+          cudaSpinorTmp2.dd.reset();
+
+          switch (dtest_type) {
+          case dslash_test_type::Dslash: dirac->Dslash(cudaSpinorTmp3, cudaSpinorTmp2, parity); break;
+          case dslash_test_type::MatPC:
+          case dslash_test_type::Mat: dirac->M(cudaSpinorTmp3, cudaSpinorTmp2); break;
+          case dslash_test_type::MatPCDagMatPC:
+          case dslash_test_type::MatDagMat: dirac->MdagM(cudaSpinorTmp3, cudaSpinorTmp2); break;
+          default:
+            errorQuda("Test type %s not support for current Dslash", get_string(dtest_type_map, dtest_type).c_str());
+          }
+
+          cudaSpinorTmp3.dd.reset(DD::mode_red_black, col / 2 == 0 ? DD::red_active : DD::black_active);
+          cudaSpinorTmp3.projectDD();
+          cudaSpinorTmp3.dd.reset();
+
+          spinorTmp = cudaSpinorTmp;
+          spinorOut = cudaSpinorTmp3;
+
+          double deviation = std::pow(10, -(double)(ColorSpinorField::Compare(spinorTmp, spinorOut)));
+          printfQuda("Deviaton source %d sink %d %e\n", col % 2, col / 2, deviation);
+
+          // here test cudaSpinorTmp==cudaSpinorTmp2
+
           blas::xpy(cudaSpinorTmp, cudaSpinorOut);
         }
       } else {
@@ -1079,6 +1114,10 @@ struct DslashTestWrapper {
 
       size_t ghost_bytes = cudaSpinor.GhostBytes();
 
+      printfQuda("dslash time event time %e %e %e\n", dslash_time.event_time, dslash_time.cpu_time, dslash_time.cpu_max);
+      printfQuda("niter %d\n", niter);
+      printfQuda("ghost_bytes %e\n", ghost_bytes);
+
       printfQuda("Effective halo bi-directional bandwidth (GB/s) GPU = %f ( CPU = %f, min = %f , max = %f ) for "
                  "aggregate message size %lu bytes\n",
                  1.0e-9 * 2 * ghost_bytes * niter / dslash_time.event_time,

From 99fa6434770eadf67b2f20027cca16282c9d9b92 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Tue, 7 May 2024 09:10:30 +0200
Subject: [PATCH 013/108] Adding flags QUDA_DIRAC_DD and GPU_DD_DIRAC

---
 CMakeLists.txt           | 2 ++
 include/quda_define.h.in | 9 +++++++++
 2 files changed, 11 insertions(+)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 03ea9f4fba..8d9ce6c032 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -151,6 +151,8 @@ option(QUDA_DIRAC_CLOVER_HASENBUSCH "build clover Hasenbusch twist operators" ${
 option(QUDA_DIRAC_NDEG_TWISTED_MASS "build non-degenerate twisted mass Dirac operators" ${QUDA_DIRAC_DEFAULT})
 option(QUDA_DIRAC_NDEG_TWISTED_CLOVER "build non-degenerate twisted clover Dirac operators" ${QUDA_DIRAC_DEFAULT})
 
+option(QUDA_DIRAC_DD "build code for domain decomposition of the Dirac operator" OFF)
+
 option(QUDA_CONTRACT "build code for bilinear contraction" OFF)
 option(QUDA_COVDEV "build code for covariant derivative" OFF)
 
diff --git a/include/quda_define.h.in b/include/quda_define.h.in
index 445d3cd544..2692833f38 100644
--- a/include/quda_define.h.in
+++ b/include/quda_define.h.in
@@ -128,6 +128,15 @@
 #define GPU_LAPLACE
 #endif
 
+#cmakedefine QUDA_DIRAC_DD
+#ifdef QUDA_DIRAC_DD
+/**
+ * @def   GPU_DD_DIRAC
+ * @brief This macro is set when we have DD-aware Dirac operator enabled
+ */
+#define GPU_DD_DIRAC
+#endif
+
 #cmakedefine QUDA_COVDEV
 #ifdef QUDA_COVDEV
 /**

From d4c448ad08873c9a5e3dafafdef0a6b1748b7f23 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Tue, 7 May 2024 12:26:57 +0200
Subject: [PATCH 014/108] Adding instantiation over DDArg

---
 include/dslash_helper.cuh    |  8 ++++----
 include/enum_quda.h          |  2 ++
 include/instantiate_dslash.h | 24 +++++++++++++++++++++++-
 3 files changed, 29 insertions(+), 5 deletions(-)

diff --git a/include/dslash_helper.cuh b/include/dslash_helper.cuh
index bcd7bc6dce..ee3846e344 100644
--- a/include/dslash_helper.cuh
+++ b/include/dslash_helper.cuh
@@ -242,7 +242,7 @@ namespace quda
     return true;
   }
 
-  template <typename Float_, int nDim_> struct DslashArg {
+  template <typename Float_, int nDim_, typename DDArg> struct DslashArg {
 
     using Float = Float_;
     using real = typename mapper<Float>::type;
@@ -287,9 +287,9 @@ namespace quda
     int exterior_dims; // dimension to run in the exterior Dslash
     int exterior_blocks;
 
-    DDParam<int_fastdiv> dd_out;
-    DDParam<int_fastdiv> dd_in;
-    DDParam<int_fastdiv> dd_x;
+    DDArg dd_out;
+    DDArg dd_in;
+    DDArg dd_x;
 
     // for shmem ...
     static constexpr bool packkernel = false;
diff --git a/include/enum_quda.h b/include/enum_quda.h
index 60c186eb77..3548b8e8a3 100644
--- a/include/enum_quda.h
+++ b/include/enum_quda.h
@@ -590,6 +590,8 @@ typedef enum QudaExtLibType_s {
   QUDA_EXTLIB_INVALID = QUDA_INVALID_ENUM
 } QudaExtLibType;
 
+typedef enum QudaDDType_s { QUDA_DD_NO, QUDA_DD_RED_BLACK, QUDA_DD_INVALID = QUDA_INVALID_ENUM } QudaDDType;
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/include/instantiate_dslash.h b/include/instantiate_dslash.h
index d9c8144de4..1c96e2e7ea 100644
--- a/include/instantiate_dslash.h
+++ b/include/instantiate_dslash.h
@@ -17,7 +17,7 @@ namespace quda
      @param[in] args Additional arguments for different dslash kernels
   */
   template <template <typename, int, QudaReconstructType> class Apply, typename Recon, typename Float, int nColor,
-            typename... Args>
+            typename DDArg, typename... Args>
   inline void instantiate(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, Args &&...args)
   {
     if (U.Reconstruct() == Recon::recon[0]) {
@@ -43,6 +43,28 @@ namespace quda
     }
   }
 
+  /**
+     @brief This instantiate function is used to instantiate the domain decomposition type
+     @param[out] out Output result field
+     @param[in] in Input field
+     @param[in] U Gauge field
+     @param[in] args Additional arguments for different dslash kernels
+  */
+  template <template <typename, int, QudaReconstructType> class Apply, typename Recon, typename Float, int nColor,
+            typename... Args>
+  inline void instantiate(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, Args &&...args)
+  {
+    if (out.dd.type == QUDA_DD_NO) {
+      instantiate<Apply, Recon, Float, 3, DDNoArg>(out, in, U, args...);
+#ifdef GPU_DD_DIRAC
+    } else if (out.dd.type == QUDA_DD_RED_BLACK) {
+      instantiate<Apply, Recon, Float, 3, DDRedBlackArg>(out, in, U, args...);
+#endif
+    } else {
+      errorQuda("Unsupported DD type %d\n", out.dd.type);
+    }
+  }
+
   /**
      @brief This instantiate function is used to instantiate the colors
      @param[out] out Output result field

From b21d9c9d27dce12eb143f6b374647b579dffd0de Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Tue, 7 May 2024 14:28:22 +0200
Subject: [PATCH 015/108] Cleaning DDParam

---
 include/domain_decomposition.h | 111 +++++++--------------------------
 1 file changed, 23 insertions(+), 88 deletions(-)

diff --git a/include/domain_decomposition.h b/include/domain_decomposition.h
index a8d4cc3258..2748569d79 100644
--- a/include/domain_decomposition.h
+++ b/include/domain_decomposition.h
@@ -1,6 +1,5 @@
 #pragma once
 
-#include <fast_intdiv.h>
 #include "declare_enum.h"
 
 namespace quda
@@ -8,50 +7,40 @@ namespace quda
 
   // using namespace quda;
 
-  DECLARE_ENUM(DD,     // name of the enum class
-               in_use, // whether DD is in use
+  DECLARE_ENUM(DD, // name of the enum class
 
-               mode_red_black, // red-black DD, e.g. for SAP
+               red_black_type, // Flags used by red_black
                red_active,     // if red blocks are active
                black_active,   // if black blocks are active
-               rb_hopping,     // if hopping between red and black is allowed
-               first_black,    // the color of the first block of the local lattice
+               block_hopping,  // if hopping between red and black is allowed
   );
 
   // Params for domain decompation
-  // we template over Int such that int_fastdiv can be used in kernels
-  template <typename Int = int> class DDParam
-  {
+  struct DDParam {
 
-  public:
+    QudaDDType type;
     bool flags[(int)DD::size];  // the default value of all flags is 0
-    Int blockDim[QUDA_MAX_DIM]; // the size of the block per direction
+    int blockDim[4];            // the size of the block per direction
 
     // Default constructor
-    DDParam() : flags {0}, blockDim {0} { }
+    DDParam() : type(QUDA_DD_NO), flags {0}, blockDim {0} { }
 
     // Default copy
     DDParam(const DDParam &dd) = default;
 
-    // Default copy
-    template <typename T> DDParam(const DDParam<T> &dd)
-    {
-#pragma unroll
-      for (auto i = 0u; i < (int)DD::size; i++) flags[i] = dd.flags[i];
-#pragma unroll
-      for (auto i = 0u; i < QUDA_MAX_DIM; i++) blockDim[i] = dd.blockDim[i];
-    }
-
     // returns false if in use
-    constexpr inline bool operator!() const { return not flags[(int)DD::in_use]; }
-
-    // returns value of given flag
-    constexpr inline bool operator&&(const DD &flag) const { return flags[(int)DD::in_use] && flags[(int)flag]; }
+    constexpr inline bool operator!() const { return type == QUDA_DD_NO; }
 
     // returns value of given flag
     constexpr inline bool is(const DD &flag) const { return flags[(int)flag]; }
 
-    constexpr inline void set(const DD &flag) { flags[(int)flag] = true; }
+    // sets given flag to true
+    constexpr inline void set(const DD &flag)
+    {
+      flags[(int)flag] = true;
+
+      if ((int)flag >= (int)DD::red_black_type) type = QUDA_DD_RED_BLACK;
+    }
 
     template <typename... Args> constexpr inline void set(const DD &flag, Args... args)
     {
@@ -61,6 +50,7 @@ namespace quda
 
     constexpr inline void reset()
     {
+      type = QUDA_DD_NO;
 #pragma unroll
       for (auto i = 0u; i < (int)DD::size; i++) flags[i] = 0;
     }
@@ -75,7 +65,7 @@ namespace quda
     void print()
     {
       if (not *this) {
-        printfQuda("DDParam not in use\n");
+        printfQuda("DD not in use\n");
         return;
       }
       printfQuda("Printing DDParam\n");
@@ -84,6 +74,7 @@ namespace quda
       for (int i = 0; i < QUDA_MAX_DIM; i++) printfQuda("blockDim[%d] = %d\n", i, static_cast<int>(blockDim[i]));
     }
 
+    // Checks if this matches to given DDParam
     // template <bool debug = false, typename T> // if true, prints debug information
     inline bool match(const DDParam &dd) const
     {
@@ -91,77 +82,21 @@ namespace quda
       // if one of the two is not in use we return true, i.e. one of the two is a full field
       if (not *this or not dd) return true;
 
-      // false if only one is red_black
-      if (is(DD::mode_red_black) ^ dd.is(DD::mode_red_black)) {
-        if (debug) printfQuda("Only one of the two is red_black\n");
+      // false if type does not match
+      if (type != dd.type) {
+        if (debug) printfQuda("DD type do not match (%d != %d)\n", type, dd.type);
         return false;
       }
 
-      if (is(DD::mode_red_black) and dd.is(DD::mode_red_black))
+      if (type == QUDA_DD_RED_BLACK)
         for (int i = 0; i < QUDA_MAX_DIM; i++)
           if (blockDim[i] != dd.blockDim[i]) {
-            if (debug)
-              printfQuda("blockDim[%d] = %d != %d \n", i, static_cast<int>(blockDim[i]),
-                         static_cast<int>(dd.blockDim[i]));
+            if (debug) printfQuda("blockDim[%d] = %d != %d \n", i, blockDim[i], dd.blockDim[i]);
             return false;
           }
 
       return true;
     }
-
-    /* if a field is zero at given coordinates */
-    template <typename Coord, typename Arg> constexpr inline bool isZero(const Coord &x, const Arg &arg) const
-    {
-
-      // if DD not in use we return immidiatelly
-      if (not is(DD::in_use)) return false;
-
-      if (is(DD::mode_red_black)) {
-
-        // Computing block_parity: 0 = red, 1 = black
-        int block_parity = is(DD::first_black);
-        for (int i = 0; i < x.size(); i++) { block_parity += x[i] / blockDim[i]; }
-        block_parity %= 2;
-
-        // Checking if my parity is active
-        if (not is(DD::red_active) and block_parity == 0) return true;
-        if (not is(DD::black_active) and block_parity == 1) return true;
-      }
-
-      return false;
-    }
-
-    /* if hopping is allowed */
-    template <typename Coord, typename Arg>
-    constexpr inline bool doHopping(const Coord &x, const int &mu, const int &dir, const Arg &arg) const
-    {
-
-      // if DD not in use we return immidiatelly
-      if (not is(DD::in_use)) return true;
-
-      if (is(DD::mode_red_black)) {
-
-        // Checking if we are on the border
-        bool on_border = (dir > 0) ? ((x[mu] + 1) % blockDim[mu] == 0) : (x[mu] % blockDim[mu] == 0);
-
-        // If all the follwing are true then it is a full operator
-        if (is(DD::red_active) and is(DD::black_active)) {
-          if (not on_border || is(DD::rb_hopping)) return true;
-          return false;
-        }
-
-        // Computing block_parity: 0 = red, 1 = black
-        int block_parity = is(DD::first_black) + on_border;
-        for (int i = 0; i < x.size(); i++) { block_parity += x[i] / blockDim[i]; }
-        block_parity %= 2;
-
-        // Checking if my parity is active
-        if (is(DD::red_active) and block_parity == 0) return true;
-        if (is(DD::black_active) and block_parity == 1) return true;
-      }
-
-      return false;
-    }
   };
 
 } // namespace quda

From f13ca712dd4199fa1e540541a3193539a8fd4d27 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Tue, 7 May 2024 14:28:43 +0200
Subject: [PATCH 016/108] Adding DDArg

---
 include/domain_decomposition_helper.cuh | 96 +++++++++++++++++++++++++
 include/dslash_helper.cuh               |  8 +--
 include/kernels/dslash_wilson.cuh       | 10 +--
 3 files changed, 105 insertions(+), 9 deletions(-)
 create mode 100644 include/domain_decomposition_helper.cuh

diff --git a/include/domain_decomposition_helper.cuh b/include/domain_decomposition_helper.cuh
new file mode 100644
index 0000000000..48c1b12299
--- /dev/null
+++ b/include/domain_decomposition_helper.cuh
@@ -0,0 +1,96 @@
+#pragma once
+
+#include <fast_intdiv.h>
+#include <color_spinor_field.h>
+
+namespace quda
+{
+
+  struct DDArg {
+
+    // Initialization of input parameters from ColorSpinorField
+    virtual DDArg(const ColorSpinorField &in);
+
+    // Whether field at given coord is zero
+    template <typename Coord> virtual constexpr inline bool isZero(const Coord &x) const;
+
+    // Whether do hopping with field at neighboring coord
+    template <typename Coord>
+    virtual constexpr inline bool doHopping(const Coord &x, const int &mu, const int &dir) const;
+  };
+
+  struct DDNoArg : public DDArg {
+
+    DDArg(const ColorSpinorField &in) { assert(in.dd.type == QUDA_DD_NO); }
+
+    constexpr inline bool isZero(const Coord &x) const { return false; }
+
+    constexpr inline bool doHopping(const Coord &x, const int &mu, const int &dir) const { return true; }
+  };
+
+  struct DDRedBlackArg : public DDArg {
+
+    const int_fastdiv blockDim[4]; // the size of the block per direction
+    const bool red_active;         // if red blocks are active
+    const bool black_active;       // if black blocks are active
+    const bool block_hopping;      // if hopping between red and black is allowed
+    const bool first_black;        // if the first block of the local lattice is black instead of red
+
+    DDArg(const ColorSpinorField &in) :
+      blockDim {in.dd.blockDim(0), in.dd.blockDim(1), in.dd.blockDim(2), in.dd.blockDim(3)},
+      red_active(in.dd.type == QUDA_DD_NO or in.dd.is(DD::red_active)),
+      black_active(in.dd.type == QUDA_DD_NO or in.dd.is(DD::black_active)),
+      block_hopping(in.dd.type == QUDA_DD_NO or in.dd.is(DD::block_hopping)),
+      first_black(false) // TODO
+    {
+      assert(in.dd.type == QUDA_DD_NO or in.dd.type == QUDA_DD_RED_BLACK);
+    }
+
+    // Computes block_parity: 0 = red, 1 = black
+    inline bool block_parity(const Coord &x) const
+    {
+      int block_parity = first_black;
+      for (int i = 0; i < x.size(); i++) { block_parity += x[i] / blockDim[i]; }
+      return block_parity % 2 == 0;
+    }
+
+    inline bool on_border(const Coord &x, const int &mu, const int &dir) const
+    {
+      return (dir > 0) ? ((x[mu] + 1) % blockDim[mu] == 0) : (x[mu] % blockDim[mu] == 0);
+    }
+
+    constexpr inline bool isZero(const Coord &x) const
+    {
+      if (red_active and black_active) return false;
+      if (not red_active and not black_active) return true;
+
+      bool is_black = block_parity(x);
+
+      // Checking if my parity is active
+      if (red_active and not is_black) return false;
+      if (black_active and is_black) return false;
+
+      return true;
+    }
+
+    constexpr inline bool doHopping(const Coord &x, const int &mu, const int &dir) const
+    {
+      if (red_active and black_active and blockhopping) return true;
+      if (not red_active and not black_active) return false;
+
+      bool swap = on_border(x, mu, dir);
+      if (swap and not block_hopping) return false;
+      if (not swap and red_active and black_active) return true;
+
+      // Neighbor color
+      bool is_black = block_parity(x) ^ swap;
+
+      // Checking if neighbor is active
+      if (red_active and not is_black) return true;
+      if (black_active and is_black) return true;
+
+      return false;
+    }
+  };
+
+} // namespace quda
diff --git a/include/dslash_helper.cuh b/include/dslash_helper.cuh
index ee3846e344..0d9ea3ed98 100644
--- a/include/dslash_helper.cuh
+++ b/include/dslash_helper.cuh
@@ -11,7 +11,7 @@
 #include <shmem_pack_helper.cuh>
 #include <kernel_helper.h>
 #include <tune_quda.h>
-#include <domain_decomposition.h>
+#include <domain_decomposition_helper.cuh>
 
 #if defined(_NVHPC_CUDA)
 #include <constant_kernel_arg.h>
@@ -342,9 +342,9 @@ namespace quda
       pack_blocks(0),
       exterior_dims(0),
       exterior_blocks(0),
-      dd_out(out.dd),
-      dd_in(in.dd),
-      dd_x(x.dd),
+      dd_out(out),
+      dd_in(in),
+      dd_x(x),
 #ifndef NVSHMEM_COMMS
       counter(0)
 #else
diff --git a/include/kernels/dslash_wilson.cuh b/include/kernels/dslash_wilson.cuh
index c5a5b03303..1c1982288e 100644
--- a/include/kernels/dslash_wilson.cuh
+++ b/include/kernels/dslash_wilson.cuh
@@ -74,7 +74,7 @@ namespace quda
 #pragma unroll
     for (int d = 0; d < 4; d++) { // loop over dimension - 4 and not nDim since this is used for DWF as well
       // Forward gather - compute fwd offset for vector fetch
-      if (arg.dd_in.doHopping(coord, d, +1, arg.dc)) {
+      if (arg.dd_in.doHopping(coord, d, +1)) {
         const int fwd_idx = getNeighborIndexCB(coord, d, +1, arg.dc);
         const int gauge_idx = (Arg::nDim == 5 ? coord.x_cb % arg.dc.volume_4d_cb : coord.x_cb);
         constexpr int proj_dir = dagger ? +1 : -1;
@@ -101,7 +101,7 @@ namespace quda
       }
 
       // Backward gather - compute back offset for spinor and gauge fetch
-      if (arg.dd_in.doHopping(coord, d, -1, arg.dc)) {
+      if (arg.dd_in.doHopping(coord, d, -1)) {
         const int back_idx = getNeighborIndexCB(coord, d, -1, arg.dc);
         const int gauge_idx = (Arg::nDim == 5 ? back_idx % arg.dc.volume_4d_cb : back_idx);
         constexpr int proj_dir = dagger ? -1 : +1;
@@ -152,7 +152,7 @@ namespace quda
       Vector out;
 
       int xs = coord.x_cb + coord.s * arg.dc.volume_4d_cb;
-      if (arg.dd_out.isZero(coord, arg.dc)) {
+      if (arg.dd_out.isZero(coord)) {
         if (mykernel_type != EXTERIOR_KERNEL_ALL || active) arg.out(xs, my_spinor_parity) = out;
         return;
       }
@@ -160,14 +160,14 @@ namespace quda
       applyWilson<nParity, dagger, mykernel_type>(out, arg, coord, parity, idx, thread_dim, active);
 
       if (xpay && mykernel_type == INTERIOR_KERNEL) {
-        if (arg.dd_x.isZero(coord, arg.dc)) {
+        if (arg.dd_x.isZero(coord)) {
           out = arg.a * out;
         } else {
           Vector x = arg.x(xs, my_spinor_parity);
           out = x + arg.a * out;
         }
       } else if (mykernel_type != INTERIOR_KERNEL && active) {
-        if (arg.dd_x.isZero(coord, arg.dc)) {
+        if (arg.dd_x.isZero(coord)) {
           out = (xpay ? arg.a * out : out);
         } else {
           Vector x = arg.out(xs, my_spinor_parity);

From aabb473ef904c0776db68ac142106954ef5c22f7 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Tue, 7 May 2024 14:37:13 +0200
Subject: [PATCH 017/108] Resolving compilation errors

---
 include/color_spinor_field.h   | 5 ++++-
 include/domain_decomposition.h | 2 +-
 include/lattice_field.h        | 3 ---
 lib/inv_mr_quda.cpp            | 8 ++++----
 4 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/include/color_spinor_field.h b/include/color_spinor_field.h
index cbc41eae7e..83b6abbf1f 100644
--- a/include/color_spinor_field.h
+++ b/include/color_spinor_field.h
@@ -149,7 +149,7 @@ namespace quda
     int composite_dim = 0; // e.g., number of eigenvectors in the set
     bool is_component = false;
     int component_id = 0; // eigenvector index
-    DDParam<int> dd {};
+    DDParam dd {};
 
     /**
        If using CUDA native fields, this function will ensure that the
@@ -385,6 +385,9 @@ namespace quda
     inline static size_t ghostFaceBytes[QUDA_MAX_DIM] = {};
     static void freeGhostBuffer(void);
 
+    /** Domain decomposition options */
+    DDParam dd {};
+
     /**
        @brief Default constructor
     */
diff --git a/include/domain_decomposition.h b/include/domain_decomposition.h
index 2748569d79..f86ac91023 100644
--- a/include/domain_decomposition.h
+++ b/include/domain_decomposition.h
@@ -58,7 +58,7 @@ namespace quda
     template <typename... Args> constexpr inline void reset(Args... args)
     {
       reset();
-      set(DD::in_use, args...);
+      set(args...);
     }
 
     // Pretty print the args struct
diff --git a/include/lattice_field.h b/include/lattice_field.h
index 0e044fe330..fa1c37a65e 100644
--- a/include/lattice_field.h
+++ b/include/lattice_field.h
@@ -479,9 +479,6 @@ namespace quda {
      */
     inline static int bufferIndex = 0;
 
-    /** Domain decomposition options */
-    DDParam<int> dd {};
-
     /**
        @brief Default constructor
     */
diff --git a/lib/inv_mr_quda.cpp b/lib/inv_mr_quda.cpp
index 3e744fa2c6..5626876c18 100644
--- a/lib/inv_mr_quda.cpp
+++ b/lib/inv_mr_quda.cpp
@@ -98,12 +98,12 @@ namespace quda
         if (param.do_block_schwarz()) {
           if (param.schwarz_type == QUDA_MULTIPLICATIVE_SCHWARZ) {
             // Red or black active
-            Ar.dd.reset(DD::mode_red_black, step % 2 == 0 ? DD::red_active : DD::black_active);
-            r_sloppy.dd.reset(DD::mode_red_black, step % 2 == 0 ? DD::red_active : DD::black_active);
+            Ar.dd.reset(DD::red_black_type, step % 2 == 0 ? DD::red_active : DD::black_active);
+            r_sloppy.dd.reset(DD::red_black_type, step % 2 == 0 ? DD::red_active : DD::black_active);
           } else {
             // Both red and black active but no hopping
-            Ar.dd.reset(DD::mode_red_black, DD::red_active, DD::black_active);
-            r_sloppy.dd.reset(DD::mode_red_black, DD::red_active, DD::black_active);
+            Ar.dd.reset(DD::red_black_type, DD::red_active, DD::black_active);
+            r_sloppy.dd.reset(DD::red_black_type, DD::red_active, DD::black_active);
           }
         }
 

From 328389b7e319cf8bc2c4312849253e39a6613a4d Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Tue, 7 May 2024 14:57:49 +0200
Subject: [PATCH 018/108] Adding constexpr

---
 include/domain_decomposition_helper.cuh | 14 +++++++-------
 include/kernels/dslash_wilson.cuh       | 21 ++++++++++++---------
 2 files changed, 19 insertions(+), 16 deletions(-)

diff --git a/include/domain_decomposition_helper.cuh b/include/domain_decomposition_helper.cuh
index 48c1b12299..3df62ecb53 100644
--- a/include/domain_decomposition_helper.cuh
+++ b/include/domain_decomposition_helper.cuh
@@ -6,20 +6,20 @@
 namespace quda
 {
 
+  // Template structure
   struct DDArg {
 
     // Initialization of input parameters from ColorSpinorField
     virtual DDArg(const ColorSpinorField &in);
 
     // Whether field at given coord is zero
-    template <typename Coord> virtual constexpr inline bool isZero(const Coord &x) const;
+    template <typename Coord> virtual inline bool isZero(const Coord &x) const;
 
     // Whether do hopping with field at neighboring coord
-    template <typename Coord>
-    virtual constexpr inline bool doHopping(const Coord &x, const int &mu, const int &dir) const;
+    template <typename Coord> virtual inline bool doHopping(const Coord &x, const int &mu, const int &dir) const;
   };
 
-  struct DDNoArg : public DDArg {
+  struct DDNo : public DDArg {
 
     DDArg(const ColorSpinorField &in) { assert(in.dd.type == QUDA_DD_NO); }
 
@@ -28,7 +28,7 @@ namespace quda
     constexpr inline bool doHopping(const Coord &x, const int &mu, const int &dir) const { return true; }
   };
 
-  struct DDRedBlackArg : public DDArg {
+  struct DDRedBlack : public DDArg {
 
     const int_fastdiv blockDim[4]; // the size of the block per direction
     const bool red_active;         // if red blocks are active
@@ -59,7 +59,7 @@ namespace quda
       return (dir > 0) ? ((x[mu] + 1) % blockDim[mu] == 0) : (x[mu] % blockDim[mu] == 0);
     }
 
-    constexpr inline bool isZero(const Coord &x) const
+    inline bool isZero(const Coord &x) const
     {
       if (red_active and black_active) return false;
       if (not red_active and not black_active) return true;
@@ -73,7 +73,7 @@ namespace quda
       return true;
     }
 
-    constexpr inline bool doHopping(const Coord &x, const int &mu, const int &dir) const
+    inline bool doHopping(const Coord &x, const int &mu, const int &dir) const
     {
       if (red_active and black_active and blockhopping) return true;
       if (not red_active and not black_active) return false;
diff --git a/include/kernels/dslash_wilson.cuh b/include/kernels/dslash_wilson.cuh
index 1c1982288e..81d14219dd 100644
--- a/include/kernels/dslash_wilson.cuh
+++ b/include/kernels/dslash_wilson.cuh
@@ -74,7 +74,8 @@ namespace quda
 #pragma unroll
     for (int d = 0; d < 4; d++) { // loop over dimension - 4 and not nDim since this is used for DWF as well
       // Forward gather - compute fwd offset for vector fetch
-      if (arg.dd_in.doHopping(coord, d, +1)) {
+      constexpr if (arg.dd_in.doHopping(coord, d, +1))
+      {
         const int fwd_idx = getNeighborIndexCB(coord, d, +1, arg.dc);
         const int gauge_idx = (Arg::nDim == 5 ? coord.x_cb % arg.dc.volume_4d_cb : coord.x_cb);
         constexpr int proj_dir = dagger ? +1 : -1;
@@ -101,7 +102,8 @@ namespace quda
       }
 
       // Backward gather - compute back offset for spinor and gauge fetch
-      if (arg.dd_in.doHopping(coord, d, -1)) {
+      constexpr if (arg.dd_in.doHopping(coord, d, -1))
+      {
         const int back_idx = getNeighborIndexCB(coord, d, -1, arg.dc);
         const int gauge_idx = (Arg::nDim == 5 ? back_idx % arg.dc.volume_4d_cb : back_idx);
         constexpr int proj_dir = dagger ? -1 : +1;
@@ -152,7 +154,8 @@ namespace quda
       Vector out;
 
       int xs = coord.x_cb + coord.s * arg.dc.volume_4d_cb;
-      if (arg.dd_out.isZero(coord)) {
+      constexpr if (arg.dd_out.isZero(coord))
+      {
         if (mykernel_type != EXTERIOR_KERNEL_ALL || active) arg.out(xs, my_spinor_parity) = out;
         return;
       }
@@ -160,16 +163,16 @@ namespace quda
       applyWilson<nParity, dagger, mykernel_type>(out, arg, coord, parity, idx, thread_dim, active);
 
       if (xpay && mykernel_type == INTERIOR_KERNEL) {
-        if (arg.dd_x.isZero(coord)) {
-          out = arg.a * out;
-        } else {
+        constexpr if (arg.dd_x.isZero(coord)) { out = arg.a * out; }
+        else
+        {
           Vector x = arg.x(xs, my_spinor_parity);
           out = x + arg.a * out;
         }
       } else if (mykernel_type != INTERIOR_KERNEL && active) {
-        if (arg.dd_x.isZero(coord)) {
-          out = (xpay ? arg.a * out : out);
-        } else {
+        constexpr if (arg.dd_x.isZero(coord)) { out = (xpay ? arg.a * out : out); }
+        else
+        {
           Vector x = arg.out(xs, my_spinor_parity);
           out = x + (xpay ? arg.a * out : out);
         }

From fee22aca3890b22bfd76ca5d48fdd653af1cd08f Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Tue, 7 May 2024 17:22:06 +0200
Subject: [PATCH 019/108] Removing base struct DDArg

---
 include/domain_decomposition_helper.cuh | 46 ++++++++++++-------------
 1 file changed, 23 insertions(+), 23 deletions(-)

diff --git a/include/domain_decomposition_helper.cuh b/include/domain_decomposition_helper.cuh
index 3df62ecb53..a0e739e4b2 100644
--- a/include/domain_decomposition_helper.cuh
+++ b/include/domain_decomposition_helper.cuh
@@ -6,29 +6,27 @@
 namespace quda
 {
 
-  // Template structure
-  struct DDArg {
+  // No DD (use also as a template for required functions)
+  struct DDNo {
 
     // Initialization of input parameters from ColorSpinorField
-    virtual DDArg(const ColorSpinorField &in);
+    DDNo(const ColorSpinorField &in)
+    {
+      if (in.dd.type != QUDA_DD_NO) { errorQuda("Unsupported type %d\n", in.dd.type); }
+    }
 
     // Whether field at given coord is zero
-    template <typename Coord> virtual inline bool isZero(const Coord &x) const;
+    template <typename Coord> constexpr inline bool isZero(const Coord &) const { return false; }
 
     // Whether do hopping with field at neighboring coord
-    template <typename Coord> virtual inline bool doHopping(const Coord &x, const int &mu, const int &dir) const;
-  };
-
-  struct DDNo : public DDArg {
-
-    DDArg(const ColorSpinorField &in) { assert(in.dd.type == QUDA_DD_NO); }
-
-    constexpr inline bool isZero(const Coord &x) const { return false; }
-
-    constexpr inline bool doHopping(const Coord &x, const int &mu, const int &dir) const { return true; }
+    template <typename Coord> constexpr inline bool doHopping(const Coord &, const int &, const int &) const
+    {
+      return true;
+    }
   };
 
-  struct DDRedBlack : public DDArg {
+  // Red-black Block DD
+  struct DDRedBlack {
 
     const int_fastdiv blockDim[4]; // the size of the block per direction
     const bool red_active;         // if red blocks are active
@@ -36,30 +34,32 @@ namespace quda
     const bool block_hopping;      // if hopping between red and black is allowed
     const bool first_black;        // if the first block of the local lattice is black instead of red
 
-    DDArg(const ColorSpinorField &in) :
-      blockDim {in.dd.blockDim(0), in.dd.blockDim(1), in.dd.blockDim(2), in.dd.blockDim(3)},
+    DDRedBlack(const ColorSpinorField &in) :
+      blockDim {in.dd.blockDim[0], in.dd.blockDim[1], in.dd.blockDim[2], in.dd.blockDim[3]},
       red_active(in.dd.type == QUDA_DD_NO or in.dd.is(DD::red_active)),
       black_active(in.dd.type == QUDA_DD_NO or in.dd.is(DD::black_active)),
       block_hopping(in.dd.type == QUDA_DD_NO or in.dd.is(DD::block_hopping)),
       first_black(false) // TODO
     {
-      assert(in.dd.type == QUDA_DD_NO or in.dd.type == QUDA_DD_RED_BLACK);
+      if (in.dd.type != QUDA_DD_NO and in.dd.type != QUDA_DD_RED_BLACK) {
+        errorQuda("Unsupported type %d\n", in.dd.type);
+      }
     }
 
     // Computes block_parity: 0 = red, 1 = black
-    inline bool block_parity(const Coord &x) const
+    template <typename Coord> inline bool block_parity(const Coord &x) const
     {
       int block_parity = first_black;
       for (int i = 0; i < x.size(); i++) { block_parity += x[i] / blockDim[i]; }
       return block_parity % 2 == 0;
     }
 
-    inline bool on_border(const Coord &x, const int &mu, const int &dir) const
+    template <typename Coord> inline bool on_border(const Coord &x, const int &mu, const int &dir) const
     {
       return (dir > 0) ? ((x[mu] + 1) % blockDim[mu] == 0) : (x[mu] % blockDim[mu] == 0);
     }
 
-    inline bool isZero(const Coord &x) const
+    template <typename Coord> inline bool isZero(const Coord &x) const
     {
       if (red_active and black_active) return false;
       if (not red_active and not black_active) return true;
@@ -73,9 +73,9 @@ namespace quda
       return true;
     }
 
-    inline bool doHopping(const Coord &x, const int &mu, const int &dir) const
+    template <typename Coord> inline bool doHopping(const Coord &x, const int &mu, const int &dir) const
     {
-      if (red_active and black_active and blockhopping) return true;
+      if (red_active and black_active and block_hopping) return true;
       if (not red_active and not black_active) return false;
 
       bool swap = on_border(x, mu, dir);

From e1e0dbb66b82ecb9268ad91cb10e79156f9ba04b Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Tue, 7 May 2024 17:22:46 +0200
Subject: [PATCH 020/108] Fixing wrong use of constexpr

---
 include/kernels/dslash_wilson.cuh | 28 +++++++++++++---------------
 1 file changed, 13 insertions(+), 15 deletions(-)

diff --git a/include/kernels/dslash_wilson.cuh b/include/kernels/dslash_wilson.cuh
index 81d14219dd..4b62a2c96d 100644
--- a/include/kernels/dslash_wilson.cuh
+++ b/include/kernels/dslash_wilson.cuh
@@ -14,8 +14,8 @@ namespace quda
   /**
      @brief Parameter structure for driving the Wilson operator
    */
-  template <typename Float, int nColor_, int nDim, QudaReconstructType reconstruct_>
-  struct WilsonArg : DslashArg<Float, nDim> {
+  template <typename Float, int nColor_, int nDim, typename DDArg, QudaReconstructType reconstruct_>
+  struct WilsonArg : DslashArg<Float, nDim, DDArg> {
     static constexpr int nColor = nColor_;
     static constexpr int nSpin = 4;
     static constexpr bool spin_project = true;
@@ -38,7 +38,8 @@ namespace quda
 
     WilsonArg(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, double a,
               const ColorSpinorField &x, int parity, bool dagger, const int *comm_override) :
-      DslashArg<Float, nDim>(out, in, U, x, parity, dagger, a != 0.0 ? true : false, 1, spin_project, comm_override),
+      DslashArg<Float, nDim, DDArg>(out, in, U, x, parity, dagger, a != 0.0 ? true : false, 1, spin_project,
+                                    comm_override),
       out(out),
       in(in),
       in_pack(in),
@@ -74,8 +75,7 @@ namespace quda
 #pragma unroll
     for (int d = 0; d < 4; d++) { // loop over dimension - 4 and not nDim since this is used for DWF as well
       // Forward gather - compute fwd offset for vector fetch
-      constexpr if (arg.dd_in.doHopping(coord, d, +1))
-      {
+      if (arg.dd_in.doHopping(coord, d, +1)) {
         const int fwd_idx = getNeighborIndexCB(coord, d, +1, arg.dc);
         const int gauge_idx = (Arg::nDim == 5 ? coord.x_cb % arg.dc.volume_4d_cb : coord.x_cb);
         constexpr int proj_dir = dagger ? +1 : -1;
@@ -102,8 +102,7 @@ namespace quda
       }
 
       // Backward gather - compute back offset for spinor and gauge fetch
-      constexpr if (arg.dd_in.doHopping(coord, d, -1))
-      {
+      if (arg.dd_in.doHopping(coord, d, -1)) {
         const int back_idx = getNeighborIndexCB(coord, d, -1, arg.dc);
         const int gauge_idx = (Arg::nDim == 5 ? back_idx % arg.dc.volume_4d_cb : back_idx);
         constexpr int proj_dir = dagger ? -1 : +1;
@@ -154,8 +153,7 @@ namespace quda
       Vector out;
 
       int xs = coord.x_cb + coord.s * arg.dc.volume_4d_cb;
-      constexpr if (arg.dd_out.isZero(coord))
-      {
+      if (arg.dd_out.isZero(coord)) {
         if (mykernel_type != EXTERIOR_KERNEL_ALL || active) arg.out(xs, my_spinor_parity) = out;
         return;
       }
@@ -163,16 +161,16 @@ namespace quda
       applyWilson<nParity, dagger, mykernel_type>(out, arg, coord, parity, idx, thread_dim, active);
 
       if (xpay && mykernel_type == INTERIOR_KERNEL) {
-        constexpr if (arg.dd_x.isZero(coord)) { out = arg.a * out; }
-        else
-        {
+        if (arg.dd_x.isZero(coord)) {
+          out = arg.a * out;
+        } else {
           Vector x = arg.x(xs, my_spinor_parity);
           out = x + arg.a * out;
         }
       } else if (mykernel_type != INTERIOR_KERNEL && active) {
-        constexpr if (arg.dd_x.isZero(coord)) { out = (xpay ? arg.a * out : out); }
-        else
-        {
+        if (arg.dd_x.isZero(coord)) {
+          out = (xpay ? arg.a * out : out);
+        } else {
           Vector x = arg.out(xs, my_spinor_parity);
           out = x + (xpay ? arg.a * out : out);
         }

From b1b3dd9fa02a80c93f1e030b613b8f6318699a62 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Tue, 7 May 2024 17:22:59 +0200
Subject: [PATCH 021/108] Adding DDArg in a number of places

---
 include/dslash_helper.cuh                     |  3 +-
 include/instantiate_dslash.h                  | 30 +++++++++-------
 include/kernels/covDev.cuh                    |  6 ++--
 include/kernels/dslash_domain_wall_4d.cuh     |  6 ++--
 .../dslash_domain_wall_4d_fused_m5.cuh        |  6 ++--
 include/kernels/dslash_domain_wall_5d.cuh     | 10 +++---
 .../kernels/dslash_ndeg_twisted_clover.cuh    | 32 ++++++++---------
 ...ash_ndeg_twisted_clover_preconditioned.cuh | 28 +++++++--------
 include/kernels/dslash_ndeg_twisted_mass.cuh  |  6 ++--
 ...slash_ndeg_twisted_mass_preconditioned.cuh | 12 +++----
 include/kernels/dslash_staggered.cuh          |  8 ++---
 .../dslash_twisted_clover_preconditioned.cuh  |  8 ++---
 include/kernels/dslash_twisted_mass.cuh       |  6 ++--
 .../dslash_twisted_mass_preconditioned.cuh    | 14 ++++----
 include/kernels/dslash_wilson_clover.cuh      |  8 ++---
 .../dslash_wilson_clover_hasenbusch_twist.cuh |  8 ++---
 ...clover_hasenbusch_twist_preconditioned.cuh |  8 ++---
 .../dslash_wilson_clover_preconditioned.cuh   |  8 ++---
 include/kernels/laplace.cuh                   |  6 ++--
 include/kernels/staggered_quark_smearing.cuh  |  6 ++--
 lib/covDev.cu                                 |  4 +--
 lib/dslash_domain_wall_4d.cu                  |  6 ++--
 lib/dslash_domain_wall_4d_fused_m5.hpp        |  4 +--
 lib/dslash_domain_wall_5d.cu                  |  5 +--
 lib/dslash_improved_staggered.cu              |  6 ++--
 lib/dslash_ndeg_twisted_clover.cu             | 33 ++++++++---------
 ...lash_ndeg_twisted_clover_preconditioned.cu | 36 ++++++++++---------
 lib/dslash_ndeg_twisted_mass.cu               |  4 +--
 ...dslash_ndeg_twisted_mass_preconditioned.cu |  9 +++--
 lib/dslash_twisted_clover.cu                  |  4 +--
 lib/dslash_twisted_clover_preconditioned.cu   |  6 ++--
 lib/dslash_twisted_mass.cu                    |  4 +--
 lib/dslash_twisted_mass_preconditioned.cu     |  9 +++--
 lib/dslash_wilson.cu                          |  4 +--
 lib/dslash_wilson_clover.cu                   |  8 ++---
 lib/dslash_wilson_clover_hasenbusch_twist.cu  |  6 ++--
 ..._clover_hasenbusch_twist_preconditioned.cu | 10 +++---
 lib/dslash_wilson_clover_preconditioned.cu    |  5 +--
 lib/laplace.cu                                |  5 +--
 lib/staggered_quark_smearing.cu               |  6 ++--
 40 files changed, 208 insertions(+), 185 deletions(-)

diff --git a/include/dslash_helper.cuh b/include/dslash_helper.cuh
index 0d9ea3ed98..354278bca4 100644
--- a/include/dslash_helper.cuh
+++ b/include/dslash_helper.cuh
@@ -422,7 +422,8 @@ namespace quda
     }
   };
 
-  template <typename Float, int nDim> std::ostream &operator<<(std::ostream &out, const DslashArg<Float, nDim> &arg)
+  template <typename Float, int nDim, typename DDArg>
+  std::ostream &operator<<(std::ostream &out, const DslashArg<Float, nDim, DDArg> &arg)
   {
     out << "parity = " << arg.parity << std::endl;
     out << "nParity = " << arg.nParity << std::endl;
diff --git a/include/instantiate_dslash.h b/include/instantiate_dslash.h
index 5a499cde8c..ac407892b5 100644
--- a/include/instantiate_dslash.h
+++ b/include/instantiate_dslash.h
@@ -5,6 +5,7 @@
 #include <color_spinor_field.h>
 #include <gauge_field.h>
 #include <instantiate.h>
+#include <domain_decomposition_helper.cuh>
 
 namespace quda
 {
@@ -16,25 +17,25 @@ namespace quda
      @param[in] U Gauge field
      @param[in] args Additional arguments for different dslash kernels
   */
-  template <template <typename, int, QudaReconstructType> class Apply, typename Recon, typename Float, int nColor,
-            typename DDArg, typename... Args>
+  template <template <typename, int, typename, QudaReconstructType> class Apply, typename Recon, typename Float,
+            int nColor, typename DDArg, typename... Args>
   inline void instantiate(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, Args &&...args)
   {
     if (U.Reconstruct() == Recon::recon[0]) {
 #if QUDA_RECONSTRUCT & 4
-      Apply<Float, nColor, Recon::recon[0]>(out, in, U, args...);
+      Apply<Float, nColor, DDArg, Recon::recon[0]>(out, in, U, args...);
 #else
       errorQuda("QUDA_RECONSTRUCT=%d does not enable reconstruct-18", QUDA_RECONSTRUCT);
 #endif
     } else if (U.Reconstruct() == Recon::recon[1]) {
 #if QUDA_RECONSTRUCT & 2
-      Apply<Float, nColor, Recon::recon[1]>(out, in, U, args...);
+      Apply<Float, nColor, DDArg, Recon::recon[1]>(out, in, U, args...);
 #else
       errorQuda("QUDA_RECONSTRUCT=%d does not enable reconstruct-12/13", QUDA_RECONSTRUCT);
 #endif
     } else if (U.Reconstruct() == Recon::recon[2]) {
 #if QUDA_RECONSTRUCT & 1
-      Apply<Float, nColor, Recon::recon[2]>(out, in, U, args...);
+      Apply<Float, nColor, DDArg, Recon::recon[2]>(out, in, U, args...);
 #else
       errorQuda("QUDA_RECONSTRUCT=%d does not enable reconstruct-8/9", QUDA_RECONSTRUCT);
 #endif
@@ -50,15 +51,15 @@ namespace quda
      @param[in] U Gauge field
      @param[in] args Additional arguments for different dslash kernels
   */
-  template <template <typename, int, QudaReconstructType> class Apply, typename Recon, typename Float, int nColor,
-            typename... Args>
+  template <template <typename, int, typename, QudaReconstructType> class Apply, typename Recon, typename Float,
+            int nColor, typename... Args>
   inline void instantiate(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, Args &&...args)
   {
     if (out.dd.type == QUDA_DD_NO) {
-      instantiate<Apply, Recon, Float, 3, DDNoArg>(out, in, U, args...);
+      instantiate<Apply, Recon, Float, 3, DDNo>(out, in, U, args...);
 #ifdef GPU_DD_DIRAC
     } else if (out.dd.type == QUDA_DD_RED_BLACK) {
-      instantiate<Apply, Recon, Float, 3, DDRedBlackArg>(out, in, U, args...);
+      instantiate<Apply, Recon, Float, 3, DDRedBlack>(out, in, U, args...);
 #endif
     } else {
       errorQuda("Unsupported DD type %d\n", out.dd.type);
@@ -72,7 +73,7 @@ namespace quda
      @param[in] U Gauge field
      @param[in] args Additional arguments for different dslash kernels
   */
-  template <template <typename, int, QudaReconstructType> class Apply, typename Recon, typename Float, typename... Args>
+  template <template <typename, int, typename, QudaReconstructType> class Apply, typename Recon, typename Float, typename... Args>
   inline void instantiate(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, Args &&...args)
   {
     if (in.Ncolor() == 3) {
@@ -89,7 +90,8 @@ namespace quda
      @param[in] U Gauge field
      @param[in] args Additional arguments for different dslash kernels
   */
-  template <template <typename, int, QudaReconstructType> class Apply, typename Recon = WilsonReconstruct, typename... Args>
+  template <template <typename, int, typename, QudaReconstructType> class Apply, typename Recon = WilsonReconstruct,
+            typename... Args>
   inline void instantiate(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, Args &&...args)
   {
     if (U.Precision() == QUDA_DOUBLE_PRECISION) {
@@ -132,7 +134,8 @@ namespace quda
      @param[in] args Additional arguments for different dslash kernels
   */
 #if (QUDA_PRECISION & 2) || (QUDA_PRECISION & 1)
-  template <template <typename, int, QudaReconstructType> class Apply, typename Recon = WilsonReconstruct, typename... Args>
+  template <template <typename, int, typename, QudaReconstructType> class Apply, typename Recon = WilsonReconstruct,
+            typename... Args>
   inline void instantiatePreconditioner(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U,
                                         Args &&...args)
   {
@@ -153,7 +156,8 @@ namespace quda
     }
   }
 #else
-  template <template <typename, int, QudaReconstructType> class Apply, typename Recon = WilsonReconstruct, typename... Args>
+  template <template <typename, int, typename, QudaReconstructType> class Apply, typename Recon = WilsonReconstruct,
+            typename... Args>
   inline void instantiatePreconditioner(ColorSpinorField &, const ColorSpinorField &, const GaugeField &U, Args &&...)
   {
     if (U.Precision() == QUDA_HALF_PRECISION) {
diff --git a/include/kernels/covDev.cuh b/include/kernels/covDev.cuh
index 28c52e9b38..e0420ff154 100644
--- a/include/kernels/covDev.cuh
+++ b/include/kernels/covDev.cuh
@@ -14,8 +14,8 @@ namespace quda
   /**
      @brief Parameter structure for driving the covariatnt derivative operator
   */
-  template <typename Float, int nColor_, QudaReconstructType reconstruct_, int nDim>
-  struct CovDevArg : DslashArg<Float, nDim> {
+  template <typename Float, int nColor_, typename DDArg, QudaReconstructType reconstruct_, int nDim>
+  struct CovDevArg : DslashArg<Float, nDim, DDArg> {
     static constexpr int nColor = nColor_;
     static constexpr int nSpin = 4;
     static constexpr bool spin_project = false;
@@ -37,7 +37,7 @@ namespace quda
 
     CovDevArg(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, int mu, int parity, bool dagger,
               const int *comm_override) :
-      DslashArg<Float, nDim>(out, in, U, in, parity, dagger, false, 1, spin_project, comm_override),
+      DslashArg<Float, nDim, DDArg>(out, in, U, in, parity, dagger, false, 1, spin_project, comm_override),
       out(out),
       in(in),
       in_pack(in),
diff --git a/include/kernels/dslash_domain_wall_4d.cuh b/include/kernels/dslash_domain_wall_4d.cuh
index 09a09a9f3e..e21375a7b1 100644
--- a/include/kernels/dslash_domain_wall_4d.cuh
+++ b/include/kernels/dslash_domain_wall_4d.cuh
@@ -5,8 +5,8 @@
 namespace quda
 {
 
-  template <typename Float, int nColor, int nDim, QudaReconstructType reconstruct_>
-  struct DomainWall4DArg : WilsonArg<Float, nColor, nDim, reconstruct_> {
+  template <typename Float, int nColor, int nDim, typename DDArg, QudaReconstructType reconstruct_>
+  struct DomainWall4DArg : WilsonArg<Float, nColor, nDim, DDArg, reconstruct_> {
     typedef typename mapper<Float>::type real;
     int Ls;                             /** fifth dimension length */
     complex<real> a_5[QUDA_MAX_DWF_LS]; /** xpay scale factor for each 4-d subvolume */
@@ -14,7 +14,7 @@ namespace quda
     DomainWall4DArg(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, double a, double m_5,
                     const Complex *b_5, const Complex *c_5, bool xpay, const ColorSpinorField &x, int parity,
                     bool dagger, const int *comm_override) :
-      WilsonArg<Float, nColor, nDim, reconstruct_>(out, in, U, xpay ? a : 0.0, x, parity, dagger, comm_override),
+      WilsonArg<Float, nColor, nDim, DDArg, reconstruct_>(out, in, U, xpay ? a : 0.0, x, parity, dagger, comm_override),
       Ls(in.X(4))
     {
       if (b_5 == nullptr || c_5 == nullptr)
diff --git a/include/kernels/dslash_domain_wall_4d_fused_m5.cuh b/include/kernels/dslash_domain_wall_4d_fused_m5.cuh
index dfcd519214..ca3f69480a 100644
--- a/include/kernels/dslash_domain_wall_4d_fused_m5.cuh
+++ b/include/kernels/dslash_domain_wall_4d_fused_m5.cuh
@@ -7,15 +7,15 @@
 namespace quda
 {
 
-  template <typename Float, int nColor_, int nDim, QudaReconstructType reconstruct_, Dslash5Type dslash5_type_>
-  struct DomainWall4DFusedM5Arg : DomainWall4DArg<Float, nColor_, nDim, reconstruct_>,
+  template <typename Float, int nColor_, int nDim, typename DDArg, QudaReconstructType reconstruct_, Dslash5Type dslash5_type_>
+  struct DomainWall4DFusedM5Arg : DomainWall4DArg<Float, nColor_, nDim, DDArg, reconstruct_>,
                                   Dslash5Arg<Float, nColor_, false, false, dslash5_type_> {
     // ^^^ Note that for Dslash5Arg we have xpay == dagger == false. This is because the xpay and dagger are determined
     // by fused kernel, not the dslash5, so the `false, false` here are simply dummy instantiations.
 
     static constexpr int nColor = nColor_;
 
-    using DomainWall4DArg = DomainWall4DArg<Float, nColor, nDim, reconstruct_>;
+    using DomainWall4DArg = DomainWall4DArg<Float, nColor, nDim, DDArg, reconstruct_>;
     using DomainWall4DArg::a_5;
     using DomainWall4DArg::dagger;
     using DomainWall4DArg::in;
diff --git a/include/kernels/dslash_domain_wall_5d.cuh b/include/kernels/dslash_domain_wall_5d.cuh
index c9c344df3d..dde72b6c2b 100644
--- a/include/kernels/dslash_domain_wall_5d.cuh
+++ b/include/kernels/dslash_domain_wall_5d.cuh
@@ -7,8 +7,8 @@ namespace quda
 
   // fixme: fused kernel (thread dim mappers set after construction?) and xpay
 
-  template <typename Float, int nColor, int nDim, QudaReconstructType reconstruct_>
-  struct DomainWall5DArg : WilsonArg<Float, nColor, nDim, reconstruct_> {
+  template <typename Float, int nColor, int nDim, typename DDArg, QudaReconstructType reconstruct_>
+  struct DomainWall5DArg : WilsonArg<Float, nColor, nDim, DDArg, reconstruct_> {
     typedef typename mapper<Float>::type real;
     int Ls;   /** fifth dimension length */
     real a;   /** xpay scale factor */
@@ -16,10 +16,12 @@ namespace quda
 
     DomainWall5DArg(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, double a, double m_f,
                     bool xpay, const ColorSpinorField &x, int parity, bool dagger, const int *comm_override) :
-      WilsonArg<Float, nColor, nDim, reconstruct_>(out, in, U, xpay ? a : 0.0, x, parity, dagger, comm_override),
+      WilsonArg<Float, nColor, nDim, DDArg, reconstruct_>(out, in, U, xpay ? a : 0.0, x, parity, dagger, comm_override),
       Ls(in.X(4)),
       a(a),
-      m_f(m_f) {}
+      m_f(m_f)
+    {
+    }
   };
 
   template <int nParity, bool dagger, bool xpay, KernelType kernel_type, typename Arg>
diff --git a/include/kernels/dslash_ndeg_twisted_clover.cuh b/include/kernels/dslash_ndeg_twisted_clover.cuh
index cb8bc61ad7..53b82e3d80 100644
--- a/include/kernels/dslash_ndeg_twisted_clover.cuh
+++ b/include/kernels/dslash_ndeg_twisted_clover.cuh
@@ -6,11 +6,11 @@
 
 namespace quda
 {
-  
-  template <typename Float, int nColor, int nDim, QudaReconstructType reconstruct_>
-    struct NdegTwistedCloverArg : WilsonArg<Float, nColor, nDim, reconstruct_> {
-    
-    using WilsonArg<Float, nColor, nDim, reconstruct_>::nSpin;
+
+  template <typename Float, int nColor, int nDim, typename DDArg, QudaReconstructType reconstruct_>
+  struct NdegTwistedCloverArg : WilsonArg<Float, nColor, nDim, DDArg, reconstruct_> {
+
+    using WilsonArg<Float, nColor, nDim, DDArg, reconstruct_>::nSpin;
     static constexpr int length = (nSpin / (nSpin / 2)) * 2 * nColor * nColor * (nSpin / 2) * (nSpin / 2) / 2;
     typedef typename clover_mapper<Float, length, true>::type C;
     typedef typename mapper<Float>::type real;
@@ -19,23 +19,23 @@ namespace quda
     real a; /** this is the Wilson-dslash scale factor */
     real b; /** this is the chiral twist factor */
     real c; /** this is the flavor twist factor */
-    
-  NdegTwistedCloverArg(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U,
-                       const CloverField &A, double a, double b,
-                       double c, const ColorSpinorField &x, int parity, bool dagger, const int *comm_override) :
-      WilsonArg<Float, nColor, nDim, reconstruct_>(out, in, U, a, x, parity, dagger, comm_override),
+
+    NdegTwistedCloverArg(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, const CloverField &A,
+                         double a, double b, double c, const ColorSpinorField &x, int parity, bool dagger,
+                         const int *comm_override) :
+      WilsonArg<Float, nColor, nDim, DDArg, reconstruct_>(out, in, U, a, x, parity, dagger, comm_override),
       A(A, false),
       a(a),
       // if dagger flip the chiral twist
-      // factor of 1/2 comes from clover normalization 
+      // factor of 1/2 comes from clover normalization
       b(dagger ? -0.5 * b : 0.5 * b),
       c(c)
-      {
-        checkPrecision(U, A);
-        checkLocation(U, A);
-      }
+    {
+      checkPrecision(U, A);
+      checkLocation(U, A);
+    }
   };
-  
+
   template <int nParity, bool dagger, bool xpay, KernelType kernel_type, typename Arg>
     struct nDegTwistedClover : dslash_default {
     
diff --git a/include/kernels/dslash_ndeg_twisted_clover_preconditioned.cuh b/include/kernels/dslash_ndeg_twisted_clover_preconditioned.cuh
index ebd8f71da6..0675f02081 100644
--- a/include/kernels/dslash_ndeg_twisted_clover_preconditioned.cuh
+++ b/include/kernels/dslash_ndeg_twisted_clover_preconditioned.cuh
@@ -7,10 +7,10 @@
 
 namespace quda
 {
-  
-  template <typename Float, int nColor, int nDim, QudaReconstructType reconstruct_>
-    struct NdegTwistedCloverPreconditionedArg : WilsonArg<Float, nColor, nDim, reconstruct_> {
-    using WilsonArg<Float, nColor, nDim, reconstruct_>::nSpin;
+
+  template <typename Float, int nColor, int nDim, typename DDArg, QudaReconstructType reconstruct_>
+  struct NdegTwistedCloverPreconditionedArg : WilsonArg<Float, nColor, nDim, DDArg, reconstruct_> {
+    using WilsonArg<Float, nColor, nDim, DDArg, reconstruct_>::nSpin;
     static constexpr int length = (nSpin / (nSpin / 2)) * 2 * nColor * nColor * (nSpin / 2) * (nSpin / 2) / 2;
     static constexpr bool dynamic_clover = clover::dynamic_inverse();
     
@@ -23,22 +23,20 @@ namespace quda
     real c;          /** this is the flavor twist factor */
     real b2_minus_c2;
 
-  NdegTwistedCloverPreconditionedArg(ColorSpinorField &out, const ColorSpinorField &in,
-                                     const GaugeField &U, const CloverField &A,
-                                     double a, double b, double c, bool xpay,
-                                     const ColorSpinorField &x, int parity, bool dagger,
-                                     const int *comm_override) :
-    WilsonArg<Float, nColor, nDim, reconstruct_>(out, in, U, xpay ? 1.0 : 0.0, x, parity, dagger, comm_override),
+    NdegTwistedCloverPreconditionedArg(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U,
+                                       const CloverField &A, double a, double b, double c, bool xpay,
+                                       const ColorSpinorField &x, int parity, bool dagger, const int *comm_override) :
+      WilsonArg<Float, nColor, nDim, DDArg, reconstruct_>(out, in, U, xpay ? 1.0 : 0.0, x, parity, dagger, comm_override),
       A(A, false),
       A2inv(A, dynamic_clover ? false : true), // if dynamic clover we don't want the inverse field
       a(a),
       b(dagger ? -0.5 * b : 0.5 * b), // if dagger flip the chiral twist
-      c(0.5*c),
+      c(0.5 * c),
       b2_minus_c2(0.25 * (b * b - c * c))
-      {
-        checkPrecision(U, A);
-        checkLocation(U, A);
-      }
+    {
+      checkPrecision(U, A);
+      checkLocation(U, A);
+    }
   };
 
   template <int nParity, bool dagger, bool xpay, KernelType kernel_type, typename Arg>
diff --git a/include/kernels/dslash_ndeg_twisted_mass.cuh b/include/kernels/dslash_ndeg_twisted_mass.cuh
index 2f88d5ee48..0327c9230e 100644
--- a/include/kernels/dslash_ndeg_twisted_mass.cuh
+++ b/include/kernels/dslash_ndeg_twisted_mass.cuh
@@ -5,8 +5,8 @@
 namespace quda
 {
 
-  template <typename Float, int nColor, int nDim, QudaReconstructType reconstruct_>
-  struct NdegTwistedMassArg : WilsonArg<Float, nColor, nDim, reconstruct_> {
+  template <typename Float, int nColor, int nDim, typename DDArg, QudaReconstructType reconstruct_>
+  struct NdegTwistedMassArg : WilsonArg<Float, nColor, nDim, DDArg, reconstruct_> {
     typedef typename mapper<Float>::type real;
     real a; /** this is the Wilson-dslash scale factor */
     real b; /** this is the chiral twist factor */
@@ -14,7 +14,7 @@ namespace quda
 
     NdegTwistedMassArg(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, double a, double b,
                        double c, const ColorSpinorField &x, int parity, bool dagger, const int *comm_override) :
-      WilsonArg<Float, nColor, nDim, reconstruct_>(out, in, U, a, x, parity, dagger, comm_override),
+      WilsonArg<Float, nColor, nDim, DDArg, reconstruct_>(out, in, U, a, x, parity, dagger, comm_override),
       a(a),
       b(dagger ? -b : b), // if dagger flip the chiral twist
       c(c)
diff --git a/include/kernels/dslash_ndeg_twisted_mass_preconditioned.cuh b/include/kernels/dslash_ndeg_twisted_mass_preconditioned.cuh
index 8bab3d5623..8532a27df4 100644
--- a/include/kernels/dslash_ndeg_twisted_mass_preconditioned.cuh
+++ b/include/kernels/dslash_ndeg_twisted_mass_preconditioned.cuh
@@ -7,8 +7,8 @@
 namespace quda
 {
 
-  template <typename Float, int nColor, int nDim, QudaReconstructType reconstruct_, bool asymmetric_>
-  struct NdegTwistedMassArg : WilsonArg<Float, nColor, nDim, reconstruct_> {
+  template <typename Float, int nColor, int nDim, typename DDArg, QudaReconstructType reconstruct_, bool asymmetric_>
+  struct NdegTwistedMassArg : WilsonArg<Float, nColor, nDim, DDArg, reconstruct_> {
     typedef typename mapper<Float>::type real;
     static constexpr bool asymmetric = asymmetric_; /** whether we are applying the asymetric operator or not */
     real a;          /** this is the Wilson-dslash scale factor */
@@ -20,7 +20,7 @@ namespace quda
 
     NdegTwistedMassArg(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, double a, double b,
                        double c, bool xpay, const ColorSpinorField &x, int parity, bool dagger, const int *comm_override) :
-      WilsonArg<Float, nColor, nDim, reconstruct_>(out, in, U, xpay ? 1.0 : 0.0, x, parity, dagger, comm_override),
+      WilsonArg<Float, nColor, nDim, DDArg, reconstruct_>(out, in, U, xpay ? 1.0 : 0.0, x, parity, dagger, comm_override),
       a(a),
       b(dagger ? -b : b), // if dagger flip the chiral twist
       c(c),
@@ -30,9 +30,9 @@ namespace quda
     {
       // set parameters for twisting in the packing kernel
       if (dagger && !asymmetric) {
-        DslashArg<Float, nDim>::twist_a = this->a;
-        DslashArg<Float, nDim>::twist_b = this->b;
-        DslashArg<Float, nDim>::twist_c = this->c;
+        DslashArg<Float, nDim, DDArg>::twist_a = this->a;
+        DslashArg<Float, nDim, DDArg>::twist_b = this->b;
+        DslashArg<Float, nDim, DDArg>::twist_c = this->c;
       }
     }
   };
diff --git a/include/kernels/dslash_staggered.cuh b/include/kernels/dslash_staggered.cuh
index deb38455f8..ba87599993 100644
--- a/include/kernels/dslash_staggered.cuh
+++ b/include/kernels/dslash_staggered.cuh
@@ -14,9 +14,9 @@ namespace quda
   /**
      @brief Parameter structure for driving the Staggered Dslash operator
   */
-  template <typename Float, int nColor_, int nDim, QudaReconstructType reconstruct_u_,
+  template <typename Float, int nColor_, int nDim, typename DDArg, QudaReconstructType reconstruct_u_,
             QudaReconstructType reconstruct_l_, bool improved_, QudaStaggeredPhase phase_ = QUDA_STAGGERED_PHASE_MILC>
-  struct StaggeredArg : DslashArg<Float, nDim> {
+  struct StaggeredArg : DslashArg<Float, nDim, DDArg> {
     typedef typename mapper<Float>::type real;
     static constexpr int nColor = nColor_;
     static constexpr int nSpin = 1;
@@ -51,8 +51,8 @@ namespace quda
 
     StaggeredArg(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, const GaugeField &L, double a,
                  const ColorSpinorField &x, int parity, bool dagger, const int *comm_override) :
-      DslashArg<Float, nDim>(out, in, U, x, parity, dagger, a == 0.0 ? false : true, improved_ ? 3 : 1, spin_project,
-                             comm_override),
+      DslashArg<Float, nDim, DDArg>(out, in, U, x, parity, dagger, a == 0.0 ? false : true, improved_ ? 3 : 1,
+                                    spin_project, comm_override),
       out(out),
       in(in, improved_ ? 3 : 1),
       in_pack(in, improved_ ? 3 : 1),
diff --git a/include/kernels/dslash_twisted_clover_preconditioned.cuh b/include/kernels/dslash_twisted_clover_preconditioned.cuh
index bc46d106f4..18141d8b9f 100644
--- a/include/kernels/dslash_twisted_clover_preconditioned.cuh
+++ b/include/kernels/dslash_twisted_clover_preconditioned.cuh
@@ -7,9 +7,9 @@
 namespace quda
 {
 
-  template <typename Float, int nColor, int nDim, QudaReconstructType reconstruct_>
-  struct TwistedCloverArg : WilsonArg<Float, nColor, nDim, reconstruct_> {
-    using WilsonArg<Float, nColor, nDim, reconstruct_>::nSpin;
+  template <typename Float, int nColor, int nDim, typename DDArg, QudaReconstructType reconstruct_>
+  struct TwistedCloverArg : WilsonArg<Float, nColor, nDim, DDArg, reconstruct_> {
+    using WilsonArg<Float, nColor, nDim, DDArg, reconstruct_>::nSpin;
     static constexpr int length = (nSpin / (nSpin / 2)) * 2 * nColor * nColor * (nSpin / 2) * (nSpin / 2) / 2;
     static constexpr bool dynamic_clover = clover::dynamic_inverse();
 
@@ -24,7 +24,7 @@ namespace quda
     TwistedCloverArg(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, const CloverField &A,
                      double a, double b, bool xpay, const ColorSpinorField &x, int parity, bool dagger,
                      const int *comm_override) :
-      WilsonArg<Float, nColor, nDim, reconstruct_>(out, in, U, xpay ? 1.0 : 0.0, x, parity, dagger, comm_override),
+      WilsonArg<Float, nColor, nDim, DDArg, reconstruct_>(out, in, U, xpay ? 1.0 : 0.0, x, parity, dagger, comm_override),
       A(A, false),
       A2inv(A, dynamic_clover ? false : true), // if dynamic clover we don't want the inverse field
       a(a),
diff --git a/include/kernels/dslash_twisted_mass.cuh b/include/kernels/dslash_twisted_mass.cuh
index 85230c729d..8da5b4493f 100644
--- a/include/kernels/dslash_twisted_mass.cuh
+++ b/include/kernels/dslash_twisted_mass.cuh
@@ -5,15 +5,15 @@
 namespace quda
 {
 
-  template <typename Float, int nColor, int nDim, QudaReconstructType reconstruct_>
-  struct TwistedMassArg : WilsonArg<Float, nColor, nDim, reconstruct_> {
+  template <typename Float, int nColor, int nDim, typename DDArg, QudaReconstructType reconstruct_>
+  struct TwistedMassArg : WilsonArg<Float, nColor, nDim, DDArg, reconstruct_> {
     typedef typename mapper<Float>::type real;
     real a; /** xpay scale facotor */
     real b; /** this is the twist factor */
 
     TwistedMassArg(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, double a, double b,
                    const ColorSpinorField &x, int parity, bool dagger, const int *comm_override) :
-      WilsonArg<Float, nColor, nDim, reconstruct_>(out, in, U, a, x, parity, dagger, comm_override),
+      WilsonArg<Float, nColor, nDim, DDArg, reconstruct_>(out, in, U, a, x, parity, dagger, comm_override),
       a(a),
       b(dagger ? -b : b) // if dagger flip the twist
     {
diff --git a/include/kernels/dslash_twisted_mass_preconditioned.cuh b/include/kernels/dslash_twisted_mass_preconditioned.cuh
index 496e12276d..785fcdbc09 100644
--- a/include/kernels/dslash_twisted_mass_preconditioned.cuh
+++ b/include/kernels/dslash_twisted_mass_preconditioned.cuh
@@ -5,8 +5,8 @@
 namespace quda
 {
 
-  template <typename Float, int nColor, int nDim, QudaReconstructType reconstruct_, bool asymmetric_>
-  struct TwistedMassArg : WilsonArg<Float, nColor, nDim, reconstruct_> {
+  template <typename Float, int nColor, int nDim, typename DDArg, QudaReconstructType reconstruct_, bool asymmetric_>
+  struct TwistedMassArg : WilsonArg<Float, nColor, nDim, DDArg, reconstruct_> {
     typedef typename mapper<Float>::type real;
     static constexpr bool asymmetric = asymmetric_; /** whether we are applying the asymmetric operator or not */
     real a;          /** this is the scaling factor */
@@ -15,9 +15,9 @@ namespace quda
     real a_inv;      /** inverse scaling factor - used to allow early xpay inclusion */
     real b_inv;      /** inverse twist factor - used to allow early xpay inclusion */
 
-    TwistedMassArg(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, double a, double b, bool xpay,
-                   const ColorSpinorField &x, int parity, bool dagger, const int *comm_override) :
-      WilsonArg<Float, nColor, nDim, reconstruct_>(out, in, U, xpay ? 1.0 : 0.0, x, parity, dagger, comm_override),
+    TwistedMassArg(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, double a, double b,
+                   bool xpay, const ColorSpinorField &x, int parity, bool dagger, const int *comm_override) :
+      WilsonArg<Float, nColor, nDim, DDArg, reconstruct_>(out, in, U, xpay ? 1.0 : 0.0, x, parity, dagger, comm_override),
       a(a),
       b(dagger ? -b : b), // if dagger flip the twist
       c(0.0),
@@ -26,8 +26,8 @@ namespace quda
     {
       // set parameters for twisting in the packing kernel
       if (dagger && !asymmetric) {
-        DslashArg<Float, nDim>::twist_a = this->a;
-        DslashArg<Float, nDim>::twist_b = this->b;
+        DslashArg<Float, nDim, DDArg>::twist_a = this->a;
+        DslashArg<Float, nDim, DDArg>::twist_b = this->b;
       }
     }
   };
diff --git a/include/kernels/dslash_wilson_clover.cuh b/include/kernels/dslash_wilson_clover.cuh
index ffc8b880e8..22b6f1dd0f 100644
--- a/include/kernels/dslash_wilson_clover.cuh
+++ b/include/kernels/dslash_wilson_clover.cuh
@@ -7,9 +7,9 @@
 namespace quda
 {
 
-  template <typename Float, int nColor, int nDim, QudaReconstructType reconstruct_, bool twist_ = false>
-  struct WilsonCloverArg : WilsonArg<Float, nColor, nDim, reconstruct_> {
-    using WilsonArg<Float, nColor, nDim, reconstruct_>::nSpin;
+  template <typename Float, int nColor, int nDim, typename DDArg, QudaReconstructType reconstruct_, bool twist_ = false>
+  struct WilsonCloverArg : WilsonArg<Float, nColor, nDim, DDArg, reconstruct_> {
+    using WilsonArg<Float, nColor, nDim, DDArg, reconstruct_>::nSpin;
     static constexpr int length = (nSpin / (nSpin / 2)) * 2 * nColor * nColor * (nSpin / 2) * (nSpin / 2) / 2;
     static constexpr bool twist = twist_;
 
@@ -22,7 +22,7 @@ namespace quda
 
     WilsonCloverArg(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, const CloverField &A,
                     double a, double b, const ColorSpinorField &x, int parity, bool dagger, const int *comm_override) :
-      WilsonArg<Float, nColor, nDim, reconstruct_>(out, in, U, a, x, parity, dagger, comm_override),
+      WilsonArg<Float, nColor, nDim, DDArg, reconstruct_>(out, in, U, a, x, parity, dagger, comm_override),
       A(A, false),
       a(a),
       b(dagger ? -0.5 * b : 0.5 * b) // factor of 1/2 comes from clover normalization we need to correct for
diff --git a/include/kernels/dslash_wilson_clover_hasenbusch_twist.cuh b/include/kernels/dslash_wilson_clover_hasenbusch_twist.cuh
index e6923bf707..6568aabc94 100644
--- a/include/kernels/dslash_wilson_clover_hasenbusch_twist.cuh
+++ b/include/kernels/dslash_wilson_clover_hasenbusch_twist.cuh
@@ -7,9 +7,9 @@
 namespace quda
 {
 
-  template <typename Float, int nColor, int nDim, QudaReconstructType reconstruct_>
-  struct WilsonCloverHasenbuschTwistArg : WilsonArg<Float, nColor, nDim, reconstruct_> {
-    using WilsonArg<Float, nColor, nDim, reconstruct_>::nSpin;
+  template <typename Float, int nColor, int nDim, typename DDArg, QudaReconstructType reconstruct_>
+  struct WilsonCloverHasenbuschTwistArg : WilsonArg<Float, nColor, nDim, DDArg, reconstruct_> {
+    using WilsonArg<Float, nColor, nDim, DDArg, reconstruct_>::nSpin;
     static constexpr int length = (nSpin / (nSpin / 2)) * 2 * nColor * nColor * (nSpin / 2) * (nSpin / 2) / 2;
 
     typedef typename clover_mapper<Float, length>::type C;
@@ -22,7 +22,7 @@ namespace quda
     WilsonCloverHasenbuschTwistArg(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U,
                                    const CloverField &A, double a, double b, const ColorSpinorField &x, int parity,
                                    bool dagger, const int *comm_override) :
-      WilsonArg<Float, nColor, nDim, reconstruct_>(out, in, U, a, x, parity, dagger, comm_override),
+      WilsonArg<Float, nColor, nDim, DDArg, reconstruct_>(out, in, U, a, x, parity, dagger, comm_override),
       A(A, false),
       a(a),
       b(dagger ? -0.5 * b : 0.5 * b) // factor of 1/2 comes from clover normalization we need to correct for
diff --git a/include/kernels/dslash_wilson_clover_hasenbusch_twist_preconditioned.cuh b/include/kernels/dslash_wilson_clover_hasenbusch_twist_preconditioned.cuh
index b9a1599e94..def33285e9 100644
--- a/include/kernels/dslash_wilson_clover_hasenbusch_twist_preconditioned.cuh
+++ b/include/kernels/dslash_wilson_clover_hasenbusch_twist_preconditioned.cuh
@@ -7,9 +7,9 @@
 namespace quda
 {
 
-  template <typename Float, int nColor, int nDim, QudaReconstructType reconstruct_, bool clov_inv_>
-  struct WilsonCloverHasenbuschTwistPCArg : WilsonArg<Float, nColor, nDim, reconstruct_> {
-    using WilsonArg<Float, nColor, nDim, reconstruct_>::nSpin;
+  template <typename Float, int nColor, int nDim, typename DDArg, QudaReconstructType reconstruct_, bool clov_inv_>
+  struct WilsonCloverHasenbuschTwistPCArg : WilsonArg<Float, nColor, nDim, DDArg, reconstruct_> {
+    using WilsonArg<Float, nColor, nDim, DDArg, reconstruct_>::nSpin;
     static constexpr int length = (nSpin / (nSpin / 2)) * 2 * nColor * nColor * (nSpin / 2) * (nSpin / 2) / 2;
     static constexpr bool dynamic_clover = clover::dynamic_inverse();
     static constexpr bool clov_inv = clov_inv_;
@@ -24,7 +24,7 @@ namespace quda
     WilsonCloverHasenbuschTwistPCArg(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U,
                                      const CloverField &A_, double a_, double b_, const ColorSpinorField &x, int parity,
                                      bool dagger, const int *comm_override) :
-      WilsonArg<Float, nColor, nDim, reconstruct_>(out, in, U, a_, x, parity, dagger, comm_override),
+      WilsonArg<Float, nColor, nDim, DDArg, reconstruct_>(out, in, U, a_, x, parity, dagger, comm_override),
       A(A_, false),
       A_inv(A_, dynamic_clover ? false : true),
       b(dagger ? -0.5 * b_ : 0.5 * b_) // if dynamic clover we don't want the inverse field
diff --git a/include/kernels/dslash_wilson_clover_preconditioned.cuh b/include/kernels/dslash_wilson_clover_preconditioned.cuh
index 9adbc64343..bb313fc449 100644
--- a/include/kernels/dslash_wilson_clover_preconditioned.cuh
+++ b/include/kernels/dslash_wilson_clover_preconditioned.cuh
@@ -7,9 +7,9 @@
 namespace quda
 {
 
-  template <typename Float, int nColor, int nDim, QudaReconstructType reconstruct_>
-  struct WilsonCloverArg : WilsonArg<Float, nColor, nDim, reconstruct_> {
-    using WilsonArg<Float, nColor, nDim, reconstruct_>::nSpin;
+  template <typename Float, int nColor, int nDim, typename DDArg, QudaReconstructType reconstruct_>
+  struct WilsonCloverArg : WilsonArg<Float, nColor, nDim, DDArg, reconstruct_> {
+    using WilsonArg<Float, nColor, nDim, DDArg, reconstruct_>::nSpin;
     static constexpr int length = (nSpin / (nSpin / 2)) * 2 * nColor * nColor * (nSpin / 2) * (nSpin / 2) / 2;
     static constexpr bool dynamic_clover = clover::dynamic_inverse();
 
@@ -21,7 +21,7 @@ namespace quda
 
     WilsonCloverArg(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, const CloverField &A,
                     double a, const ColorSpinorField &x, int parity, bool dagger, const int *comm_override) :
-      WilsonArg<Float, nColor, nDim, reconstruct_>(out, in, U, a, x, parity, dagger, comm_override),
+      WilsonArg<Float, nColor, nDim, DDArg, reconstruct_>(out, in, U, a, x, parity, dagger, comm_override),
       A(A, dynamic_clover ? false : true), // if dynamic clover we don't want the inverse field
       a(a)
     {
diff --git a/include/kernels/laplace.cuh b/include/kernels/laplace.cuh
index a029242210..c31766429d 100644
--- a/include/kernels/laplace.cuh
+++ b/include/kernels/laplace.cuh
@@ -14,8 +14,8 @@ namespace quda
   /**
      @brief Parameter structure for driving the covariatnt derivative operator
   */
-  template <typename Float, int nSpin_, int nColor_, int nDim, QudaReconstructType reconstruct_>
-  struct LaplaceArg : DslashArg<Float, nDim> {
+  template <typename Float, int nSpin_, int nColor_, int nDim, typename DDArg, QudaReconstructType reconstruct_>
+  struct LaplaceArg : DslashArg<Float, nDim, DDArg> {
     static constexpr int nColor = nColor_;
     static constexpr int nSpin = nSpin_;
     static constexpr bool spin_project = false;
@@ -40,7 +40,7 @@ namespace quda
 
     LaplaceArg(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, int dir, double a, double b,
                const ColorSpinorField &x, int parity, bool dagger, const int *comm_override) :
-      DslashArg<Float, nDim>(out, in, U, x, parity, dagger, a != 0.0 ? true : false, 1, false, comm_override),
+      DslashArg<Float, nDim, DDArg>(out, in, U, x, parity, dagger, a != 0.0 ? true : false, 1, false, comm_override),
       out(out),
       in(in),
       in_pack(in),
diff --git a/include/kernels/staggered_quark_smearing.cuh b/include/kernels/staggered_quark_smearing.cuh
index 9f4db096e8..14c5a832a2 100644
--- a/include/kernels/staggered_quark_smearing.cuh
+++ b/include/kernels/staggered_quark_smearing.cuh
@@ -14,8 +14,8 @@ namespace quda
   /**
      @brief Parameter structure for driving the covariant derivative operator
   */
-  template <typename Float, int nSpin_, int nColor_, int nDim, QudaReconstructType reconstruct_>
-  struct StaggeredQSmearArg : DslashArg<Float, nDim> {
+  template <typename Float, int nSpin_, int nColor_, int nDim, typename DDArg, QudaReconstructType reconstruct_>
+  struct StaggeredQSmearArg : DslashArg<Float, nDim, DDArg> {
     static constexpr int nColor = 3;
     static constexpr int nSpin = 1;
     static constexpr bool spin_project = false;
@@ -45,7 +45,7 @@ namespace quda
 
     StaggeredQSmearArg(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, int t0,
                        bool is_t0_kernel, int parity, int dir, bool dagger, const int *comm_override) :
-      DslashArg<Float, nDim>(out, in, U, in, parity, dagger, false, 3, false, comm_override),
+      DslashArg<Float, nDim, DDArg>(out, in, U, in, parity, dagger, false, 3, false, comm_override),
       out(out, 3),
       in(in, 3),
       in_pack(in, 3),
diff --git a/lib/covDev.cu b/lib/covDev.cu
index 11f4661aec..990c074700 100644
--- a/lib/covDev.cu
+++ b/lib/covDev.cu
@@ -127,14 +127,14 @@ namespace quda
     }
   };
 
-  template <typename Float, int nColor, QudaReconstructType recon> struct CovDevApply {
+  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon> struct CovDevApply {
 
     inline CovDevApply(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, int mu, int parity,
                        bool dagger, const int *comm_override, TimeProfile &profile)
 
     {
       constexpr int nDim = 4;
-      CovDevArg<Float, nColor, recon, nDim> arg(out, in, U, mu, parity, dagger, comm_override);
+      CovDevArg<Float, nColor, DDArg, recon, nDim> arg(out, in, U, mu, parity, dagger, comm_override);
       CovDev<decltype(arg)> covDev(arg, out, in);
 
       dslash::DslashPolicyTune<decltype(covDev)> policy(covDev, in, in.VolumeCB(), in.GhostFaceCB(), profile);
diff --git a/lib/dslash_domain_wall_4d.cu b/lib/dslash_domain_wall_4d.cu
index a41285dc14..d304aec5a1 100644
--- a/lib/dslash_domain_wall_4d.cu
+++ b/lib/dslash_domain_wall_4d.cu
@@ -36,15 +36,15 @@ namespace quda
     }
   };
 
-  template <typename Float, int nColor, QudaReconstructType recon> struct DomainWall4DApply {
+  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon> struct DomainWall4DApply {
 
     inline DomainWall4DApply(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, double a,
                              double m_5, const Complex *b_5, const Complex *c_5, const ColorSpinorField &x, int parity,
                              bool dagger, const int *comm_override, TimeProfile &profile)
     {
       constexpr int nDim = 4;
-      DomainWall4DArg<Float, nColor, nDim, recon> arg(out, in, U, a, m_5, b_5, c_5, a != 0.0, x, parity, dagger,
-                                                      comm_override);
+      DomainWall4DArg<Float, nColor, nDim, DDArg, recon> arg(out, in, U, a, m_5, b_5, c_5, a != 0.0, x, parity, dagger,
+                                                             comm_override);
       DomainWall4D<decltype(arg)> dwf(arg, out, in);
 
       dslash::DslashPolicyTune<decltype(dwf)> policy(dwf, in, in.getDslashConstant().volume_4d_cb, in.getDslashConstant().ghostFaceCB, profile);
diff --git a/lib/dslash_domain_wall_4d_fused_m5.hpp b/lib/dslash_domain_wall_4d_fused_m5.hpp
index cb66d36470..bd073d3a81 100644
--- a/lib/dslash_domain_wall_4d_fused_m5.hpp
+++ b/lib/dslash_domain_wall_4d_fused_m5.hpp
@@ -129,7 +129,7 @@ namespace quda
   template <Dslash5Type...> struct Dslash5TypeList {
   };
 
-  template <typename Float, int nColor, QudaReconstructType recon> struct DomainWall4DApplyFusedM5 {
+  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon> struct DomainWall4DApplyFusedM5 {
 
     template <Dslash5Type dslash5_type_impl, Dslash5Type... N>
     inline DomainWall4DApplyFusedM5(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, double a,
@@ -141,7 +141,7 @@ namespace quda
       errorQuda("Fused Mobius/DWF-4D kernels do not currently work with NVSHMEM.");
 #else
       constexpr int nDim = 4;
-      using Arg = DomainWall4DFusedM5Arg<Float, nColor, nDim, recon, dslash5_type_impl>;
+      using Arg = DomainWall4DFusedM5Arg<Float, nColor, nDim, DDArg, recon, dslash5_type_impl>;
       Arg arg(out, in, U, a, m_5, b_5, c_5, a != 0.0, x, y, parity, dagger, comm_override, m_f);
       DomainWall4DFusedM5<Arg> dwf(arg, out, in);
 
diff --git a/lib/dslash_domain_wall_5d.cu b/lib/dslash_domain_wall_5d.cu
index 38b2aea600..ba4e24fc3b 100644
--- a/lib/dslash_domain_wall_5d.cu
+++ b/lib/dslash_domain_wall_5d.cu
@@ -63,13 +63,14 @@ namespace quda
     }
   };
 
-  template <typename Float, int nColor, QudaReconstructType recon> struct DomainWall5DApply {
+  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon> struct DomainWall5DApply {
 
     inline DomainWall5DApply(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, double a,
         double m_f, const ColorSpinorField &x, int parity, bool dagger, const int *comm_override, TimeProfile &profile)
     {
       constexpr int nDim = 5;
-      DomainWall5DArg<Float, nColor, nDim, recon> arg(out, in, U, a, m_f, a != 0.0, x, parity, dagger, comm_override);
+      DomainWall5DArg<Float, nColor, nDim, DDArg, recon> arg(out, in, U, a, m_f, a != 0.0, x, parity, dagger,
+                                                             comm_override);
       DomainWall5D<decltype(arg)> dwf(arg, out, in);
 
       dslash::DslashPolicyTune<decltype(dwf)> policy(dwf, in, in.getDslashConstant().volume_4d_cb, in.getDslashConstant().ghostFaceCB, profile);
diff --git a/lib/dslash_improved_staggered.cu b/lib/dslash_improved_staggered.cu
index 41c3327510..ab5cc979b7 100644
--- a/lib/dslash_improved_staggered.cu
+++ b/lib/dslash_improved_staggered.cu
@@ -143,7 +143,7 @@ namespace quda
 
   };
 
-  template <typename Float, int nColor, QudaReconstructType recon_l> struct ImprovedStaggeredApply {
+  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon_l> struct ImprovedStaggeredApply {
 
     inline ImprovedStaggeredApply(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &L,
                                   const GaugeField &U, double a, const ColorSpinorField &x, int parity, bool dagger,
@@ -152,8 +152,8 @@ namespace quda
       constexpr int nDim = 4;
       constexpr bool improved = true;
       constexpr QudaReconstructType recon_u = QUDA_RECONSTRUCT_NO;
-      StaggeredArg<Float, nColor, nDim, recon_u, recon_l, improved> arg(out, in, U, L, a, x, parity, dagger,
-                                                                        comm_override);
+      StaggeredArg<Float, nColor, nDim, DDArg, recon_u, recon_l, improved> arg(out, in, U, L, a, x, parity, dagger,
+                                                                               comm_override);
       Staggered<decltype(arg)> staggered(arg, out, in);
 
       dslash::DslashPolicyTune<decltype(staggered)> policy(staggered, in, in.VolumeCB(), in.GhostFaceCB(), profile);
diff --git a/lib/dslash_ndeg_twisted_clover.cu b/lib/dslash_ndeg_twisted_clover.cu
index 2081320192..eb49e54c5e 100644
--- a/lib/dslash_ndeg_twisted_clover.cu
+++ b/lib/dslash_ndeg_twisted_clover.cu
@@ -72,22 +72,23 @@ namespace quda
         return bytes;
       }
     };
-  
-  template <typename Float, int nColor, QudaReconstructType recon> struct NdegTwistedCloverApply {
-    
-    inline NdegTwistedCloverApply(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U,
-                                  const CloverField &A, double a,
-                                  double b, double c, const ColorSpinorField &x, int parity, bool dagger,
-                                  const int *comm_override, TimeProfile &profile)
-    {
-      constexpr int nDim = 4;
-      NdegTwistedCloverArg<Float, nColor, nDim, recon> arg(out, in, U, A, a, b, c, x, parity, dagger, comm_override);
-      NdegTwistedClover<decltype(arg)> twisted(arg, out, in);
-      // in.VolumeCB() and in.GhostFaceCB() are inappropriate for a two-flavour operator
-      // (since we abuse the fifth dimension for the flavour dof)
-      dslash::DslashPolicyTune<decltype(twisted)> policy(twisted, in, in.getDslashConstant().volume_4d_cb, in.getDslashConstant().ghostFaceCB, profile);
-    }
-  };
+
+    template <typename Float, int nColor, typename DDArg, QudaReconstructType recon> struct NdegTwistedCloverApply {
+
+      inline NdegTwistedCloverApply(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U,
+                                    const CloverField &A, double a, double b, double c, const ColorSpinorField &x,
+                                    int parity, bool dagger, const int *comm_override, TimeProfile &profile)
+      {
+        constexpr int nDim = 4;
+        NdegTwistedCloverArg<Float, nColor, nDim, DDArg, recon> arg(out, in, U, A, a, b, c, x, parity, dagger,
+                                                                    comm_override);
+        NdegTwistedClover<decltype(arg)> twisted(arg, out, in);
+        // in.VolumeCB() and in.GhostFaceCB() are inappropriate for a two-flavour operator
+        // (since we abuse the fifth dimension for the flavour dof)
+        dslash::DslashPolicyTune<decltype(twisted)> policy(twisted, in, in.getDslashConstant().volume_4d_cb,
+                                                           in.getDslashConstant().ghostFaceCB, profile);
+      }
+    };
 
 #ifdef GPU_NDEG_TWISTED_CLOVER_DIRAC
   void ApplyNdegTwistedClover(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, const CloverField &A,
diff --git a/lib/dslash_ndeg_twisted_clover_preconditioned.cu b/lib/dslash_ndeg_twisted_clover_preconditioned.cu
index a852529a3b..18cc1d6816 100644
--- a/lib/dslash_ndeg_twisted_clover_preconditioned.cu
+++ b/lib/dslash_ndeg_twisted_clover_preconditioned.cu
@@ -94,23 +94,25 @@ namespace quda
         return bytes;
       }
     };
-  
-  template <typename Float, int nColor, QudaReconstructType recon> struct NdegTwistedCloverPreconditionedApply {
-    
-    inline NdegTwistedCloverPreconditionedApply(ColorSpinorField &out, const ColorSpinorField &in,
-                                                const GaugeField &U, const CloverField &A,
-                                                double a, double b, double c, bool xpay,
-                                                const ColorSpinorField &x, int parity, bool dagger,
-                                                const int *comm_override, TimeProfile &profile)
-    {
-      constexpr int nDim = 4;
-      NdegTwistedCloverPreconditionedArg<Float, nColor, nDim, recon> arg(out, in, U, A, a, b, c, xpay, x, parity, dagger, comm_override);
-      NdegTwistedCloverPreconditioned<decltype(arg)> twisted(arg, out, in);
-        
-      dslash::DslashPolicyTune<decltype(twisted)> policy(twisted, in, in.getDslashConstant().volume_4d_cb, in.getDslashConstant().ghostFaceCB, profile);
-    }
-  };
-  
+
+    template <typename Float, int nColor, typename DDArg, QudaReconstructType recon>
+    struct NdegTwistedCloverPreconditionedApply {
+
+      inline NdegTwistedCloverPreconditionedApply(ColorSpinorField &out, const ColorSpinorField &in,
+                                                  const GaugeField &U, const CloverField &A, double a, double b,
+                                                  double c, bool xpay, const ColorSpinorField &x, int parity,
+                                                  bool dagger, const int *comm_override, TimeProfile &profile)
+      {
+        constexpr int nDim = 4;
+        NdegTwistedCloverPreconditionedArg<Float, nColor, nDim, DDArg, recon> arg(out, in, U, A, a, b, c, xpay, x,
+                                                                                  parity, dagger, comm_override);
+        NdegTwistedCloverPreconditioned<decltype(arg)> twisted(arg, out, in);
+
+        dslash::DslashPolicyTune<decltype(twisted)> policy(twisted, in, in.getDslashConstant().volume_4d_cb,
+                                                           in.getDslashConstant().ghostFaceCB, profile);
+      }
+    };
+
 #ifdef GPU_NDEG_TWISTED_CLOVER_DIRAC
   void ApplyNdegTwistedCloverPreconditioned(ColorSpinorField &out, const ColorSpinorField &in,
                                             const GaugeField &U, const CloverField &A,
diff --git a/lib/dslash_ndeg_twisted_mass.cu b/lib/dslash_ndeg_twisted_mass.cu
index 7ae48e0acf..c8125095ee 100644
--- a/lib/dslash_ndeg_twisted_mass.cu
+++ b/lib/dslash_ndeg_twisted_mass.cu
@@ -51,14 +51,14 @@ namespace quda
     }
   };
 
-  template <typename Float, int nColor, QudaReconstructType recon> struct NdegTwistedMassApply {
+  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon> struct NdegTwistedMassApply {
 
     inline NdegTwistedMassApply(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, double a,
                                 double b, double c, const ColorSpinorField &x, int parity, bool dagger,
                                 const int *comm_override, TimeProfile &profile)
     {
       constexpr int nDim = 4;
-      NdegTwistedMassArg<Float, nColor, nDim, recon> arg(out, in, U, a, b, c, x, parity, dagger, comm_override);
+      NdegTwistedMassArg<Float, nColor, nDim, DDArg, recon> arg(out, in, U, a, b, c, x, parity, dagger, comm_override);
       NdegTwistedMass<decltype(arg)> twisted(arg, out, in);
 
       dslash::DslashPolicyTune<decltype(twisted)> policy(twisted, in, in.getDslashConstant().volume_4d_cb, in.getDslashConstant().ghostFaceCB, profile);
diff --git a/lib/dslash_ndeg_twisted_mass_preconditioned.cu b/lib/dslash_ndeg_twisted_mass_preconditioned.cu
index d076dea140..c2cef151f6 100644
--- a/lib/dslash_ndeg_twisted_mass_preconditioned.cu
+++ b/lib/dslash_ndeg_twisted_mass_preconditioned.cu
@@ -92,7 +92,8 @@ namespace quda
     }
   };
 
-  template <typename Float, int nColor, QudaReconstructType recon> struct NdegTwistedMassPreconditionedApply {
+  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon>
+  struct NdegTwistedMassPreconditionedApply {
 
     inline NdegTwistedMassPreconditionedApply(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U,
         double a, double b, double c, bool xpay, const ColorSpinorField &x, int parity, bool dagger, bool asymmetric,
@@ -100,12 +101,14 @@ namespace quda
     {
       constexpr int nDim = 4;
       if (asymmetric) {
-        NdegTwistedMassArg<Float, nColor, nDim, recon, true> arg(out, in, U, a, b, c, xpay, x, parity, dagger, comm_override);
+        NdegTwistedMassArg<Float, nColor, nDim, DDArg, recon, true> arg(out, in, U, a, b, c, xpay, x, parity, dagger,
+                                                                        comm_override);
         NdegTwistedMassPreconditioned<decltype(arg)> twisted(arg, out, in);
 
         dslash::DslashPolicyTune<decltype(twisted)> policy(twisted, in, in.getDslashConstant().volume_4d_cb, in.getDslashConstant().ghostFaceCB, profile);
       } else {
-        NdegTwistedMassArg<Float, nColor, nDim, recon, false> arg(out, in, U, a, b, c, xpay, x, parity, dagger, comm_override);
+        NdegTwistedMassArg<Float, nColor, nDim, DDArg, recon, false> arg(out, in, U, a, b, c, xpay, x, parity, dagger,
+                                                                         comm_override);
         NdegTwistedMassPreconditioned<decltype(arg)> twisted(arg, out, in);
 
         dslash::DslashPolicyTune<decltype(twisted)> policy(twisted, in, in.getDslashConstant().volume_4d_cb, in.getDslashConstant().ghostFaceCB, profile);
diff --git a/lib/dslash_twisted_clover.cu b/lib/dslash_twisted_clover.cu
index 1d0e63ae2e..0b0e1d2760 100644
--- a/lib/dslash_twisted_clover.cu
+++ b/lib/dslash_twisted_clover.cu
@@ -62,14 +62,14 @@ namespace quda
     }
   };
 
-  template <typename Float, int nColor, QudaReconstructType recon> struct TwistedCloverApply {
+  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon> struct TwistedCloverApply {
 
     inline TwistedCloverApply(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U,
                               const CloverField &C, double a, double b, const ColorSpinorField &x, int parity,
                               bool dagger, const int *comm_override, TimeProfile &profile)
     {
       constexpr int nDim = 4;
-      WilsonCloverArg<Float, nColor, nDim, recon, true> arg(out, in, U, C, a, b, x, parity, dagger, comm_override);
+      WilsonCloverArg<Float, nColor, nDim, DDArg, recon, true> arg(out, in, U, C, a, b, x, parity, dagger, comm_override);
       TwistedClover<decltype(arg)> twisted(arg, out, in);
 
       dslash::DslashPolicyTune<decltype(twisted)> policy(twisted, in, in.VolumeCB(), in.GhostFaceCB(), profile);
diff --git a/lib/dslash_twisted_clover_preconditioned.cu b/lib/dslash_twisted_clover_preconditioned.cu
index 5e94ed300a..c94fc6da76 100644
--- a/lib/dslash_twisted_clover_preconditioned.cu
+++ b/lib/dslash_twisted_clover_preconditioned.cu
@@ -108,14 +108,16 @@ namespace quda
     }
   };
 
-  template <typename Float, int nColor, QudaReconstructType recon> struct TwistedCloverPreconditionedApply {
+  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon>
+  struct TwistedCloverPreconditionedApply {
 
     inline TwistedCloverPreconditionedApply(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U,
         const CloverField &C, double a, double b, bool xpay, const ColorSpinorField &x, int parity, bool dagger,
         const int *comm_override, TimeProfile &profile)
     {
       constexpr int nDim = 4;
-      TwistedCloverArg<Float, nColor, nDim, recon> arg(out, in, U, C, a, b, xpay, x, parity, dagger, comm_override);
+      TwistedCloverArg<Float, nColor, nDim, DDArg, recon> arg(out, in, U, C, a, b, xpay, x, parity, dagger,
+                                                              comm_override);
       TwistedCloverPreconditioned<decltype(arg)> twisted(arg, out, in);
 
       dslash::DslashPolicyTune<decltype(twisted)> policy(twisted, in, in.VolumeCB(), in.GhostFaceCB(), profile);
diff --git a/lib/dslash_twisted_mass.cu b/lib/dslash_twisted_mass.cu
index a0dba6701e..6969055acd 100644
--- a/lib/dslash_twisted_mass.cu
+++ b/lib/dslash_twisted_mass.cu
@@ -47,14 +47,14 @@ namespace quda
     }
   };
 
-  template <typename Float, int nColor, QudaReconstructType recon> struct TwistedMassApply {
+  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon> struct TwistedMassApply {
 
     inline TwistedMassApply(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, double a, double b,
                             const ColorSpinorField &x, int parity, bool dagger, const int *comm_override,
                             TimeProfile &profile)
     {
       constexpr int nDim = 4;
-      TwistedMassArg<Float, nColor, nDim, recon> arg(out, in, U, a, b, x, parity, dagger, comm_override);
+      TwistedMassArg<Float, nColor, nDim, DDArg, recon> arg(out, in, U, a, b, x, parity, dagger, comm_override);
       TwistedMass<decltype(arg)> twisted(arg, out, in);
 
       dslash::DslashPolicyTune<decltype(twisted)> policy(twisted, in, in.VolumeCB(), in.GhostFaceCB(), profile);
diff --git a/lib/dslash_twisted_mass_preconditioned.cu b/lib/dslash_twisted_mass_preconditioned.cu
index 35a36052ad..742f73d3a8 100644
--- a/lib/dslash_twisted_mass_preconditioned.cu
+++ b/lib/dslash_twisted_mass_preconditioned.cu
@@ -68,7 +68,8 @@ namespace quda
     }
   };
 
-  template <typename Float, int nColor, QudaReconstructType recon> struct TwistedMassPreconditionedApply {
+  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon>
+  struct TwistedMassPreconditionedApply {
 
     inline TwistedMassPreconditionedApply(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U,
         double a, double b, bool xpay, const ColorSpinorField &x, int parity, bool dagger, bool asymmetric,
@@ -76,12 +77,14 @@ namespace quda
     {
       constexpr int nDim = 4;
       if (asymmetric) {
-        TwistedMassArg<Float, nColor, nDim, recon, true> arg(out, in, U, a, b, xpay, x, parity, dagger, comm_override);
+        TwistedMassArg<Float, nColor, nDim, DDArg, recon, true> arg(out, in, U, a, b, xpay, x, parity, dagger,
+                                                                    comm_override);
         TwistedMassPreconditioned<decltype(arg)> twisted(arg, out, in);
 
         dslash::DslashPolicyTune<decltype(twisted)> policy(twisted, in, in.VolumeCB(), in.GhostFaceCB(), profile);
       } else {
-        TwistedMassArg<Float, nColor, nDim, recon, false> arg(out, in, U, a, b, xpay, x, parity, dagger, comm_override);
+        TwistedMassArg<Float, nColor, nDim, DDArg, recon, false> arg(out, in, U, a, b, xpay, x, parity, dagger,
+                                                                     comm_override);
         TwistedMassPreconditioned<decltype(arg)> twisted(arg, out, in);
 
         dslash::DslashPolicyTune<decltype(twisted)> policy(twisted, in, in.VolumeCB(), in.GhostFaceCB(), profile);
diff --git a/lib/dslash_wilson.cu b/lib/dslash_wilson.cu
index 0f82c2373d..82ca0297eb 100644
--- a/lib/dslash_wilson.cu
+++ b/lib/dslash_wilson.cu
@@ -35,13 +35,13 @@ namespace quda
     }
   };
 
-  template <typename Float, int nColor, QudaReconstructType recon> struct WilsonApply {
+  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon> struct WilsonApply {
 
     inline WilsonApply(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, double a,
                        const ColorSpinorField &x, int parity, bool dagger, const int *comm_override, TimeProfile &profile)
     {
       constexpr int nDim = 4;
-      WilsonArg<Float, nColor, nDim, recon> arg(out, in, U, a, x, parity, dagger, comm_override);
+      WilsonArg<Float, nColor, nDim, DDArg, recon> arg(out, in, U, a, x, parity, dagger, comm_override);
       Wilson<decltype(arg)> wilson(arg, out, in);
 
       dslash::DslashPolicyTune<decltype(wilson)> policy(wilson, in, in.VolumeCB(), in.GhostFaceCB(), profile);
diff --git a/lib/dslash_wilson_clover.cu b/lib/dslash_wilson_clover.cu
index 3a5f569750..45710c0f4c 100644
--- a/lib/dslash_wilson_clover.cu
+++ b/lib/dslash_wilson_clover.cu
@@ -62,27 +62,27 @@ namespace quda
     }
   };
 
-  template <typename Float, int nColor, QudaReconstructType recon> struct WilsonCloverApply {
+  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon> struct WilsonCloverApply {
 
     inline WilsonCloverApply(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, const CloverField &A,
         double a, const ColorSpinorField &x, int parity, bool dagger, const int *comm_override, TimeProfile &profile)
     {
       constexpr int nDim = 4;
-      WilsonCloverArg<Float, nColor, nDim, recon> arg(out, in, U, A, a, 0.0, x, parity, dagger, comm_override);
+      WilsonCloverArg<Float, nColor, nDim, DDArg, recon> arg(out, in, U, A, a, 0.0, x, parity, dagger, comm_override);
       WilsonClover<decltype(arg)> wilson(arg, out, in);
 
       dslash::DslashPolicyTune<decltype(wilson)> policy(wilson, in, in.VolumeCB(), in.GhostFaceCB(), profile);
     }
   };
 
-  template <typename Float, int nColor, QudaReconstructType recon> struct WilsonCloverWithTwistApply {
+  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon> struct WilsonCloverWithTwistApply {
 
     inline WilsonCloverWithTwistApply(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U,
                                       const CloverField &A, double a, double b, const ColorSpinorField &x, int parity,
                                       bool dagger, const int *comm_override, TimeProfile &profile)
     {
       constexpr int nDim = 4;
-      WilsonCloverArg<Float, nColor, nDim, recon, true> arg(out, in, U, A, a, b, x, parity, dagger, comm_override);
+      WilsonCloverArg<Float, nColor, nDim, DDArg, recon, true> arg(out, in, U, A, a, b, x, parity, dagger, comm_override);
       WilsonClover<decltype(arg)> wilson(arg, out, in);
 
       dslash::DslashPolicyTune<decltype(wilson)> policy(wilson, in, in.VolumeCB(), in.GhostFaceCB(), profile);
diff --git a/lib/dslash_wilson_clover_hasenbusch_twist.cu b/lib/dslash_wilson_clover_hasenbusch_twist.cu
index cc7694462e..1dbfc74415 100644
--- a/lib/dslash_wilson_clover_hasenbusch_twist.cu
+++ b/lib/dslash_wilson_clover_hasenbusch_twist.cu
@@ -71,14 +71,16 @@ namespace quda
     }
   };
 
-  template <typename Float, int nColor, QudaReconstructType recon> struct WilsonCloverHasenbuschTwistApply {
+  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon>
+  struct WilsonCloverHasenbuschTwistApply {
 
     inline WilsonCloverHasenbuschTwistApply(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U,
                                             const CloverField &A, double a, double b, const ColorSpinorField &x,
                                             int parity, bool dagger, const int *comm_override, TimeProfile &profile)
     {
       constexpr int nDim = 4;
-      WilsonCloverHasenbuschTwistArg<Float, nColor, nDim, recon> arg(out, in, U, A, a, b, x, parity, dagger, comm_override);
+      WilsonCloverHasenbuschTwistArg<Float, nColor, nDim, DDArg, recon> arg(out, in, U, A, a, b, x, parity, dagger,
+                                                                            comm_override);
       WilsonCloverHasenbuschTwist<decltype(arg)> wilson(arg, out, in);
 
       dslash::DslashPolicyTune<decltype(wilson)> policy(wilson, in, in.VolumeCB(), in.GhostFaceCB(), profile);
diff --git a/lib/dslash_wilson_clover_hasenbusch_twist_preconditioned.cu b/lib/dslash_wilson_clover_hasenbusch_twist_preconditioned.cu
index ca8ce572d9..1ff86d5a2d 100644
--- a/lib/dslash_wilson_clover_hasenbusch_twist_preconditioned.cu
+++ b/lib/dslash_wilson_clover_hasenbusch_twist_preconditioned.cu
@@ -114,7 +114,8 @@ namespace quda
     }
   };
 
-  template <typename Float, int nColor, QudaReconstructType recon> struct WilsonCloverHasenbuschTwistPCNoClovInvApply {
+  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon>
+  struct WilsonCloverHasenbuschTwistPCNoClovInvApply {
 
     inline WilsonCloverHasenbuschTwistPCNoClovInvApply(ColorSpinorField &out, const ColorSpinorField &in,
                                                        const GaugeField &U, const CloverField &A, double a, double b,
@@ -122,7 +123,7 @@ namespace quda
                                                        const int *comm_override, TimeProfile &profile)
     {
       constexpr int nDim = 4;
-      using ArgType = WilsonCloverHasenbuschTwistPCArg<Float, nColor, nDim, recon, false>;
+      using ArgType = WilsonCloverHasenbuschTwistPCArg<Float, nColor, nDim, DDArg, recon, false>;
 
       ArgType arg(out, in, U, A, a, b, x, parity, dagger, comm_override);
       WilsonCloverHasenbuschTwistPCNoClovInv<ArgType> wilson(arg, out, in);
@@ -261,7 +262,8 @@ namespace quda
     }
   };
 
-  template <typename Float, int nColor, QudaReconstructType recon> struct WilsonCloverHasenbuschTwistPCClovInvApply {
+  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon>
+  struct WilsonCloverHasenbuschTwistPCClovInvApply {
 
     inline WilsonCloverHasenbuschTwistPCClovInvApply(ColorSpinorField &out, const ColorSpinorField &in,
                                                      const GaugeField &U, const CloverField &A, double kappa, double mu,
@@ -269,7 +271,7 @@ namespace quda
                                                      const int *comm_override, TimeProfile &profile)
     {
       constexpr int nDim = 4;
-      using ArgType = WilsonCloverHasenbuschTwistPCArg<Float, nColor, nDim, recon, true>;
+      using ArgType = WilsonCloverHasenbuschTwistPCArg<Float, nColor, nDim, DDArg, recon, true>;
       ArgType arg(out, in, U, A, kappa, mu, x, parity, dagger, comm_override);
       WilsonCloverHasenbuschTwistPCClovInv<ArgType> wilson(arg, out, in);
 
diff --git a/lib/dslash_wilson_clover_preconditioned.cu b/lib/dslash_wilson_clover_preconditioned.cu
index 2552c92e8e..3f034cbd1b 100644
--- a/lib/dslash_wilson_clover_preconditioned.cu
+++ b/lib/dslash_wilson_clover_preconditioned.cu
@@ -107,14 +107,15 @@ namespace quda
 
   };
 
-  template <typename Float, int nColor, QudaReconstructType recon> struct WilsonCloverPreconditionedApply {
+  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon>
+  struct WilsonCloverPreconditionedApply {
 
     inline WilsonCloverPreconditionedApply(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U,
         const CloverField &A, double a, const ColorSpinorField &x, int parity, bool dagger, const int *comm_override,
         TimeProfile &profile)
     {
       constexpr int nDim = 4;
-      WilsonCloverArg<Float, nColor, nDim, recon> arg(out, in, U, A, a, x, parity, dagger, comm_override);
+      WilsonCloverArg<Float, nColor, nDim, DDArg, recon> arg(out, in, U, A, a, x, parity, dagger, comm_override);
       WilsonCloverPreconditioned<decltype(arg)> wilson(arg, out, in);
 
       dslash::DslashPolicyTune<decltype(wilson)> policy(wilson, in, in.VolumeCB(), in.GhostFaceCB(), profile);
diff --git a/lib/laplace.cu b/lib/laplace.cu
index c476aed6d2..e15ff92c67 100644
--- a/lib/laplace.cu
+++ b/lib/laplace.cu
@@ -140,7 +140,7 @@ namespace quda
     }
   };
 
-  template <typename Float, int nColor, QudaReconstructType recon> struct LaplaceApply {
+  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon> struct LaplaceApply {
 
     LaplaceApply(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, int dir,
                  double a, double b, const ColorSpinorField &x, int parity, bool dagger, const int * comm_override, TimeProfile &profile)
@@ -149,7 +149,8 @@ namespace quda
         if (in.Nspin() == 1) {
           constexpr int nDim = 4;
           constexpr int nSpin = 1;
-          LaplaceArg<Float, nSpin, nColor, nDim, recon> arg(out, in, U, dir, a, b, x, parity, dagger, comm_override);
+          LaplaceArg<Float, nSpin, nColor, nDim, DDArg, recon> arg(out, in, U, dir, a, b, x, parity, dagger,
+                                                                   comm_override);
           Laplace<decltype(arg)> laplace(arg, out, in);
 
           dslash::DslashPolicyTune<decltype(laplace)> policy(laplace, in, in.VolumeCB(), in.GhostFaceCB(), profile);
diff --git a/lib/staggered_quark_smearing.cu b/lib/staggered_quark_smearing.cu
index c133d57648..e0a8de404d 100644
--- a/lib/staggered_quark_smearing.cu
+++ b/lib/staggered_quark_smearing.cu
@@ -181,7 +181,7 @@ namespace quda
     }
   };
 
-  template <typename Float, int nColor, QudaReconstructType recon> struct StaggeredQSmearApply {
+  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon> struct StaggeredQSmearApply {
 
     inline StaggeredQSmearApply(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, int t0,
                                 bool is_tslice_kernel, int parity, int dir, bool dagger, const int *comm_override,
@@ -193,8 +193,8 @@ namespace quda
 
         const int volume = is_tslice_kernel ? in.VolumeCB() / in.X(3) : in.VolumeCB();
 
-        StaggeredQSmearArg<Float, nSpin, nColor, nDim, recon> arg(out, in, U, t0, is_tslice_kernel, parity, dir, dagger,
-                                                                  comm_override);
+        StaggeredQSmearArg<Float, nSpin, nColor, nDim, DDArg, recon> arg(out, in, U, t0, is_tslice_kernel, parity, dir,
+                                                                         dagger, comm_override);
 
         StaggeredQSmear<decltype(arg)> staggered_qsmear(arg, out, in);
 

From 6fd40ea775f75de299974ef76a7a4459c9a8f2c5 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Tue, 7 May 2024 17:41:10 +0200
Subject: [PATCH 022/108] Solving a few more compilation issues

---
 include/domain_decomposition_helper.cuh |  8 +++---
 lib/color_spinor_util.in.cu             | 33 +++++++++++++++----------
 2 files changed, 24 insertions(+), 17 deletions(-)

diff --git a/include/domain_decomposition_helper.cuh b/include/domain_decomposition_helper.cuh
index a0e739e4b2..adcc46e5ec 100644
--- a/include/domain_decomposition_helper.cuh
+++ b/include/domain_decomposition_helper.cuh
@@ -47,19 +47,19 @@ namespace quda
     }
 
     // Computes block_parity: 0 = red, 1 = black
-    template <typename Coord> inline bool block_parity(const Coord &x) const
+    template <typename Coord> constexpr inline bool block_parity(const Coord &x) const
     {
       int block_parity = first_black;
       for (int i = 0; i < x.size(); i++) { block_parity += x[i] / blockDim[i]; }
       return block_parity % 2 == 0;
     }
 
-    template <typename Coord> inline bool on_border(const Coord &x, const int &mu, const int &dir) const
+    template <typename Coord> constexpr inline bool on_border(const Coord &x, const int &mu, const int &dir) const
     {
       return (dir > 0) ? ((x[mu] + 1) % blockDim[mu] == 0) : (x[mu] % blockDim[mu] == 0);
     }
 
-    template <typename Coord> inline bool isZero(const Coord &x) const
+    template <typename Coord> constexpr inline bool isZero(const Coord &x) const
     {
       if (red_active and black_active) return false;
       if (not red_active and not black_active) return true;
@@ -73,7 +73,7 @@ namespace quda
       return true;
     }
 
-    template <typename Coord> inline bool doHopping(const Coord &x, const int &mu, const int &dir) const
+    template <typename Coord> constexpr inline bool doHopping(const Coord &x, const int &mu, const int &dir) const
     {
       if (red_active and black_active and block_hopping) return true;
       if (not red_active and not black_active) return false;
diff --git a/lib/color_spinor_util.in.cu b/lib/color_spinor_util.in.cu
index 9dfbad1fb7..faf4ef9615 100644
--- a/lib/color_spinor_util.in.cu
+++ b/lib/color_spinor_util.in.cu
@@ -5,6 +5,7 @@
 #include <index_helper.cuh>
 #include <blas_quda.h>
 #include <instantiate.h>
+#include <domain_decomposition_helper.cuh>
 
 namespace quda {
 
@@ -423,7 +424,7 @@ namespace quda {
     resize(v, new_size, param);
   }
 
-  template <class P> void projectDD(P &p, const ColorSpinorField &meta)
+  template <typename P, typename DDArg> void projectDD(P &p, DDArg &dd, const ColorSpinorField &meta)
   {
     Coord<4> coord;
     int X[4] = {meta.X(0), meta.X(1), meta.X(2), meta.X(3)};
@@ -433,41 +434,47 @@ namespace quda {
       for (int x_cb = 0; x_cb < p.VolumeCB(); x_cb++) {
         getCoords(coord, x_cb, X, parity);
 
-        if (meta.dd.isZero(coord, meta.getDslashConstant())) {
-          //	  printfQuda("Should be zero %d\n",x_cb);
+        if (dd.isZero(coord)) {
           for (int s = 0; s < p.Nspin(); s++)
             for (int c = 0; c < p.Ncolor(); c++) p(parity, x_cb, s, c) = 0;
-        } else {
-          //	  printfQuda("Should not be zero %d\n",x_cb);
         }
       }
     }
   }
 
-  template <typename Float, int nSpin, int nColor, QudaFieldOrder order> void genericProjectDD(ColorSpinorField &a)
+  template <typename Float, typename DDArg, int nSpin, int nColor, QudaFieldOrder order>
+  void genericProjectDD(ColorSpinorField &a)
   {
     FieldOrderCB<Float, nSpin, nColor, 1, order> A(a);
-    projectDD(A, a);
+    DDArg dd(a);
+    projectDD(A, dd, a);
   }
 
-  template <typename Float> void genericProjectDD(ColorSpinorField &a)
+  template <typename Float, typename DDArg> void genericProjectDD(ColorSpinorField &a)
   {
-
     switch (a.Nspin()) {
     case (1):
-      if constexpr (is_enabled_spin(1)) genericProjectDD<Float, 1, 3, QUDA_SPACE_SPIN_COLOR_FIELD_ORDER>(a);
+      if constexpr (is_enabled_spin(1)) genericProjectDD<Float, DDArg, 1, 3, QUDA_SPACE_SPIN_COLOR_FIELD_ORDER>(a);
       break;
-
     case (2):
-      if constexpr (is_enabled_spin(2)) genericProjectDD<Float, 2, 3, QUDA_SPACE_SPIN_COLOR_FIELD_ORDER>(a);
+      if constexpr (is_enabled_spin(2)) genericProjectDD<Float, DDArg, 2, 3, QUDA_SPACE_SPIN_COLOR_FIELD_ORDER>(a);
       break;
     case (4):
-      if constexpr (is_enabled_spin(4)) genericProjectDD<Float, 4, 3, QUDA_SPACE_SPIN_COLOR_FIELD_ORDER>(a);
+      if constexpr (is_enabled_spin(4)) genericProjectDD<Float, DDArg, 4, 3, QUDA_SPACE_SPIN_COLOR_FIELD_ORDER>(a);
       break;
     default: errorQuda("Nspin %d not implemented", a.Nspin());
     }
   }
 
+  template <typename Float> void genericProjectDD(ColorSpinorField &a)
+  {
+    switch (a.dd.type) {
+    case QUDA_DD_NO: genericProjectDD<Float, DDNo>(a); break;
+    case QUDA_DD_RED_BLACK: genericProjectDD<Float, DDRedBlack>(a); break;
+    default: errorQuda("DD type %d not implemented", a.dd.type);
+    }
+  }
+
   void genericProjectDD(ColorSpinorField &a)
   {
     switch (a.Precision()) {

From 392ea76c3e96ee0b5eb67a38f1324cbd171a7739 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Tue, 7 May 2024 17:44:24 +0200
Subject: [PATCH 023/108] Removing unnecessary copy constructor

---
 include/domain_decomposition.h | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/include/domain_decomposition.h b/include/domain_decomposition.h
index f86ac91023..1f73283626 100644
--- a/include/domain_decomposition.h
+++ b/include/domain_decomposition.h
@@ -25,9 +25,6 @@ namespace quda
     // Default constructor
     DDParam() : type(QUDA_DD_NO), flags {0}, blockDim {0} { }
 
-    // Default copy
-    DDParam(const DDParam &dd) = default;
-
     // returns false if in use
     constexpr inline bool operator!() const { return type == QUDA_DD_NO; }
 

From cf4705d754069b5d8bb455202e3872b8cb403ff9 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Tue, 7 May 2024 21:25:08 +0200
Subject: [PATCH 024/108] other missing fixes

---
 lib/dslash_mdw_fused_impl.hpp | 6 +++---
 lib/dslash_staggered.cu       | 6 +++---
 tests/dslash_test_utils.h     | 8 ++++----
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/lib/dslash_mdw_fused_impl.hpp b/lib/dslash_mdw_fused_impl.hpp
index b3dd58c9e6..0ceba65cc9 100644
--- a/lib/dslash_mdw_fused_impl.hpp
+++ b/lib/dslash_mdw_fused_impl.hpp
@@ -21,7 +21,7 @@ namespace quda
 
 #ifdef QUDA_MMA_AVAILABLE
 
-    template <class store_t, int nColor, QudaReconstructType recon, int Ls_> class FusedDslash : public TunableGridStrideKernel2D
+    template <class store_t, int nColor, typename DDArg, QudaReconstructType recon, int Ls_> class FusedDslash : public TunableGridStrideKernel2D
     {
       ColorSpinorField &out;
       const ColorSpinorField &in;
@@ -256,8 +256,8 @@ namespace quda
 #if defined(GPU_DOMAIN_WALL_DIRAC) && defined(QUDA_MMA_AVAILABLE)
     template <int Ls>
     struct FusedDslashLs {
-      template <class store_t, int nColor, QudaReconstructType recon>
-      using type = FusedDslash<store_t, nColor, recon, Ls>;
+      template <class store_t, int nColor, typename DDArg, QudaReconstructType recon>
+      using type = FusedDslash<store_t, nColor, DDArg, recon, Ls>;
     };
 #endif
   } // namespace mobius_tensor_core
diff --git a/lib/dslash_staggered.cu b/lib/dslash_staggered.cu
index 267cb760ad..95c9e09e26 100644
--- a/lib/dslash_staggered.cu
+++ b/lib/dslash_staggered.cu
@@ -44,7 +44,7 @@ namespace quda
     }
   };
 
-  template <typename Float, int nColor, QudaReconstructType recon_u> struct StaggeredApply {
+  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon_u> struct StaggeredApply {
 
 #if defined(BUILD_MILC_INTERFACE) || defined(BUILD_TIFR_INTERFACE)
     StaggeredApply(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, double a,
@@ -56,7 +56,7 @@ namespace quda
         constexpr int nDim = 4;
         constexpr bool improved = false;
 
-        StaggeredArg<Float, nColor, nDim, recon_u, QUDA_RECONSTRUCT_NO, improved, QUDA_STAGGERED_PHASE_MILC> arg(
+        StaggeredArg<Float, nColor, nDim, DDArg, recon_u, QUDA_RECONSTRUCT_NO, improved, QUDA_STAGGERED_PHASE_MILC> arg(
           out, in, U, U, a, x, parity, dagger, comm_override);
         Staggered<decltype(arg)> staggered(arg, out, in);
 
@@ -69,7 +69,7 @@ namespace quda
         constexpr int nDim = 4;
         constexpr bool improved = false;
 
-        StaggeredArg<Float, nColor, nDim, recon_u, QUDA_RECONSTRUCT_NO, improved, QUDA_STAGGERED_PHASE_TIFR> arg(
+        StaggeredArg<Float, nColor, nDim, DDArg, recon_u, QUDA_RECONSTRUCT_NO, improved, QUDA_STAGGERED_PHASE_TIFR> arg(
           out, in, U, U, a, x, parity, dagger, comm_override);
         Staggered<decltype(arg)> staggered(arg, out, in);
 
diff --git a/tests/dslash_test_utils.h b/tests/dslash_test_utils.h
index f5d2d6e4e8..3b93779fd9 100644
--- a/tests/dslash_test_utils.h
+++ b/tests/dslash_test_utils.h
@@ -818,8 +818,8 @@ struct DslashTestWrapper {
         // We test it applying all 4 possible operators Drr, Drb, Dbr, Dbb
         for (int col = 0; col < 4; col++) {
 
-          cudaSpinor.dd.reset(DD::mode_red_black, col % 2 == 0 ? DD::red_active : DD::black_active);
-          cudaSpinorTmp.dd.reset(DD::mode_red_black, col / 2 == 0 ? DD::red_active : DD::black_active);
+          cudaSpinor.dd.reset(DD::red_black_type, col % 2 == 0 ? DD::red_active : DD::black_active);
+          cudaSpinorTmp.dd.reset(DD::red_black_type, col / 2 == 0 ? DD::red_active : DD::black_active);
 
           switch (dtest_type) {
           case dslash_test_type::Dslash: dirac->Dslash(cudaSpinorTmp, cudaSpinor, parity); break;
@@ -835,7 +835,7 @@ struct DslashTestWrapper {
           cudaSpinorTmp.dd.reset();
 
           cudaSpinorTmp2 = spinor;
-          cudaSpinorTmp2.dd.reset(DD::mode_red_black, col % 2 == 0 ? DD::red_active : DD::black_active);
+          cudaSpinorTmp2.dd.reset(DD::red_black_type, col % 2 == 0 ? DD::red_active : DD::black_active);
           cudaSpinorTmp2.projectDD();
           cudaSpinorTmp2.dd.reset();
 
@@ -849,7 +849,7 @@ struct DslashTestWrapper {
             errorQuda("Test type %s not support for current Dslash", get_string(dtest_type_map, dtest_type).c_str());
           }
 
-          cudaSpinorTmp3.dd.reset(DD::mode_red_black, col / 2 == 0 ? DD::red_active : DD::black_active);
+          cudaSpinorTmp3.dd.reset(DD::red_black_type, col / 2 == 0 ? DD::red_active : DD::black_active);
           cudaSpinorTmp3.projectDD();
           cudaSpinorTmp3.dd.reset();
 

From ae7f918b50f0553a8524bc0487ad6b2c5e5cd42a Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Wed, 8 May 2024 07:33:25 +0200
Subject: [PATCH 025/108] Enabling debug flag

---
 include/domain_decomposition.h | 3 +--
 include/lattice_field.h        | 4 ++--
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/include/domain_decomposition.h b/include/domain_decomposition.h
index 1f73283626..dce3038ed8 100644
--- a/include/domain_decomposition.h
+++ b/include/domain_decomposition.h
@@ -72,10 +72,9 @@ namespace quda
     }
 
     // Checks if this matches to given DDParam
-    // template <bool debug = false, typename T> // if true, prints debug information
+    template <bool debug = false> // if true, prints debug information
     inline bool match(const DDParam &dd) const
     {
-      bool debug = true;
       // if one of the two is not in use we return true, i.e. one of the two is a full field
       if (not *this or not dd) return true;
 
diff --git a/include/lattice_field.h b/include/lattice_field.h
index fa1c37a65e..37ea85271a 100644
--- a/include/lattice_field.h
+++ b/include/lattice_field.h
@@ -940,7 +940,7 @@ namespace quda {
   {
     const unwrap_t<T1> &a(a_);
     const unwrap_t<T2> &b(b_);
-    if (!a.dd.match(b.dd)) errorQuda("DD not match (%s:%d in %s())", file, line, func);
+    if (!a.dd.match<true>(b.dd)) errorQuda("DD not match (%s:%d in %s())", file, line, func);
     return true;
   }
 
@@ -955,7 +955,7 @@ namespace quda {
   inline bool DD_(const char *func, const char *file, int line, const T1 &a, const T2 &b, const Args &...args)
   {
     // checking all possible pairs
-    return (DD_(func, file, line, a, b) & DD_(func, file, line, a, args...) & DD_(func, file, line, b, args...));
+    return (DD_(func, file, line, a, b) && DD_(func, file, line, a, args...) && DD_(func, file, line, b, args...));
   }
 
 #define checkDD(...) DD_(__func__, __FILE__, __LINE__, __VA_ARGS__)

From 4327f706aa21614bc47e8e112a628f88f579ec30 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Wed, 8 May 2024 08:13:43 +0200
Subject: [PATCH 026/108] cleaning

---
 include/domain_decomposition.h |  7 +++----
 include/lattice_field.h        |  2 +-
 tests/dslash_test_utils.h      | 15 +++++----------
 3 files changed, 9 insertions(+), 15 deletions(-)

diff --git a/include/domain_decomposition.h b/include/domain_decomposition.h
index dce3038ed8..d476b5f68f 100644
--- a/include/domain_decomposition.h
+++ b/include/domain_decomposition.h
@@ -72,22 +72,21 @@ namespace quda
     }
 
     // Checks if this matches to given DDParam
-    template <bool debug = false> // if true, prints debug information
-    inline bool match(const DDParam &dd) const
+    inline bool match(const DDParam &dd, bool verbose = false) const
     {
       // if one of the two is not in use we return true, i.e. one of the two is a full field
       if (not *this or not dd) return true;
 
       // false if type does not match
       if (type != dd.type) {
-        if (debug) printfQuda("DD type do not match (%d != %d)\n", type, dd.type);
+        if (verbose) printfQuda("DD type do not match (%d != %d)\n", type, dd.type);
         return false;
       }
 
       if (type == QUDA_DD_RED_BLACK)
         for (int i = 0; i < QUDA_MAX_DIM; i++)
           if (blockDim[i] != dd.blockDim[i]) {
-            if (debug) printfQuda("blockDim[%d] = %d != %d \n", i, blockDim[i], dd.blockDim[i]);
+            if (verbose) printfQuda("blockDim[%d] = %d != %d \n", i, blockDim[i], dd.blockDim[i]);
             return false;
           }
 
diff --git a/include/lattice_field.h b/include/lattice_field.h
index 37ea85271a..44b2c047da 100644
--- a/include/lattice_field.h
+++ b/include/lattice_field.h
@@ -940,7 +940,7 @@ namespace quda {
   {
     const unwrap_t<T1> &a(a_);
     const unwrap_t<T2> &b(b_);
-    if (!a.dd.match<true>(b.dd)) errorQuda("DD not match (%s:%d in %s())", file, line, func);
+    if (!a.dd.match(b.dd, true)) errorQuda("DD not match (%s:%d in %s())", file, line, func);
     return true;
   }
 
diff --git a/tests/dslash_test_utils.h b/tests/dslash_test_utils.h
index 3b93779fd9..640a7db9ac 100644
--- a/tests/dslash_test_utils.h
+++ b/tests/dslash_test_utils.h
@@ -815,7 +815,7 @@ struct DslashTestWrapper {
 
         blas::zero(cudaSpinorOut);
 
-        // We test it applying all 4 possible operators Drr, Drb, Dbr, Dbb
+        // We test that applying all 4 possible operators Drr, Drb, Dbr, Dbb gived D
         for (int col = 0; col < 4; col++) {
 
           cudaSpinor.dd.reset(DD::red_black_type, col % 2 == 0 ? DD::red_active : DD::black_active);
@@ -834,6 +834,9 @@ struct DslashTestWrapper {
           cudaSpinor.dd.reset();
           cudaSpinorTmp.dd.reset();
 
+          blas::xpy(cudaSpinorTmp, cudaSpinorOut);
+
+          // We also test that Dyx is same as D applied to projected in and out spinors
           cudaSpinorTmp2 = spinor;
           cudaSpinorTmp2.dd.reset(DD::red_black_type, col % 2 == 0 ? DD::red_active : DD::black_active);
           cudaSpinorTmp2.projectDD();
@@ -857,11 +860,7 @@ struct DslashTestWrapper {
           spinorOut = cudaSpinorTmp3;
 
           double deviation = std::pow(10, -(double)(ColorSpinorField::Compare(spinorTmp, spinorOut)));
-          printfQuda("Deviaton source %d sink %d %e\n", col % 2, col / 2, deviation);
-
-          // here test cudaSpinorTmp==cudaSpinorTmp2
-
-          blas::xpy(cudaSpinorTmp, cudaSpinorOut);
+          printfQuda("Deviaton for (D-PDP)_{%d,%d}*spinor is %e\n", col % 2, col / 2, deviation);
         }
       } else {
         errorQuda("Test dd type not supported");
@@ -1116,10 +1115,6 @@ struct DslashTestWrapper {
 
       size_t ghost_bytes = cudaSpinor.GhostBytes();
 
-      printfQuda("dslash time event time %e %e %e\n", dslash_time.event_time, dslash_time.cpu_time, dslash_time.cpu_max);
-      printfQuda("niter %d\n", niter);
-      printfQuda("ghost_bytes %e\n", ghost_bytes);
-
       printfQuda("Effective halo bi-directional bandwidth (GB/s) GPU = %f ( CPU = %f, min = %f , max = %f ) for "
                  "aggregate message size %lu bytes\n",
                  1.0e-9 * 2 * ghost_bytes * niter / dslash_time.event_time,

From 57e5bc041a5c38f85e64f4d9f0e1f5f80b38c67b Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Mon, 20 May 2024 08:10:08 +0200
Subject: [PATCH 027/108] Bug fix

---
 include/domain_decomposition_helper.cuh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/include/domain_decomposition_helper.cuh b/include/domain_decomposition_helper.cuh
index adcc46e5ec..29e2b41ae6 100644
--- a/include/domain_decomposition_helper.cuh
+++ b/include/domain_decomposition_helper.cuh
@@ -51,7 +51,7 @@ namespace quda
     {
       int block_parity = first_black;
       for (int i = 0; i < x.size(); i++) { block_parity += x[i] / blockDim[i]; }
-      return block_parity % 2 == 0;
+      return block_parity % 2 == 1;
     }
 
     template <typename Coord> constexpr inline bool on_border(const Coord &x, const int &mu, const int &dir) const

From 8e532d167da2f3cb8b0a6d35f4fee387987e902b Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Mon, 20 May 2024 09:18:01 +0200
Subject: [PATCH 028/108] Applying clang-format

---
 include/kernels/dslash_wilson.cuh                       | 3 ++-
 include/kernels/dslash_wilson_clover.cuh                | 5 +++--
 include/kernels/dslash_wilson_clover_preconditioned.cuh | 2 +-
 lib/dslash_wilson.hpp                                   | 3 ++-
 lib/dslash_wilson_clover.hpp                            | 4 ++--
 lib/dslash_wilson_clover_preconditioned.cu              | 1 -
 lib/dslash_wilson_clover_preconditioned.hpp             | 7 ++++---
 7 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/include/kernels/dslash_wilson.cuh b/include/kernels/dslash_wilson.cuh
index c7a57adc36..af6159ba78 100644
--- a/include/kernels/dslash_wilson.cuh
+++ b/include/kernels/dslash_wilson.cuh
@@ -46,7 +46,8 @@ namespace quda
 
     WilsonArg(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, double a, const ColorSpinorField &x,
               int parity, bool dagger, const int *comm_override, double alpha0 = 0.0, int t0 = -1) :
-      DslashArg<Float, nDim, DDArg>(out, in, U, x, parity, dagger, a != 0.0 ? true : false, 1, spin_project, comm_override),
+      DslashArg<Float, nDim, DDArg>(out, in, U, x, parity, dagger, a != 0.0 ? true : false, 1, spin_project,
+                                    comm_override),
       out(out),
       in(in),
       in_pack(in),
diff --git a/include/kernels/dslash_wilson_clover.cuh b/include/kernels/dslash_wilson_clover.cuh
index 30f2cd283d..ba00ec4075 100644
--- a/include/kernels/dslash_wilson_clover.cuh
+++ b/include/kernels/dslash_wilson_clover.cuh
@@ -7,7 +7,8 @@
 namespace quda
 {
 
-  template <typename Float, int nColor, int nDim, typename DDArg, QudaReconstructType reconstruct_, bool twist_ = false, bool distance_pc_ = false>
+  template <typename Float, int nColor, int nDim, typename DDArg, QudaReconstructType reconstruct_, bool twist_ = false,
+            bool distance_pc_ = false>
   struct WilsonCloverArg : WilsonArg<Float, nColor, nDim, DDArg, reconstruct_, distance_pc_> {
     using WilsonArg<Float, nColor, nDim, DDArg, reconstruct_, distance_pc_>::nSpin;
     static constexpr int length = (nSpin / (nSpin / 2)) * 2 * nColor * nColor * (nSpin / 2) * (nSpin / 2) / 2;
@@ -24,7 +25,7 @@ namespace quda
                     double a, double b, const ColorSpinorField &x, int parity, bool dagger, const int *comm_override,
                     double alpha0 = 0.0, int t0 = -1) :
       WilsonArg<Float, nColor, DDArg, nDim, reconstruct_, distance_pc_>(out, in, U, a, x, parity, dagger, comm_override,
-                                                                 alpha0, t0),
+                                                                        alpha0, t0),
       A(A, false),
       a(a),
       b(dagger ? -0.5 * b : 0.5 * b) // factor of 1/2 comes from clover normalization we need to correct for
diff --git a/include/kernels/dslash_wilson_clover_preconditioned.cuh b/include/kernels/dslash_wilson_clover_preconditioned.cuh
index d97c43a998..33995f38b5 100644
--- a/include/kernels/dslash_wilson_clover_preconditioned.cuh
+++ b/include/kernels/dslash_wilson_clover_preconditioned.cuh
@@ -23,7 +23,7 @@ namespace quda
                     double a, const ColorSpinorField &x, int parity, bool dagger, const int *comm_override,
                     double alpha0 = 0.0, int t0 = -1) :
       WilsonArg<Float, nColor, nDim, DDArg, reconstruct_, distance_pc_>(out, in, U, a, x, parity, dagger, comm_override,
-                                                                 alpha0, t0),
+                                                                        alpha0, t0),
       A(A, dynamic_clover ? false : true), // if dynamic clover we don't want the inverse field
       a(a)
     {
diff --git a/lib/dslash_wilson.hpp b/lib/dslash_wilson.hpp
index a4cade1c12..ef7616c9b1 100644
--- a/lib/dslash_wilson.hpp
+++ b/lib/dslash_wilson.hpp
@@ -44,7 +44,8 @@ namespace quda
                        DistanceType<distance_pc>, TimeProfile &profile)
     {
       constexpr int nDim = 4;
-      WilsonArg<Float, nColor, nDim, DDArg, recon, distance_pc> arg(out, in, U, a, x, parity, dagger, comm_override, alpha0, t0);
+      WilsonArg<Float, nColor, nDim, DDArg, recon, distance_pc> arg(out, in, U, a, x, parity, dagger, comm_override,
+                                                                    alpha0, t0);
       Wilson<decltype(arg)> wilson(arg, out, in);
 
       dslash::DslashPolicyTune<decltype(wilson)> policy(wilson, in, in.VolumeCB(), in.GhostFaceCB(), profile);
diff --git a/lib/dslash_wilson_clover.hpp b/lib/dslash_wilson_clover.hpp
index 983c074bde..d5decfdfe8 100644
--- a/lib/dslash_wilson_clover.hpp
+++ b/lib/dslash_wilson_clover.hpp
@@ -73,8 +73,8 @@ namespace quda
                              const int *comm_override, DistanceType<distance_pc>, TimeProfile &profile)
     {
       constexpr int nDim = 4;
-      WilsonCloverArg<Float, nColor, nDim, DDArg, recon, false, distance_pc> arg(out, in, U, A, a, 0.0, x, parity, dagger,
-                                                                          comm_override, alpha0, t0);
+      WilsonCloverArg<Float, nColor, nDim, DDArg, recon, false, distance_pc> arg(out, in, U, A, a, 0.0, x, parity,
+                                                                                 dagger, comm_override, alpha0, t0);
       WilsonClover<decltype(arg)> wilson(arg, out, in);
 
       dslash::DslashPolicyTune<decltype(wilson)> policy(wilson, in, in.VolumeCB(), in.GhostFaceCB(), profile);
diff --git a/lib/dslash_wilson_clover_preconditioned.cu b/lib/dslash_wilson_clover_preconditioned.cu
index dc8aa4b756..2223ec6514 100644
--- a/lib/dslash_wilson_clover_preconditioned.cu
+++ b/lib/dslash_wilson_clover_preconditioned.cu
@@ -7,7 +7,6 @@
 namespace quda
 {
 
-
   // Apply the preconditioned Wilson-clover operator
   // out(x) = M*in = a * A(x)^{-1} (\sum_mu U_{-\mu}(x)in(x+mu) + U^\dagger_mu(x-mu)in(x-mu))
   // Uses the kappa normalization for the Wilson operator.
diff --git a/lib/dslash_wilson_clover_preconditioned.hpp b/lib/dslash_wilson_clover_preconditioned.hpp
index daf7d78cb3..b6149d7eab 100644
--- a/lib/dslash_wilson_clover_preconditioned.hpp
+++ b/lib/dslash_wilson_clover_preconditioned.hpp
@@ -109,7 +109,8 @@ namespace quda
   template <bool distance_pc> struct DistanceType {
   };
 
-  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon> struct WilsonCloverPreconditionedApply {
+  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon>
+  struct WilsonCloverPreconditionedApply {
 
     template <bool distance_pc>
     inline WilsonCloverPreconditionedApply(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U,
@@ -118,8 +119,8 @@ namespace quda
                                            DistanceType<distance_pc>, TimeProfile &profile)
     {
       constexpr int nDim = 4;
-      WilsonCloverArg<Float, nColor, nDim, DDArg, recon, distance_pc> arg(out, in, U, A, a, x, parity, dagger, comm_override,
-                                                                   alpha0, t0);
+      WilsonCloverArg<Float, nColor, nDim, DDArg, recon, distance_pc> arg(out, in, U, A, a, x, parity, dagger,
+                                                                          comm_override, alpha0, t0);
       WilsonCloverPreconditioned<decltype(arg)> wilson(arg, out, in);
 
       dslash::DslashPolicyTune<decltype(wilson)> policy(wilson, in, in.VolumeCB(), in.GhostFaceCB(), profile);

From c553724779cb58c5847997b71d872d7358dfa317 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Mon, 20 May 2024 10:01:15 +0200
Subject: [PATCH 029/108] Fixing typo in merging

---
 include/kernels/dslash_wilson_clover.cuh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/include/kernels/dslash_wilson_clover.cuh b/include/kernels/dslash_wilson_clover.cuh
index ba00ec4075..83f8f27e81 100644
--- a/include/kernels/dslash_wilson_clover.cuh
+++ b/include/kernels/dslash_wilson_clover.cuh
@@ -24,7 +24,7 @@ namespace quda
     WilsonCloverArg(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, const CloverField &A,
                     double a, double b, const ColorSpinorField &x, int parity, bool dagger, const int *comm_override,
                     double alpha0 = 0.0, int t0 = -1) :
-      WilsonArg<Float, nColor, DDArg, nDim, reconstruct_, distance_pc_>(out, in, U, a, x, parity, dagger, comm_override,
+      WilsonArg<Float, nColor, nDim, DDArg, reconstruct_, distance_pc_>(out, in, U, a, x, parity, dagger, comm_override,
                                                                         alpha0, t0),
       A(A, false),
       a(a),

From c55f733370a55067ea42dda8d2588f1b05fbff7e Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Tue, 21 May 2024 08:41:01 +0200
Subject: [PATCH 030/108] Adding GPU implementation of projectDD

---
 include/kernels/color_spinor_project_dd.cuh |  50 +++++++
 lib/CMakeLists.txt                          |   2 +-
 lib/color_spinor_field.cpp                  |  21 +--
 lib/color_spinor_project_dd.cu              | 141 ++++++++++++++++++++
 lib/color_spinor_util.in.cu                 |  60 ---------
 5 files changed, 193 insertions(+), 81 deletions(-)
 create mode 100644 include/kernels/color_spinor_project_dd.cuh
 create mode 100644 lib/color_spinor_project_dd.cu

diff --git a/include/kernels/color_spinor_project_dd.cuh b/include/kernels/color_spinor_project_dd.cuh
new file mode 100644
index 0000000000..eec44d33f7
--- /dev/null
+++ b/include/kernels/color_spinor_project_dd.cuh
@@ -0,0 +1,50 @@
+#include <color_spinor_field_order.h>
+#include <dslash_helper.cuh>
+#include <kernel.h>
+
+namespace quda
+{
+
+  using namespace colorspinor;
+
+  template <typename Float, typename DDArg, int nSpin_, int nColor_, typename Order>
+  struct ProjectDDArg : kernel_param<> {
+    using real = typename mapper<Float>::type;
+    static constexpr int nSpin = nSpin_;
+    static constexpr int nColor = nColor_;
+    Order out;
+    const DDArg dd;
+    const int parity;
+    const int nParity; // number of parities we're working on
+    const int_fastdiv X0h;
+    const int_fastdiv dim[5]; // full lattice dimensions
+    ProjectDDArg(ColorSpinorField &out) :
+      kernel_param(dim3(out.VolumeCB(), out.SiteSubset(), 1)),
+      out(out),
+      dd(out),
+      parity(out.SiteOrder() == QUDA_ODD_EVEN_SITE_ORDER ? 1 : 0),
+      nParity(out.SiteSubset()),
+      X0h(nParity == 2 ? out.X(0) / 2 : out.X(0)),
+      dim {(3 - nParity) * out.X(0), out.X(1), out.X(2), out.X(3), out.Ndim() == 5 ? out.X(4) : 1}
+    {
+    }
+  };
+
+  template <typename Arg> struct ProjectDD_ {
+    const Arg &arg;
+    constexpr ProjectDD_(const Arg &arg) : arg(arg) { }
+    static constexpr const char *filename() { return KERNEL_FILE; }
+
+    __device__ __host__ inline void operator()(int x_cb, int parity)
+    {
+      Coord<4> coord;
+      coord.X = getCoordsCB(coord, x_cb, arg.dim, arg.X0h, parity);
+      if (arg.dd.isZero(coord)) {
+        ColorSpinor<typename Arg::real, Arg::nColor, Arg::nSpin> out;
+        // for (int i=0; i<Arg::nSpin*Arg::nColor; i++) out[i] = 0;
+        arg.out(x_cb, (parity + arg.parity) & 1) = out;
+      }
+    }
+  };
+
+} // namespace quda
diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt
index 7d3fa2509a..e495259f61 100644
--- a/lib/CMakeLists.txt
+++ b/lib/CMakeLists.txt
@@ -32,7 +32,7 @@ set (QUDA_OBJS
   inv_cg3_quda.cpp inv_ca_gcr.cpp inv_ca_cg.cpp
   inv_gcr_quda.cpp inv_mr_quda.cpp inv_sd_quda.cpp
   inv_pcg_quda.cpp inv_mre.cpp interface_quda.cpp util_quda.cpp
-  color_spinor_field.cpp color_spinor_util.cu
+  color_spinor_field.cpp color_spinor_util.cu color_spinor_project_dd.cu
   field_cache.cpp
   gauge_covdev.cpp dirac.cpp
   clover_field.cpp lattice_field.cpp gauge_field.cpp
diff --git a/lib/color_spinor_field.cpp b/lib/color_spinor_field.cpp
index d0f558f533..2c4f75e307 100644
--- a/lib/color_spinor_field.cpp
+++ b/lib/color_spinor_field.cpp
@@ -1534,26 +1534,7 @@ namespace quda
     genericPrintVector(*this, parity, x_cb, rank);
   }
 
-  void ColorSpinorField::projectDD()
-  {
-    if (Location() == QUDA_CPU_FIELD_LOCATION) {
-      genericProjectDD(*this);
-    } else {
-      ColorSpinorParam param(*this);
-      param.fieldOrder = QUDA_SPACE_SPIN_COLOR_FIELD_ORDER;
-      param.location = QUDA_CPU_FIELD_LOCATION;
-      param.setPrecision((param.Precision() == QUDA_HALF_PRECISION || param.Precision() == QUDA_QUARTER_PRECISION) ?
-                           QUDA_SINGLE_PRECISION :
-                           param.Precision());
-      param.create = QUDA_COPY_FIELD_CREATE;
-      // since CPU fields cannot be low precision, use single precision instead
-      if (precision < QUDA_SINGLE_PRECISION) param.setPrecision(QUDA_SINGLE_PRECISION, QUDA_INVALID_PRECISION, false);
-
-      ColorSpinorField tmp(param);
-      tmp.projectDD();
-      *this = tmp;
-    }
-  }
+  void ColorSpinorField::projectDD() { genericProjectDD(*this); }
 
   int ColorSpinorField::Compare(const ColorSpinorField &a, const ColorSpinorField &b, const int tol)
   {
diff --git a/lib/color_spinor_project_dd.cu b/lib/color_spinor_project_dd.cu
new file mode 100644
index 0000000000..1137f2f8f7
--- /dev/null
+++ b/lib/color_spinor_project_dd.cu
@@ -0,0 +1,141 @@
+#include <tuple>
+#include <memory>
+#include <color_spinor_field.h>
+#include <color_spinor_field_order.h>
+#include <index_helper.cuh>
+#include <blas_quda.h>
+#include <instantiate.h>
+#include <domain_decomposition_helper.cuh>
+#include <tunable_nd.h>
+#include <kernels/color_spinor_project_dd.cuh>
+
+namespace quda
+{
+
+  template <typename Float, typename DDArg, int nSpin, int nColor, typename Order>
+  class ProjectDD : public TunableKernel2D
+  {
+    using Arg = ProjectDDArg<Float, DDArg, nSpin, nColor, Order>;
+    ColorSpinorField &out;
+
+    bool advanceSharedBytes(TuneParam &) const { return false; } // Don't tune shared mem
+    unsigned int minThreads() const { return out.VolumeCB(); }
+
+  public:
+    ProjectDD(ColorSpinorField &out) : TunableKernel2D(out, out.SiteSubset()), out(out)
+    {
+      strcat(aux, out.AuxString().c_str());
+      switch (out.dd.type) {
+      case QUDA_DD_NO: strcat(aux, ",DDNo"); break;
+      case QUDA_DD_RED_BLACK: strcat(aux, ",DDRedBlack"); break;
+      default: errorQuda("DD type %d not implemented", out.dd.type);
+      }
+
+      apply(device::get_default_stream());
+    }
+
+    void apply(const qudaStream_t &stream)
+    {
+      constexpr bool enable_host = true;
+      TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
+      launch<ProjectDD_, enable_host>(tp, stream, Arg(out));
+    }
+
+    long long flops() const { return 0; }
+    long long bytes() const { return out.Bytes(); }
+  };
+
+  template <typename P, typename DDArg> void projectDD(P &p, DDArg &dd, const ColorSpinorField &meta)
+  {
+    Coord<4> coord;
+    int X[4] = {meta.X(0), meta.X(1), meta.X(2), meta.X(3)};
+    X[0] *= (p.Nparity() == 1) ? 2 : 1; // need full lattice dims
+
+    for (int parity = 0; parity < p.Nparity(); parity++) {
+      for (int x_cb = 0; x_cb < p.VolumeCB(); x_cb++) {
+        getCoords(coord, x_cb, X, parity);
+
+        if (dd.isZero(coord)) {
+          for (int s = 0; s < p.Nspin(); s++)
+            for (int c = 0; c < p.Ncolor(); c++) p(parity, x_cb, s, c) = 0;
+        }
+      }
+    }
+  }
+
+  template <typename Float, typename DDArg, int nSpin, int nColor, typename Order>
+  void genericProjectDD(ColorSpinorField &a)
+  {
+    /* Reference CPU implementation
+    if (a.Location() == QUDA_CPU_FIELD_LOCATION and a.FieldOrder() == QUDA_SPACE_SPIN_COLOR_FIELD_ORDER) {
+      FieldOrderCB<Float, nSpin, nColor, 1, QUDA_SPACE_SPIN_COLOR_FIELD_ORDER> A(a);
+      DDArg dd(a);
+      return projectDD(A, dd, a);
+    } */
+
+    ProjectDD<Float, DDArg, nSpin, nColor, Order> A(a);
+  }
+
+  /** Decide on the field order*/
+  template <typename Float, typename DDArg, int nSpin, int nColor> void genericProjectDD(ColorSpinorField &a)
+  {
+    if (a.isNative()) {
+      using Order = typename colorspinor_mapper<Float, nSpin, nColor>::type;
+      genericProjectDD<Float, DDArg, nSpin, nColor, Order>(a);
+    } else if (a.FieldOrder() == QUDA_SPACE_SPIN_COLOR_FIELD_ORDER) {
+      using Order = SpaceSpinorColorOrder<Float, nSpin, nColor>;
+      genericProjectDD<Float, DDArg, nSpin, nColor, Order>(a);
+    } else if (a.FieldOrder() == QUDA_SPACE_COLOR_SPIN_FIELD_ORDER) {
+      using Order = SpaceColorSpinorOrder<Float, nSpin, nColor>;
+      genericProjectDD<Float, DDArg, nSpin, nColor, Order>(a);
+    } else if (a.FieldOrder() == QUDA_PADDED_SPACE_SPIN_COLOR_FIELD_ORDER) {
+      using Order = PaddedSpaceSpinorColorOrder<Float, nSpin, nColor>;
+      if constexpr (is_enabled<QUDA_TIFR_GAUGE_ORDER>())
+        genericProjectDD<Float, DDArg, nSpin, nColor, Order>(a);
+      else
+        errorQuda("TIFR interface has not been built");
+    } else if (a.FieldOrder() == QUDA_QDPJIT_FIELD_ORDER) {
+      using Order = QDPJITDiracOrder<Float, nSpin, nColor>;
+      if constexpr (is_enabled<QUDA_QDPJIT_GAUGE_ORDER>())
+        genericProjectDD<Float, DDArg, nSpin, nColor, Order>(a);
+      else
+        errorQuda("QDPJIT interface has not been built");
+    } else {
+      errorQuda("Order %d not defined (Ns=%d, Nc=%d, precision = %d)", a.FieldOrder(), nSpin, nColor, a.Precision());
+    }
+  }
+
+  template <typename Float, typename DDArg> void genericProjectDD(ColorSpinorField &a)
+  {
+    switch (a.Nspin()) {
+    case (1):
+      if constexpr (is_enabled_spin(1)) genericProjectDD<Float, DDArg, 1, 3>(a);
+      break;
+    case (2):
+      if constexpr (is_enabled_spin(2)) genericProjectDD<Float, DDArg, 2, 3>(a);
+      break;
+    case (4):
+      if constexpr (is_enabled_spin(4)) genericProjectDD<Float, DDArg, 4, 3>(a);
+      break;
+    default: errorQuda("Nspin %d not implemented", a.Nspin());
+    }
+  }
+
+  template <typename Float> void genericProjectDD(ColorSpinorField &a)
+  {
+    switch (a.dd.type) {
+    case QUDA_DD_NO: genericProjectDD<Float, DDNo>(a); break;
+    case QUDA_DD_RED_BLACK: genericProjectDD<Float, DDRedBlack>(a); break;
+    default: errorQuda("DD type %d not implemented", a.dd.type);
+    }
+  }
+
+  void genericProjectDD(ColorSpinorField &a)
+  {
+    switch (a.Precision()) {
+    case QUDA_DOUBLE_PRECISION: genericProjectDD<double>(a); break;
+    case QUDA_SINGLE_PRECISION: genericProjectDD<float>(a); break;
+    default: errorQuda("Precision %d not implemented", a.Precision());
+    }
+  }
+} // namespace quda
diff --git a/lib/color_spinor_util.in.cu b/lib/color_spinor_util.in.cu
index faf4ef9615..0ffc7f082e 100644
--- a/lib/color_spinor_util.in.cu
+++ b/lib/color_spinor_util.in.cu
@@ -424,64 +424,4 @@ namespace quda {
     resize(v, new_size, param);
   }
 
-  template <typename P, typename DDArg> void projectDD(P &p, DDArg &dd, const ColorSpinorField &meta)
-  {
-    Coord<4> coord;
-    int X[4] = {meta.X(0), meta.X(1), meta.X(2), meta.X(3)};
-    X[0] *= (p.Nparity() == 1) ? 2 : 1; // need full lattice dims
-
-    for (int parity = 0; parity < p.Nparity(); parity++) {
-      for (int x_cb = 0; x_cb < p.VolumeCB(); x_cb++) {
-        getCoords(coord, x_cb, X, parity);
-
-        if (dd.isZero(coord)) {
-          for (int s = 0; s < p.Nspin(); s++)
-            for (int c = 0; c < p.Ncolor(); c++) p(parity, x_cb, s, c) = 0;
-        }
-      }
-    }
-  }
-
-  template <typename Float, typename DDArg, int nSpin, int nColor, QudaFieldOrder order>
-  void genericProjectDD(ColorSpinorField &a)
-  {
-    FieldOrderCB<Float, nSpin, nColor, 1, order> A(a);
-    DDArg dd(a);
-    projectDD(A, dd, a);
-  }
-
-  template <typename Float, typename DDArg> void genericProjectDD(ColorSpinorField &a)
-  {
-    switch (a.Nspin()) {
-    case (1):
-      if constexpr (is_enabled_spin(1)) genericProjectDD<Float, DDArg, 1, 3, QUDA_SPACE_SPIN_COLOR_FIELD_ORDER>(a);
-      break;
-    case (2):
-      if constexpr (is_enabled_spin(2)) genericProjectDD<Float, DDArg, 2, 3, QUDA_SPACE_SPIN_COLOR_FIELD_ORDER>(a);
-      break;
-    case (4):
-      if constexpr (is_enabled_spin(4)) genericProjectDD<Float, DDArg, 4, 3, QUDA_SPACE_SPIN_COLOR_FIELD_ORDER>(a);
-      break;
-    default: errorQuda("Nspin %d not implemented", a.Nspin());
-    }
-  }
-
-  template <typename Float> void genericProjectDD(ColorSpinorField &a)
-  {
-    switch (a.dd.type) {
-    case QUDA_DD_NO: genericProjectDD<Float, DDNo>(a); break;
-    case QUDA_DD_RED_BLACK: genericProjectDD<Float, DDRedBlack>(a); break;
-    default: errorQuda("DD type %d not implemented", a.dd.type);
-    }
-  }
-
-  void genericProjectDD(ColorSpinorField &a)
-  {
-    switch (a.Precision()) {
-    case QUDA_DOUBLE_PRECISION: genericProjectDD<double>(a); break;
-    case QUDA_SINGLE_PRECISION: genericProjectDD<float>(a); break;
-    default: errorQuda("Precision %d not implemented", a.Precision());
-    }
-  }
-
 } // namespace quda

From 57791d9a9f561e0b898d155a852a530bec75b6c8 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Tue, 21 May 2024 08:59:37 +0200
Subject: [PATCH 031/108] Changing block_hopping to no_block_hopping

---
 include/domain_decomposition.h          | 8 ++++----
 include/domain_decomposition_helper.cuh | 2 +-
 lib/inv_mr_quda.cpp                     | 4 ++--
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/include/domain_decomposition.h b/include/domain_decomposition.h
index d476b5f68f..c5fc6b6093 100644
--- a/include/domain_decomposition.h
+++ b/include/domain_decomposition.h
@@ -9,10 +9,10 @@ namespace quda
 
   DECLARE_ENUM(DD, // name of the enum class
 
-               red_black_type, // Flags used by red_black
-               red_active,     // if red blocks are active
-               black_active,   // if black blocks are active
-               block_hopping,  // if hopping between red and black is allowed
+               red_black_type,   // Flags used by red_black
+               red_active,       // if red blocks are active
+               black_active,     // if black blocks are active
+               no_block_hopping, // if hopping between red and black is allowed
   );
 
   // Params for domain decompation
diff --git a/include/domain_decomposition_helper.cuh b/include/domain_decomposition_helper.cuh
index 29e2b41ae6..f169261a04 100644
--- a/include/domain_decomposition_helper.cuh
+++ b/include/domain_decomposition_helper.cuh
@@ -38,7 +38,7 @@ namespace quda
       blockDim {in.dd.blockDim[0], in.dd.blockDim[1], in.dd.blockDim[2], in.dd.blockDim[3]},
       red_active(in.dd.type == QUDA_DD_NO or in.dd.is(DD::red_active)),
       black_active(in.dd.type == QUDA_DD_NO or in.dd.is(DD::black_active)),
-      block_hopping(in.dd.type == QUDA_DD_NO or in.dd.is(DD::block_hopping)),
+      block_hopping(in.dd.type == QUDA_DD_NO or not in.dd.is(DD::no_block_hopping)),
       first_black(false) // TODO
     {
       if (in.dd.type != QUDA_DD_NO and in.dd.type != QUDA_DD_RED_BLACK) {
diff --git a/lib/inv_mr_quda.cpp b/lib/inv_mr_quda.cpp
index 5626876c18..11b618a74a 100644
--- a/lib/inv_mr_quda.cpp
+++ b/lib/inv_mr_quda.cpp
@@ -102,8 +102,8 @@ namespace quda
             r_sloppy.dd.reset(DD::red_black_type, step % 2 == 0 ? DD::red_active : DD::black_active);
           } else {
             // Both red and black active but no hopping
-            Ar.dd.reset(DD::red_black_type, DD::red_active, DD::black_active);
-            r_sloppy.dd.reset(DD::red_black_type, DD::red_active, DD::black_active);
+            Ar.dd.reset(DD::red_black_type, DD::red_active, DD::black_active, DD::no_block_hopping);
+            r_sloppy.dd.reset(DD::red_black_type, DD::red_active, DD::black_active, DD::no_block_hopping);
           }
         }
 

From 6ecce5bbceff235bab6a429373bfb4d858c79516 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Tue, 21 May 2024 08:59:51 +0200
Subject: [PATCH 032/108] Improving tests

---
 tests/dslash_test_utils.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/dslash_test_utils.h b/tests/dslash_test_utils.h
index 595789017e..9f5fc44ce6 100644
--- a/tests/dslash_test_utils.h
+++ b/tests/dslash_test_utils.h
@@ -836,7 +836,7 @@ struct DslashTestWrapper {
           blas::xpy(cudaSpinorTmp, cudaSpinorOut);
 
           // We also test that Dyx is same as D applied to projected in and out spinors
-          cudaSpinorTmp2 = spinor;
+          cudaSpinorTmp2 = cudaSpinor;
           cudaSpinorTmp2.dd.reset(DD::red_black_type, col % 2 == 0 ? DD::red_active : DD::black_active);
           cudaSpinorTmp2.projectDD();
           cudaSpinorTmp2.dd.reset();
@@ -860,6 +860,8 @@ struct DslashTestWrapper {
 
           double deviation = std::pow(10, -(double)(ColorSpinorField::Compare(spinorTmp, spinorOut)));
           printfQuda("Deviaton for (D-PDP)_{%d,%d}*spinor is %e\n", col % 2, col / 2, deviation);
+          double tol = getTolerance(cuda_prec);
+          EXPECT_LE(deviation, tol) << "Projected Dirac and project spinors do not agree";
         }
       } else {
         errorQuda("Test dd type not supported");

From 9926d807f477a1e71e450c5f3c13e001fbeed992 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Tue, 21 May 2024 09:29:10 +0200
Subject: [PATCH 033/108] Bug fix

---
 include/kernels/dslash_wilson.cuh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/include/kernels/dslash_wilson.cuh b/include/kernels/dslash_wilson.cuh
index af6159ba78..640f9a7ec4 100644
--- a/include/kernels/dslash_wilson.cuh
+++ b/include/kernels/dslash_wilson.cuh
@@ -194,7 +194,7 @@ namespace quda
           out = x + arg.a * out;
         }
       } else if (mykernel_type != INTERIOR_KERNEL && active) {
-        if (arg.dd_x.isZero(coord)) {
+        if (arg.dd_out.isZero(coord)) {
           out = (xpay ? arg.a * out : out);
         } else {
           Vector x = arg.out(xs, my_spinor_parity);

From d318b51ea9d91e413ecce171f3320c03515f11d4 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Tue, 21 May 2024 09:33:13 +0200
Subject: [PATCH 034/108] Adding enum also to enum_quda_fortran.h

---
 include/enum_quda_fortran.h | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/include/enum_quda_fortran.h b/include/enum_quda_fortran.h
index ada17e2cc8..2339d644ff 100644
--- a/include/enum_quda_fortran.h
+++ b/include/enum_quda_fortran.h
@@ -519,3 +519,8 @@
 #define QUDA_CUSOLVE_EXTLIB 0
 #define QUDA_EIGEN_EXTLIB 1
 #define QUDA_EXTLIB_INVALID QUDA_INVALID_ENUM
+
+#define QudaDDType integer(4)
+#define QUDA_DD_NO 0
+#define QUDA_DD_RED_BLACK 1
+#define QUDA_DD_INVALID QUDA_INVALID_ENUM

From c919c8576900bb9fcf086b9793478f9e457334a5 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Tue, 21 May 2024 09:35:25 +0200
Subject: [PATCH 035/108] Renaming QUDA_DIRAC_DD to
 QUDA_DIRAC_DOMAIN_DECOMPOSITION

---
 CMakeLists.txt           | 2 +-
 include/quda_define.h.in | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index c5039c9864..cd2ae4eb4f 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -151,7 +151,7 @@ option(QUDA_DIRAC_CLOVER_HASENBUSCH "build clover Hasenbusch twist operators" ${
 option(QUDA_DIRAC_NDEG_TWISTED_MASS "build non-degenerate twisted mass Dirac operators" ${QUDA_DIRAC_DEFAULT})
 option(QUDA_DIRAC_NDEG_TWISTED_CLOVER "build non-degenerate twisted clover Dirac operators" ${QUDA_DIRAC_DEFAULT})
 option(QUDA_DIRAC_LAPLACE "build laplace operator" ${QUDA_DIRAC_DEFAULT})
-option(QUDA_DIRAC_DD "build code for domain decomposition of the Dirac operator" OFF)
+option(QUDA_DIRAC_DOMAIN_DECOMPOSITION "build code for domain decomposition of the Dirac operator" OFF)
 option(QUDA_DIRAC_DISTANCE_PRECONDITIONING "build code for distance preconditioned Wilson/clover Dirac operators" OFF)
 
 option(QUDA_CONTRACT "build code for bilinear contraction" OFF)
diff --git a/include/quda_define.h.in b/include/quda_define.h.in
index d709d8bfa3..b67a2b5f56 100644
--- a/include/quda_define.h.in
+++ b/include/quda_define.h.in
@@ -128,8 +128,8 @@
 #define GPU_LAPLACE
 #endif
 
-#cmakedefine QUDA_DIRAC_DD
-#ifdef QUDA_DIRAC_DD
+#cmakedefine QUDA_DIRAC_DOMAIN_DECOMPOSITION
+#ifdef QUDA_DIRAC_DOMAIN_DECOMPOSITION
 /**
  * @def   GPU_DD_DIRAC
  * @brief This macro is set when we have DD-aware Dirac operator enabled

From 3563a275f700d796b9f2b72af0ef7e054d61e563 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Tue, 21 May 2024 09:40:35 +0200
Subject: [PATCH 036/108] Changing DD to Domain Decomposition

---
 include/quda_define.h.in            | 2 +-
 tests/utils/command_line_params.cpp | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/include/quda_define.h.in b/include/quda_define.h.in
index b67a2b5f56..ae675a4acd 100644
--- a/include/quda_define.h.in
+++ b/include/quda_define.h.in
@@ -132,7 +132,7 @@
 #ifdef QUDA_DIRAC_DOMAIN_DECOMPOSITION
 /**
  * @def   GPU_DD_DIRAC
- * @brief This macro is set when we have DD-aware Dirac operator enabled
+ * @brief This macro is set when we have Domain Decomposition (DD)-aware Dirac operator enabled
  */
 #define GPU_DD_DIRAC
 #endif
diff --git a/tests/utils/command_line_params.cpp b/tests/utils/command_line_params.cpp
index 737b377801..1e27ab7b34 100644
--- a/tests/utils/command_line_params.cpp
+++ b/tests/utils/command_line_params.cpp
@@ -1061,7 +1061,7 @@ void add_eofa_option_group(std::shared_ptr<QUDAApp> quda_app)
 
 void add_dd_option_group(std::shared_ptr<QUDAApp> quda_app)
 {
-  auto opgroup = quda_app->add_option_group("DD", "Options controlling DD parameters");
+  auto opgroup = quda_app->add_option_group("DD", "Options controlling Domain Decomposition parameters");
   opgroup
     ->add_option("--dd-block-size", dd_block_size,
                  "Set the domain decomposition block size in all four dimension (default 4 4 4 4)")

From 5410ea0f02ef18c576fb2886c43b76663550d6b8 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Tue, 21 May 2024 09:53:24 +0200
Subject: [PATCH 037/108] Removing most of constexpr

---
 include/domain_decomposition.h          | 12 ++++++------
 include/domain_decomposition_helper.cuh | 18 ++++++++++++------
 2 files changed, 18 insertions(+), 12 deletions(-)

diff --git a/include/domain_decomposition.h b/include/domain_decomposition.h
index c5fc6b6093..e5f8204f18 100644
--- a/include/domain_decomposition.h
+++ b/include/domain_decomposition.h
@@ -26,33 +26,33 @@ namespace quda
     DDParam() : type(QUDA_DD_NO), flags {0}, blockDim {0} { }
 
     // returns false if in use
-    constexpr inline bool operator!() const { return type == QUDA_DD_NO; }
+    inline bool operator!() const { return type == QUDA_DD_NO; }
 
     // returns value of given flag
-    constexpr inline bool is(const DD &flag) const { return flags[(int)flag]; }
+    inline bool is(const DD &flag) const { return flags[(int)flag]; }
 
     // sets given flag to true
-    constexpr inline void set(const DD &flag)
+    inline void set(const DD &flag)
     {
       flags[(int)flag] = true;
 
       if ((int)flag >= (int)DD::red_black_type) type = QUDA_DD_RED_BLACK;
     }
 
-    template <typename... Args> constexpr inline void set(const DD &flag, Args... args)
+    template <typename... Args> inline void set(const DD &flag, Args... args)
     {
       set(flag);
       set(args...);
     }
 
-    constexpr inline void reset()
+    inline void reset()
     {
       type = QUDA_DD_NO;
 #pragma unroll
       for (auto i = 0u; i < (int)DD::size; i++) flags[i] = 0;
     }
 
-    template <typename... Args> constexpr inline void reset(Args... args)
+    template <typename... Args> inline void reset(Args... args)
     {
       reset();
       set(args...);
diff --git a/include/domain_decomposition_helper.cuh b/include/domain_decomposition_helper.cuh
index f169261a04..44ffdf9ef0 100644
--- a/include/domain_decomposition_helper.cuh
+++ b/include/domain_decomposition_helper.cuh
@@ -16,10 +16,14 @@ namespace quda
     }
 
     // Whether field at given coord is zero
-    template <typename Coord> constexpr inline bool isZero(const Coord &) const { return false; }
+    template <typename Coord> constexpr __forceinline__ __device__ __host__ bool isZero(const Coord &) const
+    {
+      return false;
+    }
 
     // Whether do hopping with field at neighboring coord
-    template <typename Coord> constexpr inline bool doHopping(const Coord &, const int &, const int &) const
+    template <typename Coord>
+    constexpr __forceinline__ __device__ __host__ bool doHopping(const Coord &, const int &, const int &) const
     {
       return true;
     }
@@ -47,19 +51,20 @@ namespace quda
     }
 
     // Computes block_parity: 0 = red, 1 = black
-    template <typename Coord> constexpr inline bool block_parity(const Coord &x) const
+    template <typename Coord> __forceinline__ __device__ __host__ bool block_parity(const Coord &x) const
     {
       int block_parity = first_black;
       for (int i = 0; i < x.size(); i++) { block_parity += x[i] / blockDim[i]; }
       return block_parity % 2 == 1;
     }
 
-    template <typename Coord> constexpr inline bool on_border(const Coord &x, const int &mu, const int &dir) const
+    template <typename Coord>
+    __forceinline__ __device__ __host__ bool on_border(const Coord &x, const int &mu, const int &dir) const
     {
       return (dir > 0) ? ((x[mu] + 1) % blockDim[mu] == 0) : (x[mu] % blockDim[mu] == 0);
     }
 
-    template <typename Coord> constexpr inline bool isZero(const Coord &x) const
+    template <typename Coord> __forceinline__ __device__ __host__ bool isZero(const Coord &x) const
     {
       if (red_active and black_active) return false;
       if (not red_active and not black_active) return true;
@@ -73,7 +78,8 @@ namespace quda
       return true;
     }
 
-    template <typename Coord> constexpr inline bool doHopping(const Coord &x, const int &mu, const int &dir) const
+    template <typename Coord>
+    __forceinline__ __device__ __host__ bool doHopping(const Coord &x, const int &mu, const int &dir) const
     {
       if (red_active and black_active and block_hopping) return true;
       if (not red_active and not black_active) return false;

From cdebfe11eac59c187038f0d9149263f24eb75652 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Tue, 21 May 2024 10:02:43 +0200
Subject: [PATCH 038/108] Removing another constexpr

---
 include/invert_quda.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/include/invert_quda.h b/include/invert_quda.h
index 5dc83ce7bf..abee79ec99 100644
--- a/include/invert_quda.h
+++ b/include/invert_quda.h
@@ -405,7 +405,7 @@ namespace quda {
     // for incremental eigCG:
     void updateRhsIndex(QudaInvertParam &param) { rhs_idx = param.rhs_idx; }
 
-    constexpr inline bool do_block_schwarz() const
+    inline bool do_block_schwarz() const
     {
       if (schwarz_type == QUDA_INVALID_SCHWARZ) return false;
       for (int i = 0; i < QUDA_MAX_DIM; i++)

From 64d6256ba3647aadaa621d8bd573dd8bcae9a671 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Tue, 21 May 2024 12:15:32 +0200
Subject: [PATCH 039/108] Adding global coordinates

---
 include/domain_decomposition_helper.cuh     | 10 ++++------
 include/dslash_helper.cuh                   |  7 +++++++
 include/index_helper.cuh                    |  1 +
 include/kernels/color_spinor_project_dd.cuh | 13 +++++++++----
 include/kernels/dslash_wilson.cuh           | 10 +++-------
 lib/color_spinor_project_dd.cu              |  2 ++
 6 files changed, 26 insertions(+), 17 deletions(-)

diff --git a/include/domain_decomposition_helper.cuh b/include/domain_decomposition_helper.cuh
index 44ffdf9ef0..7f7775ae14 100644
--- a/include/domain_decomposition_helper.cuh
+++ b/include/domain_decomposition_helper.cuh
@@ -36,14 +36,12 @@ namespace quda
     const bool red_active;         // if red blocks are active
     const bool black_active;       // if black blocks are active
     const bool block_hopping;      // if hopping between red and black is allowed
-    const bool first_black;        // if the first block of the local lattice is black instead of red
 
     DDRedBlack(const ColorSpinorField &in) :
       blockDim {in.dd.blockDim[0], in.dd.blockDim[1], in.dd.blockDim[2], in.dd.blockDim[3]},
       red_active(in.dd.type == QUDA_DD_NO or in.dd.is(DD::red_active)),
       black_active(in.dd.type == QUDA_DD_NO or in.dd.is(DD::black_active)),
-      block_hopping(in.dd.type == QUDA_DD_NO or not in.dd.is(DD::no_block_hopping)),
-      first_black(false) // TODO
+      block_hopping(in.dd.type == QUDA_DD_NO or not in.dd.is(DD::no_block_hopping))
     {
       if (in.dd.type != QUDA_DD_NO and in.dd.type != QUDA_DD_RED_BLACK) {
         errorQuda("Unsupported type %d\n", in.dd.type);
@@ -53,15 +51,15 @@ namespace quda
     // Computes block_parity: 0 = red, 1 = black
     template <typename Coord> __forceinline__ __device__ __host__ bool block_parity(const Coord &x) const
     {
-      int block_parity = first_black;
-      for (int i = 0; i < x.size(); i++) { block_parity += x[i] / blockDim[i]; }
+      int block_parity = 0;
+      for (int i = 0; i < x.size(); i++) { block_parity += x.gx[i] / blockDim[i]; }
       return block_parity % 2 == 1;
     }
 
     template <typename Coord>
     __forceinline__ __device__ __host__ bool on_border(const Coord &x, const int &mu, const int &dir) const
     {
-      return (dir > 0) ? ((x[mu] + 1) % blockDim[mu] == 0) : (x[mu] % blockDim[mu] == 0);
+      return (dir > 0) ? ((x.gx[mu] + 1) % blockDim[mu] == 0) : (x.gx[mu] % blockDim[mu] == 0);
     }
 
     template <typename Coord> __forceinline__ __device__ __host__ bool isZero(const Coord &x) const
diff --git a/include/dslash_helper.cuh b/include/dslash_helper.cuh
index 354278bca4..0f7c065b55 100644
--- a/include/dslash_helper.cuh
+++ b/include/dslash_helper.cuh
@@ -156,6 +156,7 @@ namespace quda
         coordsFromFaceIndex<nDim, pc_type, 3, nface_>(coord.X, coord.x_cb, coord, idx, face_num, parity, arg);
       }
     }
+    for (int i = 0; i < nDim; i++) { coord.gx[i] = arg.commCoord[i] + coord.x[i]; }
     coord.s = s;
     return coord;
   }
@@ -258,6 +259,9 @@ namespace quda
     const int volumeCB;       // checkerboarded volume
     int commDim[4];           // whether a given dimension is partitioned or not (potentially overridden for Schwarz)
 
+    const int commCoord[5];
+    const int globalDim3;
+
     const bool dagger; // dagger
     const bool xpay;   // whether we are doing xpay or not
 
@@ -324,6 +328,9 @@ namespace quda
       X0h(nParity == 2 ? in.X(0) / 2 : in.X(0)),
       dim {(3 - nParity) * in.X(0), in.X(1), in.X(2), in.X(3), in.Ndim() == 5 ? in.X(4) : 1},
       volumeCB(in.VolumeCB()),
+      commCoord {comm_coord(0) * dim[0], comm_coord(1) * dim[1], comm_coord(2) * dim[2], comm_coord(3) * dim[3],
+                 comm_coord(4) * dim[4]},
+      globalDim3(comm_dim(3) * this->dim[3]),
       dagger(dagger),
       xpay(xpay),
       kernel_type(INTERIOR_KERNEL),
diff --git a/include/index_helper.cuh b/include/index_helper.cuh
index eae33e1e8d..26ea377940 100644
--- a/include/index_helper.cuh
+++ b/include/index_helper.cuh
@@ -231,6 +231,7 @@ namespace quda {
   template <int nDim>
   struct Coord {
     int x[nDim]; // nDim lattice coordinates
+    int gx[nDim]; // nDim global lattice coordinates
     int x_cb;    // checkerboard lattice site index
     int s;       // fifth dimension coord
     int X;       // full lattice site index
diff --git a/include/kernels/color_spinor_project_dd.cuh b/include/kernels/color_spinor_project_dd.cuh
index eec44d33f7..2c9432c306 100644
--- a/include/kernels/color_spinor_project_dd.cuh
+++ b/include/kernels/color_spinor_project_dd.cuh
@@ -18,6 +18,8 @@ namespace quda
     const int nParity; // number of parities we're working on
     const int_fastdiv X0h;
     const int_fastdiv dim[5]; // full lattice dimensions
+    const int commCoord[5];
+
     ProjectDDArg(ColorSpinorField &out) :
       kernel_param(dim3(out.VolumeCB(), out.SiteSubset(), 1)),
       out(out),
@@ -25,7 +27,10 @@ namespace quda
       parity(out.SiteOrder() == QUDA_ODD_EVEN_SITE_ORDER ? 1 : 0),
       nParity(out.SiteSubset()),
       X0h(nParity == 2 ? out.X(0) / 2 : out.X(0)),
-      dim {(3 - nParity) * out.X(0), out.X(1), out.X(2), out.X(3), out.Ndim() == 5 ? out.X(4) : 1}
+      dim {(3 - nParity) * out.X(0), out.X(1), out.X(2), out.X(3), out.Ndim() == 5 ? out.X(4) : 1},
+      commCoord {comm_coord(0) * dim[0], comm_coord(1) * dim[1], comm_coord(2) * dim[2], comm_coord(3) * dim[3],
+                 comm_coord(4) * dim[4]}
+
     {
     }
   };
@@ -39,10 +44,10 @@ namespace quda
     {
       Coord<4> coord;
       coord.X = getCoordsCB(coord, x_cb, arg.dim, arg.X0h, parity);
+      for (int i = 0; i < coord.size(); i++) { coord.gx[i] = arg.commCoord[i] + coord.x[i]; }
       if (arg.dd.isZero(coord)) {
-        ColorSpinor<typename Arg::real, Arg::nColor, Arg::nSpin> out;
-        // for (int i=0; i<Arg::nSpin*Arg::nColor; i++) out[i] = 0;
-        arg.out(x_cb, (parity + arg.parity) & 1) = out;
+        ColorSpinor<typename Arg::real, Arg::nColor, Arg::nSpin> zero;
+        arg.out(x_cb, (parity + arg.parity) & 1) = zero;
       }
     }
   };
diff --git a/include/kernels/dslash_wilson.cuh b/include/kernels/dslash_wilson.cuh
index 640f9a7ec4..3f70fc6c87 100644
--- a/include/kernels/dslash_wilson.cuh
+++ b/include/kernels/dslash_wilson.cuh
@@ -41,8 +41,6 @@ namespace quda
     /** parameters for distance preconditioning */
     const real alpha0;
     const int t0;
-    const int comm_coord_dim_3;
-    const int comm_dim_dim_3;
 
     WilsonArg(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, double a, const ColorSpinorField &x,
               int parity, bool dagger, const int *comm_override, double alpha0 = 0.0, int t0 = -1) :
@@ -55,9 +53,7 @@ namespace quda
       U(U),
       a(a),
       alpha0(alpha0),
-      t0(t0),
-      comm_coord_dim_3(comm_coord(3) * this->dim[3]),
-      comm_dim_dim_3(comm_dim(3) * this->dim[3])
+      t0(t0)
     {
     }
   };
@@ -87,8 +83,8 @@ namespace quda
     real fwd_coeff_3 = 1.0;
     real bwd_coeff_3 = 1.0;
     if constexpr (Arg::distance_pc) {
-      const int t = arg.comm_coord_dim_3 + coord[3];
-      const int nt = arg.comm_dim_dim_3;
+      const int t = coord.gx[3];
+      const int nt = arg.globalDim3;
       fwd_coeff_3 = distanceWeight(arg.alpha0, arg.t0, t + 1, nt) / distanceWeight(arg.alpha0, arg.t0, t, nt);
       bwd_coeff_3 = distanceWeight(arg.alpha0, arg.t0, t - 1, nt) / distanceWeight(arg.alpha0, arg.t0, t, nt);
     } else {
diff --git a/lib/color_spinor_project_dd.cu b/lib/color_spinor_project_dd.cu
index 1137f2f8f7..932ef8c917 100644
--- a/lib/color_spinor_project_dd.cu
+++ b/lib/color_spinor_project_dd.cu
@@ -50,10 +50,12 @@ namespace quda
     Coord<4> coord;
     int X[4] = {meta.X(0), meta.X(1), meta.X(2), meta.X(3)};
     X[0] *= (p.Nparity() == 1) ? 2 : 1; // need full lattice dims
+    int commCoord[4] = {comm_coord(0) * X[0], comm_coord(1) * X[1], comm_coord(2) * X[2], comm_coord(3) * X[3]};
 
     for (int parity = 0; parity < p.Nparity(); parity++) {
       for (int x_cb = 0; x_cb < p.VolumeCB(); x_cb++) {
         getCoords(coord, x_cb, X, parity);
+        for (int i = 0; i < coord.size(); i++) { coord.gx[i] = commCoord[i] + coord.x[i]; }
 
         if (dd.isZero(coord)) {
           for (int s = 0; s < p.Nspin(); s++)

From e93707fb8243f1a0b20a69ea300a837cd3d58f22 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Tue, 21 May 2024 13:29:46 +0200
Subject: [PATCH 040/108] Adding check of DD parameters

---
 include/domain_decomposition.h | 30 +++++++++++++++++++++++++++++-
 include/lattice_field.h        |  2 ++
 2 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/include/domain_decomposition.h b/include/domain_decomposition.h
index e5f8204f18..5cc44f6eca 100644
--- a/include/domain_decomposition.h
+++ b/include/domain_decomposition.h
@@ -71,6 +71,33 @@ namespace quda
       for (int i = 0; i < QUDA_MAX_DIM; i++) printfQuda("blockDim[%d] = %d\n", i, static_cast<int>(blockDim[i]));
     }
 
+    // Checks if this matches to given DDParam
+    template <typename F> inline bool check(const F &field, bool verbose = false) const
+    {
+      if (not *this) return true;
+
+      if (type == QUDA_DD_RED_BLACK) {
+        for (int i = 0; i < field.Ndim(); i++) {
+          if (blockDim[i] <= 0) {
+            if (verbose) printfQuda("blockDim[%d] = %d is not positive \n", i, blockDim[i]);
+            return false;
+          }
+          int globalDim = comm_dim(i) * field.X(i);
+          if (i == 0) globalDim *= (3 - field.SiteSubset());
+          if (globalDim % blockDim[i] != 0) {
+            if (verbose) printfQuda("blockDim[%d] = %d does not divide %d \n", i, blockDim[i], globalDim);
+            return false;
+          }
+          if ((globalDim / blockDim[i]) % 2 != 0) {
+            if (verbose) printfQuda("blockDim[%d] = %d does not divide %d **evenly** \n", i, blockDim[i], globalDim);
+            return false;
+          }
+        }
+      }
+
+      return true;
+    }
+
     // Checks if this matches to given DDParam
     inline bool match(const DDParam &dd, bool verbose = false) const
     {
@@ -83,12 +110,13 @@ namespace quda
         return false;
       }
 
-      if (type == QUDA_DD_RED_BLACK)
+      if (type == QUDA_DD_RED_BLACK) {
         for (int i = 0; i < QUDA_MAX_DIM; i++)
           if (blockDim[i] != dd.blockDim[i]) {
             if (verbose) printfQuda("blockDim[%d] = %d != %d \n", i, blockDim[i], dd.blockDim[i]);
             return false;
           }
+      }
 
       return true;
     }
diff --git a/include/lattice_field.h b/include/lattice_field.h
index 44b2c047da..35724e986e 100644
--- a/include/lattice_field.h
+++ b/include/lattice_field.h
@@ -940,6 +940,8 @@ namespace quda {
   {
     const unwrap_t<T1> &a(a_);
     const unwrap_t<T2> &b(b_);
+    if (!a.dd.check(a, true)) errorQuda("DD checks not passed (%s:%d in %s())", file, line, func);
+    if (!b.dd.check(b, true)) errorQuda("DD checks not passed (%s:%d in %s())", file, line, func);
     if (!a.dd.match(b.dd, true)) errorQuda("DD not match (%s:%d in %s())", file, line, func);
     return true;
   }

From b8923e21cf6a0bbfd0b8b7237406a3fcf6329ddc Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Tue, 21 May 2024 13:54:44 +0200
Subject: [PATCH 041/108] Using QUDA_MAX_DIM instead of 4

---
 include/domain_decomposition.h          | 2 +-
 include/domain_decomposition_helper.cuh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/include/domain_decomposition.h b/include/domain_decomposition.h
index 5cc44f6eca..d9146ef434 100644
--- a/include/domain_decomposition.h
+++ b/include/domain_decomposition.h
@@ -20,7 +20,7 @@ namespace quda
 
     QudaDDType type;
     bool flags[(int)DD::size];  // the default value of all flags is 0
-    int blockDim[4];            // the size of the block per direction
+    int blockDim[QUDA_MAX_DIM]; // the size of the block per direction
 
     // Default constructor
     DDParam() : type(QUDA_DD_NO), flags {0}, blockDim {0} { }
diff --git a/include/domain_decomposition_helper.cuh b/include/domain_decomposition_helper.cuh
index 7f7775ae14..2abd62f31b 100644
--- a/include/domain_decomposition_helper.cuh
+++ b/include/domain_decomposition_helper.cuh
@@ -32,7 +32,7 @@ namespace quda
   // Red-black Block DD
   struct DDRedBlack {
 
-    const int_fastdiv blockDim[4]; // the size of the block per direction
+    const int_fastdiv blockDim[QUDA_MAX_DIM]; // the size of the block per direction
     const bool red_active;         // if red blocks are active
     const bool black_active;       // if black blocks are active
     const bool block_hopping;      // if hopping between red and black is allowed

From 0538afb5dd14bcf837db5e28730869dc86919627 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Tue, 21 May 2024 21:53:07 +0200
Subject: [PATCH 042/108] Restoring use of constexpr

---
 include/domain_decomposition.h          | 12 ++++++------
 include/domain_decomposition_helper.cuh | 21 ++++++---------------
 2 files changed, 12 insertions(+), 21 deletions(-)

diff --git a/include/domain_decomposition.h b/include/domain_decomposition.h
index d9146ef434..e1a4c21703 100644
--- a/include/domain_decomposition.h
+++ b/include/domain_decomposition.h
@@ -26,33 +26,33 @@ namespace quda
     DDParam() : type(QUDA_DD_NO), flags {0}, blockDim {0} { }
 
     // returns false if in use
-    inline bool operator!() const { return type == QUDA_DD_NO; }
+    constexpr bool operator!() const { return type == QUDA_DD_NO; }
 
     // returns value of given flag
-    inline bool is(const DD &flag) const { return flags[(int)flag]; }
+    constexpr bool is(const DD &flag) const { return flags[(int)flag]; }
 
     // sets given flag to true
-    inline void set(const DD &flag)
+    constexpr void set(const DD &flag)
     {
       flags[(int)flag] = true;
 
       if ((int)flag >= (int)DD::red_black_type) type = QUDA_DD_RED_BLACK;
     }
 
-    template <typename... Args> inline void set(const DD &flag, Args... args)
+    template <typename... Args> constexpr void set(const DD &flag, Args... args)
     {
       set(flag);
       set(args...);
     }
 
-    inline void reset()
+    constexpr void reset()
     {
       type = QUDA_DD_NO;
 #pragma unroll
       for (auto i = 0u; i < (int)DD::size; i++) flags[i] = 0;
     }
 
-    template <typename... Args> inline void reset(Args... args)
+    template <typename... Args> constexpr void reset(Args... args)
     {
       reset();
       set(args...);
diff --git a/include/domain_decomposition_helper.cuh b/include/domain_decomposition_helper.cuh
index 2abd62f31b..f7440763f7 100644
--- a/include/domain_decomposition_helper.cuh
+++ b/include/domain_decomposition_helper.cuh
@@ -16,17 +16,10 @@ namespace quda
     }
 
     // Whether field at given coord is zero
-    template <typename Coord> constexpr __forceinline__ __device__ __host__ bool isZero(const Coord &) const
-    {
-      return false;
-    }
+    template <typename Coord> constexpr bool isZero(const Coord &) const { return false; }
 
     // Whether do hopping with field at neighboring coord
-    template <typename Coord>
-    constexpr __forceinline__ __device__ __host__ bool doHopping(const Coord &, const int &, const int &) const
-    {
-      return true;
-    }
+    template <typename Coord> constexpr bool doHopping(const Coord &, const int &, const int &) const { return true; }
   };
 
   // Red-black Block DD
@@ -49,20 +42,19 @@ namespace quda
     }
 
     // Computes block_parity: 0 = red, 1 = black
-    template <typename Coord> __forceinline__ __device__ __host__ bool block_parity(const Coord &x) const
+    template <typename Coord> constexpr bool block_parity(const Coord &x) const
     {
       int block_parity = 0;
       for (int i = 0; i < x.size(); i++) { block_parity += x.gx[i] / blockDim[i]; }
       return block_parity % 2 == 1;
     }
 
-    template <typename Coord>
-    __forceinline__ __device__ __host__ bool on_border(const Coord &x, const int &mu, const int &dir) const
+    template <typename Coord> constexpr bool on_border(const Coord &x, const int &mu, const int &dir) const
     {
       return (dir > 0) ? ((x.gx[mu] + 1) % blockDim[mu] == 0) : (x.gx[mu] % blockDim[mu] == 0);
     }
 
-    template <typename Coord> __forceinline__ __device__ __host__ bool isZero(const Coord &x) const
+    template <typename Coord> constexpr bool isZero(const Coord &x) const
     {
       if (red_active and black_active) return false;
       if (not red_active and not black_active) return true;
@@ -76,8 +68,7 @@ namespace quda
       return true;
     }
 
-    template <typename Coord>
-    __forceinline__ __device__ __host__ bool doHopping(const Coord &x, const int &mu, const int &dir) const
+    template <typename Coord> constexpr bool doHopping(const Coord &x, const int &mu, const int &dir) const
     {
       if (red_active and black_active and block_hopping) return true;
       if (not red_active and not black_active) return false;

From 9dd6cc19817a9e8305fa8ca380a6687ef57a5b77 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Wed, 22 May 2024 07:46:33 +0200
Subject: [PATCH 043/108] Adding check on no_block_hopping

---
 include/domain_decomposition.h | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/include/domain_decomposition.h b/include/domain_decomposition.h
index e1a4c21703..fa89c5da23 100644
--- a/include/domain_decomposition.h
+++ b/include/domain_decomposition.h
@@ -116,6 +116,10 @@ namespace quda
             if (verbose) printfQuda("blockDim[%d] = %d != %d \n", i, blockDim[i], dd.blockDim[i]);
             return false;
           }
+        if (is(DD::no_block_hopping) != dd.is(DD::no_block_hopping)) {
+          if (verbose) printfQuda("no_block_hopping do not match.\n");
+          return false;
+        }
       }
 
       return true;

From 9fc016ff6f9afe9656f34420c12435aae14cc5a7 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Wed, 22 May 2024 07:47:08 +0200
Subject: [PATCH 044/108] Disabling communication qhen possible

---
 include/domain_decomposition_helper.cuh | 24 ++++++++++++++++++++++++
 include/dslash_helper.cuh               |  2 +-
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/include/domain_decomposition_helper.cuh b/include/domain_decomposition_helper.cuh
index f7440763f7..489537115f 100644
--- a/include/domain_decomposition_helper.cuh
+++ b/include/domain_decomposition_helper.cuh
@@ -15,6 +15,15 @@ namespace quda
       if (in.dd.type != QUDA_DD_NO) { errorQuda("Unsupported type %d\n", in.dd.type); }
     }
 
+    // Only DDNo returns true. All others return false
+    constexpr bool operator!() const { return true; }
+
+    // Whether comms are required along given direction
+    template <typename DDArg, typename Arg> constexpr bool commDim(const int &, const DDArg &, const Arg &) const
+    {
+      return true;
+    }
+
     // Whether field at given coord is zero
     template <typename Coord> constexpr bool isZero(const Coord &) const { return false; }
 
@@ -41,6 +50,21 @@ namespace quda
       }
     }
 
+    constexpr bool operator!() const { return false; }
+
+    // Whether comms are required along given direction
+    template <typename DDArg, typename Arg> constexpr bool commDim(const int &d, const DDArg &dd, const Arg &arg) const
+    {
+      if (not red_active and not black_active) return false;
+      if (not dd.red_active and not dd.black_active) return false;
+      if (arg.dim[d] % blockDim[d] == 0) {
+        if (not red_active and not dd.red_active) return false;
+        if (not black_active and not dd.black_active) return false;
+        if (not block_hopping and not dd.block_hopping) return false;
+      }
+      return true;
+    }
+
     // Computes block_parity: 0 = red, 1 = black
     template <typename Coord> constexpr bool block_parity(const Coord &x) const
     {
diff --git a/include/dslash_helper.cuh b/include/dslash_helper.cuh
index 0f7c065b55..223a3f4c69 100644
--- a/include/dslash_helper.cuh
+++ b/include/dslash_helper.cuh
@@ -373,7 +373,7 @@ namespace quda
         errorQuda("Unsupported field order colorspinor=%d gauge=%d combination\n", in.FieldOrder(), U.FieldOrder());
 
       for (int d = 0; d < 4; d++) {
-        commDim[d] = (comm_override[d] == 0) ? 0 : comm_dim_partitioned(d);
+        commDim[d] = (comm_override[d] == 0) ? 0 : (comm_dim_partitioned(d) * dd_out.commDim(d, dd_in, *this));
       }
 
       if (in.Location() == QUDA_CUDA_FIELD_LOCATION) {

From d6c0162b0c36ecbd91d66cf409b692d2c5967d56 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Wed, 22 May 2024 07:47:40 +0200
Subject: [PATCH 045/108] Taking into account also dd of in during
 initialization

---
 include/instantiate_dslash.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/include/instantiate_dslash.h b/include/instantiate_dslash.h
index ac407892b5..6d82629be8 100644
--- a/include/instantiate_dslash.h
+++ b/include/instantiate_dslash.h
@@ -55,10 +55,10 @@ namespace quda
             int nColor, typename... Args>
   inline void instantiate(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, Args &&...args)
   {
-    if (out.dd.type == QUDA_DD_NO) {
+    if (out.dd.type == QUDA_DD_NO and in.dd.type == QUDA_DD_NO) {
       instantiate<Apply, Recon, Float, 3, DDNo>(out, in, U, args...);
 #ifdef GPU_DD_DIRAC
-    } else if (out.dd.type == QUDA_DD_RED_BLACK) {
+    } else if (out.dd.type == QUDA_DD_RED_BLACK or in.dd.type == QUDA_DD_RED_BLACK) {
       instantiate<Apply, Recon, Float, 3, DDRedBlack>(out, in, U, args...);
 #endif
     } else {

From 7d7b320b475f0d6a9b96ab66d2d2b6d0ded7abee Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Wed, 22 May 2024 08:04:13 +0200
Subject: [PATCH 046/108] Using full_dim

---
 include/domain_decomposition.h              | 3 +--
 include/kernels/color_spinor_project_dd.cuh | 2 +-
 lib/color_spinor_project_dd.cu              | 3 +--
 3 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/include/domain_decomposition.h b/include/domain_decomposition.h
index fa89c5da23..2cfb2e90a2 100644
--- a/include/domain_decomposition.h
+++ b/include/domain_decomposition.h
@@ -82,8 +82,7 @@ namespace quda
             if (verbose) printfQuda("blockDim[%d] = %d is not positive \n", i, blockDim[i]);
             return false;
           }
-          int globalDim = comm_dim(i) * field.X(i);
-          if (i == 0) globalDim *= (3 - field.SiteSubset());
+          int globalDim = comm_dim(i) * field.full_dim(i);
           if (globalDim % blockDim[i] != 0) {
             if (verbose) printfQuda("blockDim[%d] = %d does not divide %d \n", i, blockDim[i], globalDim);
             return false;
diff --git a/include/kernels/color_spinor_project_dd.cuh b/include/kernels/color_spinor_project_dd.cuh
index 2c9432c306..30bb7b0221 100644
--- a/include/kernels/color_spinor_project_dd.cuh
+++ b/include/kernels/color_spinor_project_dd.cuh
@@ -27,7 +27,7 @@ namespace quda
       parity(out.SiteOrder() == QUDA_ODD_EVEN_SITE_ORDER ? 1 : 0),
       nParity(out.SiteSubset()),
       X0h(nParity == 2 ? out.X(0) / 2 : out.X(0)),
-      dim {(3 - nParity) * out.X(0), out.X(1), out.X(2), out.X(3), out.Ndim() == 5 ? out.X(4) : 1},
+      dim {out.full_dim(0), out.full_dim(1), out.full_dim(2), out.full_dim(3), out.Ndim() == 5 ? out.full_dim(4) : 1},
       commCoord {comm_coord(0) * dim[0], comm_coord(1) * dim[1], comm_coord(2) * dim[2], comm_coord(3) * dim[3],
                  comm_coord(4) * dim[4]}
 
diff --git a/lib/color_spinor_project_dd.cu b/lib/color_spinor_project_dd.cu
index 932ef8c917..6e4459fd1a 100644
--- a/lib/color_spinor_project_dd.cu
+++ b/lib/color_spinor_project_dd.cu
@@ -48,8 +48,7 @@ namespace quda
   template <typename P, typename DDArg> void projectDD(P &p, DDArg &dd, const ColorSpinorField &meta)
   {
     Coord<4> coord;
-    int X[4] = {meta.X(0), meta.X(1), meta.X(2), meta.X(3)};
-    X[0] *= (p.Nparity() == 1) ? 2 : 1; // need full lattice dims
+    int X[4] = {meta.full_dim(0), meta.full_dim(1), meta.full_dim(2), meta.full_dim(3)};
     int commCoord[4] = {comm_coord(0) * X[0], comm_coord(1) * X[1], comm_coord(2) * X[2], comm_coord(3) * X[3]};
 
     for (int parity = 0; parity < p.Nparity(); parity++) {

From 6bc7b5371f09cda1d0d36134533346e5414788b8 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Wed, 22 May 2024 08:16:40 +0200
Subject: [PATCH 047/108] disabling global reduction is blocks are local

---
 lib/inv_mr_quda.cpp | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/lib/inv_mr_quda.cpp b/lib/inv_mr_quda.cpp
index 11b618a74a..32ad1d66c7 100644
--- a/lib/inv_mr_quda.cpp
+++ b/lib/inv_mr_quda.cpp
@@ -28,6 +28,15 @@ namespace quda
       ColorSpinorParam csParam(b);
       csParam.create = QUDA_NULL_FIELD_CREATE;
 
+      // Setting the value of blockDim and checking if blocks are local
+      bool local = true;
+      for (int i = 0; i < QUDA_MAX_DIM; i++) {
+        csParam.dd.blockDim[i] = param.schwarz_block[i];
+        local &= (x.full_dim(i) % csParam.dd.blockDim[i] == 0);
+      }
+      // Disabling global_reduction if blocks are local and we do block_schwarz
+      if (param.do_block_schwarz() and param.global_reduction) param.global_reduction = not local;
+
       r = ColorSpinorField(csParam);
 
       // now allocate sloppy fields

From b024d4c9ed8f5a8ea3d0e2a697abf70eb9aacc4d Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Wed, 22 May 2024 09:11:37 +0200
Subject: [PATCH 048/108] Adding domain decomposition to ctest

---
 tests/dslash_ctest.cpp    | 41 ++++++++++++++++++++++++++++-----------
 tests/dslash_test.cpp     |  6 ++++++
 tests/dslash_test_utils.h | 33 ++++++++++++++++++++++++++++++-
 3 files changed, 68 insertions(+), 12 deletions(-)

diff --git a/tests/dslash_ctest.cpp b/tests/dslash_ctest.cpp
index 631d640203..751d14c94a 100644
--- a/tests/dslash_ctest.cpp
+++ b/tests/dslash_ctest.cpp
@@ -17,10 +17,10 @@ using ::testing::Range;
 using ::testing::TestWithParam;
 using ::testing::Values;
 
-class DslashTest : public ::testing::TestWithParam<::testing::tuple<int, int, int>>
+class DslashTest : public ::testing::TestWithParam<::testing::tuple<int, int, int, int>>
 {
 protected:
-  ::testing::tuple<int, int, int> param;
+  ::testing::tuple<int, int, int, int> param;
 
   bool skip()
   {
@@ -41,6 +41,9 @@ class DslashTest : public ::testing::TestWithParam<::testing::tuple<int, int, in
                                                   true, false, false, false, true, false, true, true};
     if (!ctest_all_partitions && !partition_enabled[::testing::get<2>(GetParam())]) return true;
 
+    const std::array<bool, 3> domain_decomposition_enabled {true, true, false};
+    if (!ctest_all_partitions && !domain_decomposition_enabled[::testing::get<3>(GetParam())]) return true;
+
     return false;
   }
 
@@ -61,6 +64,12 @@ class DslashTest : public ::testing::TestWithParam<::testing::tuple<int, int, in
       printfQuda("Testing with split grid: %d  %d  %d  %d\n", grid_partition[0], grid_partition[1], grid_partition[2],
                  grid_partition[3]);
     }
+
+    if (dslash_test_wrapper.test_domain_decomposition) {
+      if (dd_red_black)
+        printfQuda("Testing DD Red Black with block: %d  %d  %d  %d\n", dd_block_size[0], dd_block_size[1],
+                   dd_block_size[2], dd_block_size[3]);
+    }
   }
 
 public:
@@ -77,7 +86,8 @@ class DslashTest : public ::testing::TestWithParam<::testing::tuple<int, int, in
     }
     updateR();
 
-    dslash_test_wrapper.init_ctest(argc_copy, argv_copy, prec, recon);
+    int dd_value = ::testing::get<3>(GetParam());
+    dslash_test_wrapper.init_ctest(argc_copy, argv_copy, prec, recon, dd_value);
     display_test_info(prec, recon);
   }
 
@@ -157,30 +167,39 @@ int main(int argc, char **argv)
   return test_rc;
 }
 
-std::string getdslashtestname(testing::TestParamInfo<::testing::tuple<int, int, int>> param)
+std::string getdslashtestname(testing::TestParamInfo<::testing::tuple<int, int, int, int>> param)
 {
   const int prec = ::testing::get<0>(param.param);
   const int recon = ::testing::get<1>(param.param);
   const int part = ::testing::get<2>(param.param);
+  const int dd = ::testing::get<3>(param.param);
   std::stringstream ss;
   // std::cout << "getdslashtestname" << get_dslash_str(dslash_type) << "_" << prec_str[prec] << "_r" << recon <<
   // "_partition" << part << std::endl; ss << get_dslash_str(dslash_type) << "_";
   ss << get_prec_str(getPrecision(prec));
   ss << "_r" << recon;
   ss << "_partition" << part;
+  ss << "_domain_decomposition" << dd;
   return ss.str();
 }
 
 #ifdef MULTI_GPU
-INSTANTIATE_TEST_SUITE_P(QUDA, DslashTest,
-                         Combine(Range(0, 4),
-                                 ::testing::Values(QUDA_RECONSTRUCT_NO, QUDA_RECONSTRUCT_12, QUDA_RECONSTRUCT_8),
-                                 Range(0, 16)),
-                         getdslashtestname);
+#define N_PARTITIONS 16
 #else
+#define N_PARTITIONS 1
+#endif
+
+#ifdef GPU_DD_DIRAC
+#define N_DD_TESTS 3
+#else
+#define N_DD_TESTS 1
+#endif
+
 INSTANTIATE_TEST_SUITE_P(QUDA, DslashTest,
                          Combine(Range(0, 4),
                                  ::testing::Values(QUDA_RECONSTRUCT_NO, QUDA_RECONSTRUCT_12, QUDA_RECONSTRUCT_8),
-                                 ::testing::Values(0)),
+                                 Range(0, N_PARTITIONS), Range(0, N_DD_TESTS)),
                          getdslashtestname);
-#endif
+
+#undef N_PARTITIONS
+#undef N_DD_TESTS
diff --git a/tests/dslash_test.cpp b/tests/dslash_test.cpp
index 107d2ccba1..e3290c2e2a 100644
--- a/tests/dslash_test.cpp
+++ b/tests/dslash_test.cpp
@@ -30,6 +30,12 @@ class DslashTest : public ::testing::Test
       printfQuda("Testing with split grid: %d  %d  %d  %d\n", grid_partition[0], grid_partition[1], grid_partition[2],
                  grid_partition[3]);
     }
+
+    if (dslash_test_wrapper.test_domain_decomposition) {
+      if (dd_red_black)
+        printfQuda("Testing DD Red Black with block: %d  %d  %d  %d\n", dd_block_size[0], dd_block_size[1],
+                   dd_block_size[2], dd_block_size[3]);
+    }
   }
 
 public:
diff --git a/tests/dslash_test_utils.h b/tests/dslash_test_utils.h
index 9f5fc44ce6..ebe98590b5 100644
--- a/tests/dslash_test_utils.h
+++ b/tests/dslash_test_utils.h
@@ -90,7 +90,7 @@ struct DslashTestWrapper {
 
   const bool transfer = false;
 
-  void init_ctest(int argc, char **argv, int precision, QudaReconstructType link_recon)
+  void init_ctest(int argc, char **argv, int precision, QudaReconstructType link_recon, int dd_value)
   {
     if (first_time) {
       gauge_param = newQudaGaugeParam();
@@ -118,6 +118,8 @@ struct DslashTestWrapper {
     inv_param.clover_cuda_prec_precondition = cuda_prec;
     inv_param.clover_cuda_prec_refinement_sloppy = cuda_prec;
 
+    init_domain_decomposition(dd_value);
+
     init();
   }
 
@@ -273,6 +275,35 @@ struct DslashTestWrapper {
     inv_param.verbosity = verbosity;
   }
 
+  void init_domain_decomposition(int value)
+  {
+    if (value == 0) {
+      test_domain_decomposition = false;
+      return;
+    }
+    test_domain_decomposition = true;
+
+    if (value < 3) {
+      dd_red_black = true;
+
+      // dd_block_size is half of the local lattice
+      if (value == 1) {
+        for (auto i = 0u; i < 4; i++) dd_block_size[i] = gauge_param.X[i] / 2;
+        return;
+      }
+
+      // dd_block_size is half of the global lattice
+      if (value == 2) {
+        for (auto i = 0u; i < 4; i++) dd_block_size[i] = (gauge_param.X[i] * comm_dim(i)) / 2;
+        return;
+      }
+
+    } else {
+      dd_red_black = false;
+    }
+    errorQuda("Unexpected value for domain decomposition (%d)", value);
+  }
+
   void init()
   {
     printfQuda("Sending gauge field to GPU\n");

From b0bd6a7860cbe036eeed6f38ce7aa9293abe686d Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Wed, 22 May 2024 12:37:59 +0200
Subject: [PATCH 049/108] Adding option --dd-test-projection

---
 tests/dslash_test_utils.h           | 7 +++++--
 tests/utils/command_line_params.cpp | 2 ++
 tests/utils/command_line_params.h   | 1 +
 3 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/tests/dslash_test_utils.h b/tests/dslash_test_utils.h
index ebe98590b5..495685a323 100644
--- a/tests/dslash_test_utils.h
+++ b/tests/dslash_test_utils.h
@@ -864,8 +864,6 @@ struct DslashTestWrapper {
           cudaSpinor.dd.reset();
           cudaSpinorTmp.dd.reset();
 
-          blas::xpy(cudaSpinorTmp, cudaSpinorOut);
-
           // We also test that Dyx is same as D applied to projected in and out spinors
           cudaSpinorTmp2 = cudaSpinor;
           cudaSpinorTmp2.dd.reset(DD::red_black_type, col % 2 == 0 ? DD::red_active : DD::black_active);
@@ -886,6 +884,11 @@ struct DslashTestWrapper {
           cudaSpinorTmp3.projectDD();
           cudaSpinorTmp3.dd.reset();
 
+          if (dd_test_projection)
+            blas::xpy(cudaSpinorTmp3, cudaSpinorOut);
+          else
+            blas::xpy(cudaSpinorTmp, cudaSpinorOut);
+
           spinorTmp = cudaSpinorTmp;
           spinorOut = cudaSpinorTmp3;
 
diff --git a/tests/utils/command_line_params.cpp b/tests/utils/command_line_params.cpp
index 1e27ab7b34..2b5501ceda 100644
--- a/tests/utils/command_line_params.cpp
+++ b/tests/utils/command_line_params.cpp
@@ -78,6 +78,7 @@ QudaAcceleratorType precon_accelerator_type = QUDA_INVALID_ACCELERATOR;
 
 std::array<int, 4> dd_block_size = {4, 4, 4, 4};
 bool dd_red_black = false;
+bool dd_test_projection = false;
 
 double madwf_diagonal_suppressor = 0.0;
 int madwf_ls = 4;
@@ -1067,6 +1068,7 @@ void add_dd_option_group(std::shared_ptr<QUDAApp> quda_app)
                  "Set the domain decomposition block size in all four dimension (default 4 4 4 4)")
     ->expected(4);
   opgroup->add_option("--dd-red-black", dd_red_black, "Enable red-black domain decomposition (default false)");
+  opgroup->add_option("--dd-test-projection", dd_red_black, "Compare against the projected result (default false)");
 }
 
 void add_madwf_option_group(std::shared_ptr<QUDAApp> quda_app)
diff --git a/tests/utils/command_line_params.h b/tests/utils/command_line_params.h
index 7f31f0cfe5..18ff5d9173 100644
--- a/tests/utils/command_line_params.h
+++ b/tests/utils/command_line_params.h
@@ -216,6 +216,7 @@ extern QudaAcceleratorType precon_accelerator_type;
 
 extern std::array<int, 4> dd_block_size;
 extern bool dd_red_black;
+extern bool dd_test_projection;
 
 extern double madwf_diagonal_suppressor;
 extern int madwf_ls;

From a549b639dbbc86ed449c496e6635402b5fda8dc1 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Wed, 22 May 2024 12:50:04 +0200
Subject: [PATCH 050/108] Adding option to disable tests on domain
 decomposition

---
 tests/dslash_ctest.cpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/dslash_ctest.cpp b/tests/dslash_ctest.cpp
index 751d14c94a..fd8a286e4b 100644
--- a/tests/dslash_ctest.cpp
+++ b/tests/dslash_ctest.cpp
@@ -7,6 +7,7 @@ int argc_copy;
 char **argv_copy;
 dslash_test_type dtest_type = dslash_test_type::Dslash;
 bool ctest_all_partitions = false;
+bool ctest_domain_decomposition = false; // currently disabled by default due to unresolved issues
 
 // For googletest names must be non-empty, unique, and may only contain ASCII
 // alphanumeric characters or underscore
@@ -41,8 +42,7 @@ class DslashTest : public ::testing::TestWithParam<::testing::tuple<int, int, in
                                                   true, false, false, false, true, false, true, true};
     if (!ctest_all_partitions && !partition_enabled[::testing::get<2>(GetParam())]) return true;
 
-    const std::array<bool, 3> domain_decomposition_enabled {true, true, false};
-    if (!ctest_all_partitions && !domain_decomposition_enabled[::testing::get<3>(GetParam())]) return true;
+    if (!ctest_domain_decomposition && ::testing::get<3>(GetParam())>0) return true;
 
     return false;
   }
@@ -141,6 +141,7 @@ int main(int argc, char **argv)
   auto app = make_app();
   app->add_option("--test", dtest_type, "Test method")->transform(CLI::CheckedTransformer(dtest_type_map));
   app->add_option("--all-partitions", ctest_all_partitions, "Test all instead of reduced combination of partitions");
+  app->add_option("--test-domain-decomposition", ctest_domain_decomposition, "Test domain decomposition");
   add_comms_option_group(app);
   try {
     app->parse(argc, argv);

From 822921ecb11cf49a49c0ec628ac759f09f17256a Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Wed, 22 May 2024 14:24:10 +0200
Subject: [PATCH 051/108] Start making DD-aware more operators

---
 include/kernels/dslash_domain_wall_4d.cuh | 12 ++++++++++--
 include/kernels/dslash_domain_wall_5d.cuh | 13 ++++++++++---
 include/kernels/dslash_twisted_mass.cuh   |  9 ++++++++-
 include/kernels/dslash_wilson.cuh         | 20 +++++++-------------
 include/kernels/dslash_wilson_clover.cuh  |  9 ++++++++-
 5 files changed, 43 insertions(+), 20 deletions(-)

diff --git a/include/kernels/dslash_domain_wall_4d.cuh b/include/kernels/dslash_domain_wall_4d.cuh
index e21375a7b1..961025722c 100644
--- a/include/kernels/dslash_domain_wall_4d.cuh
+++ b/include/kernels/dslash_domain_wall_4d.cuh
@@ -44,10 +44,18 @@ namespace quda
 
       const int my_spinor_parity = nParity == 2 ? parity : 0;
       Vector out;
-      applyWilson<nParity, dagger, mykernel_type>(out, arg, coord, parity, idx, thread_dim, active);
 
       int xs = coord.x_cb + s * arg.dc.volume_4d_cb;
-      if (xpay && mykernel_type == INTERIOR_KERNEL) {
+      if (arg.dd_out.isZero(coord)) {
+        if (mykernel_type != EXTERIOR_KERNEL_ALL || active) arg.out(xs, my_spinor_parity) = out;
+        return;
+      }
+
+      applyWilson<nParity, dagger, mykernel_type>(out, arg, coord, parity, idx, thread_dim, active);
+
+      if (xpay && mykernel_type == INTERIOR_KERNEL && arg.dd_x.isZero(coord)) {
+        out = arg.a_5[s] * out;
+      } else if (xpay && mykernel_type == INTERIOR_KERNEL) {
         Vector x = arg.x(xs, my_spinor_parity);
         out = x + arg.a_5[s] * out;
       } else if (mykernel_type != INTERIOR_KERNEL && active) {
diff --git a/include/kernels/dslash_domain_wall_5d.cuh b/include/kernels/dslash_domain_wall_5d.cuh
index dde72b6c2b..9f25a56272 100644
--- a/include/kernels/dslash_domain_wall_5d.cuh
+++ b/include/kernels/dslash_domain_wall_5d.cuh
@@ -46,13 +46,18 @@ namespace quda
       const int my_spinor_parity = nParity == 2 ? parity : 0;
       Vector out;
 
+      if (arg.dd_out.isZero(coord)) {
+        if (mykernel_type != EXTERIOR_KERNEL_ALL || active) arg.out(coord.x_cb, my_spinor_parity) = out;
+        return;
+      }
+
       applyWilson<nParity, dagger, mykernel_type>(out, arg, coord, parity, idx, thread_dim, active);
 
       if (mykernel_type == INTERIOR_KERNEL) { // 5th dimension derivative always local
         constexpr int d = 4;
         const int s = coord[4];
         const int their_spinor_parity = nParity == 2 ? 1 - parity : 0;
-        {
+        if (arg.dd_in.doHopping(coord, d, +1)) {
           const int fwd_idx = getNeighborIndexCB(coord, d, +1, arg.dc);
           constexpr int proj_dir = dagger ? +1 : -1;
           Vector in = arg.in(fwd_idx, their_spinor_parity);
@@ -63,7 +68,7 @@ namespace quda
           }
         }
 
-        {
+        if (arg.dd_in.doHopping(coord, d, -1)) {
           const int back_idx = getNeighborIndexCB(coord, d, -1, arg.dc);
           constexpr int proj_dir = dagger ? -1 : +1;
           Vector in = arg.in(back_idx, their_spinor_parity);
@@ -75,7 +80,9 @@ namespace quda
         }
       }
 
-      if (xpay && mykernel_type == INTERIOR_KERNEL) {
+      if (xpay && mykernel_type == INTERIOR_KERNEL and arg.dd_x.isZero(coord)) {
+        out = arg.a * out;
+      } else if (xpay && mykernel_type == INTERIOR_KERNEL) {
         Vector x = arg.x(coord.x_cb, my_spinor_parity);
         out = x + arg.a * out;
       } else if (mykernel_type != INTERIOR_KERNEL && active) {
diff --git a/include/kernels/dslash_twisted_mass.cuh b/include/kernels/dslash_twisted_mass.cuh
index 8da5b4493f..f5bee552ec 100644
--- a/include/kernels/dslash_twisted_mass.cuh
+++ b/include/kernels/dslash_twisted_mass.cuh
@@ -46,10 +46,17 @@ namespace quda
       const int my_spinor_parity = nParity == 2 ? parity : 0;
       Vector out;
 
+      if (arg.dd_out.isZero(coord)) {
+        if (mykernel_type != EXTERIOR_KERNEL_ALL || active) arg.out(coord.x_cb, my_spinor_parity) = out;
+        return;
+      }
+
       // defined in dslash_wilson.cuh
       applyWilson<nParity, dagger, mykernel_type>(out, arg, coord, parity, idx, thread_dim, active);
 
-      if (mykernel_type == INTERIOR_KERNEL) {
+      if (mykernel_type == INTERIOR_KERNEL and arg.dd_x.isZero(coord)) {
+        out = arg.a * out;
+      } else if (mykernel_type == INTERIOR_KERNEL) {
         Vector x = arg.x(coord.x_cb, my_spinor_parity);
         x += arg.b * x.igamma(4);
         out = x + arg.a * out;
diff --git a/include/kernels/dslash_wilson.cuh b/include/kernels/dslash_wilson.cuh
index 3f70fc6c87..5e718217e8 100644
--- a/include/kernels/dslash_wilson.cuh
+++ b/include/kernels/dslash_wilson.cuh
@@ -182,20 +182,14 @@ namespace quda
 
       applyWilson<nParity, dagger, mykernel_type>(out, arg, coord, parity, idx, thread_dim, active);
 
-      if (xpay && mykernel_type == INTERIOR_KERNEL) {
-        if (arg.dd_x.isZero(coord)) {
-          out = arg.a * out;
-        } else {
-          Vector x = arg.x(xs, my_spinor_parity);
-          out = x + arg.a * out;
-        }
+      if (xpay && mykernel_type == INTERIOR_KERNEL && arg.dd_x.isZero(coord)) {
+        out = arg.a * out;
+      } else if (xpay && mykernel_type == INTERIOR_KERNEL) {
+        Vector x = arg.x(xs, my_spinor_parity);
+        out = x + arg.a * out;
       } else if (mykernel_type != INTERIOR_KERNEL && active) {
-        if (arg.dd_out.isZero(coord)) {
-          out = (xpay ? arg.a * out : out);
-        } else {
-          Vector x = arg.out(xs, my_spinor_parity);
-          out = x + (xpay ? arg.a * out : out);
-        }
+        Vector x = arg.out(xs, my_spinor_parity);
+        out = x + (xpay ? arg.a * out : out);
       }
 
       if (mykernel_type != EXTERIOR_KERNEL_ALL || active) arg.out(xs, my_spinor_parity) = out;
diff --git a/include/kernels/dslash_wilson_clover.cuh b/include/kernels/dslash_wilson_clover.cuh
index 83f8f27e81..8d02138c28 100644
--- a/include/kernels/dslash_wilson_clover.cuh
+++ b/include/kernels/dslash_wilson_clover.cuh
@@ -62,10 +62,17 @@ namespace quda
       const int my_spinor_parity = nParity == 2 ? parity : 0;
       Vector out;
 
+      if (arg.dd_out.isZero(coord)) {
+        if (mykernel_type != EXTERIOR_KERNEL_ALL || active) arg.out(coord.x_cb, my_spinor_parity) = out;
+        return;
+      }
+
       // defined in dslash_wilson.cuh
       applyWilson<nParity, dagger, mykernel_type>(out, arg, coord, parity, idx, thread_dim, active);
 
-      if (mykernel_type == INTERIOR_KERNEL) {
+      if (mykernel_type == INTERIOR_KERNEL and arg.dd_x.isZero(coord)) {
+        out = arg.a * out;
+      } else if (mykernel_type == INTERIOR_KERNEL) {
         Vector x = arg.x(coord.x_cb, my_spinor_parity);
         x.toRel(); // switch to chiral basis
 

From 2635a9d561ca9da01061fda3d9ef2944e001808d Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Mon, 17 Jun 2024 16:57:52 +0200
Subject: [PATCH 052/108] Adding missing [src_idx]

---
 include/kernels/dslash_twisted_mass.cuh  | 2 +-
 include/kernels/dslash_wilson_clover.cuh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/include/kernels/dslash_twisted_mass.cuh b/include/kernels/dslash_twisted_mass.cuh
index 69235a9a84..b202fabb01 100644
--- a/include/kernels/dslash_twisted_mass.cuh
+++ b/include/kernels/dslash_twisted_mass.cuh
@@ -48,7 +48,7 @@ namespace quda
       Vector out;
 
       if (arg.dd_out.isZero(coord)) {
-        if (mykernel_type != EXTERIOR_KERNEL_ALL || active) arg.out(coord.x_cb, my_spinor_parity) = out;
+        if (mykernel_type != EXTERIOR_KERNEL_ALL || active) arg.out[src_idx](coord.x_cb, my_spinor_parity) = out;
         return;
       }
 
diff --git a/include/kernels/dslash_wilson_clover.cuh b/include/kernels/dslash_wilson_clover.cuh
index a712285457..5746c41014 100644
--- a/include/kernels/dslash_wilson_clover.cuh
+++ b/include/kernels/dslash_wilson_clover.cuh
@@ -64,7 +64,7 @@ namespace quda
       Vector out;
 
       if (arg.dd_out.isZero(coord)) {
-        if (mykernel_type != EXTERIOR_KERNEL_ALL || active) arg.out(coord.x_cb, my_spinor_parity) = out;
+        if (mykernel_type != EXTERIOR_KERNEL_ALL || active) arg.out[src_idx](coord.x_cb, my_spinor_parity) = out;
         return;
       }
 

From f0cf17534bf8552c4817cc789b1f38ac567b1b57 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Mon, 17 Jun 2024 16:58:24 +0200
Subject: [PATCH 053/108] Adding DD method for accessing and editing domain
 decompositions

---
 include/color_spinor_field.h                | 25 ++++++++---
 include/domain_decomposition.h              | 50 ++++++++++++++-------
 include/domain_decomposition_helper.cuh     | 18 ++++----
 include/dslash_helper.cuh                   |  6 +--
 include/instantiate_dslash.h                | 13 +++---
 include/kernels/color_spinor_project_dd.cuh |  2 +-
 include/lattice_field.h                     |  6 +--
 include/reference_wrapper_helper.h          | 16 +++++++
 lib/color_spinor_project_dd.cu              |  8 ++--
 lib/inv_mr_quda.cpp                         | 12 ++---
 tests/dslash_test_utils.h                   | 24 +++++-----
 11 files changed, 113 insertions(+), 67 deletions(-)

diff --git a/include/color_spinor_field.h b/include/color_spinor_field.h
index 285664b8a7..f5d3f449a6 100644
--- a/include/color_spinor_field.h
+++ b/include/color_spinor_field.h
@@ -357,6 +357,9 @@ namespace quda
     //
     CompositeColorSpinorField components;
 
+    /** Domain decomposition options */
+    DDParam dd {};
+
     /**
        Compute the required extended ghost zone sizes and offsets
        @param[in] nFace The depth of the halo
@@ -385,9 +388,6 @@ namespace quda
     inline static size_t ghostFaceBytes[QUDA_MAX_DIM] = {};
     static void freeGhostBuffer(void);
 
-    /** Domain decomposition options */
-    DDParam dd {};
-
     /**
        @brief Default constructor
     */
@@ -443,12 +443,25 @@ namespace quda
     void copy(const ColorSpinorField &src);
 
     /**
-       @brief Project the field to a domain determined by
-       ddParam
+       @brief Project the field to a domain determined by DDParam
      */
-
     void projectDD();
 
+    /**
+       @brief Returns DDParam
+     */
+    DDParam DD() const { return dd; }
+
+    /**
+       @brief Sets DDParam from a given DDParam
+     */
+    void DD(const DDParam &in) { dd = in; }
+
+    /**
+       @brief Sets DDParam from a given list of options (DD flags)
+     */
+    template <typename... Args> void DD(const quda::DD &flag, const Args &...args) { dd.set(flag, args...); }
+
     /**
        @brief Zero all elements of this field
      */
diff --git a/include/domain_decomposition.h b/include/domain_decomposition.h
index 2cfb2e90a2..a27632c2f4 100644
--- a/include/domain_decomposition.h
+++ b/include/domain_decomposition.h
@@ -9,6 +9,8 @@ namespace quda
 
   DECLARE_ENUM(DD, // name of the enum class
 
+               reset, // No domain decomposition. It sets all flags to zero.
+
                red_black_type,   // Flags used by red_black
                red_active,       // if red blocks are active
                black_active,     // if black blocks are active
@@ -36,25 +38,18 @@ namespace quda
     {
       flags[(int)flag] = true;
 
-      if ((int)flag >= (int)DD::red_black_type) type = QUDA_DD_RED_BLACK;
-    }
-
-    template <typename... Args> constexpr void set(const DD &flag, Args... args)
-    {
-      set(flag);
-      set(args...);
-    }
-
-    constexpr void reset()
-    {
-      type = QUDA_DD_NO;
+      if ((int)flag == (int)DD::reset) {
 #pragma unroll
-      for (auto i = 0u; i < (int)DD::size; i++) flags[i] = 0;
+        for (auto i = 0u; i < (int)DD::size; i++) flags[i] = 0;
+        type = QUDA_DD_NO;
+      } else if ((int)flag >= (int)DD::red_black_type) {
+        type = QUDA_DD_RED_BLACK;
+      }
     }
 
-    template <typename... Args> constexpr void reset(Args... args)
+    template <typename... Args> constexpr void set(const DD &flag, const Args &...args)
     {
-      reset();
+      set(flag);
       set(args...);
     }
 
@@ -82,7 +77,7 @@ namespace quda
             if (verbose) printfQuda("blockDim[%d] = %d is not positive \n", i, blockDim[i]);
             return false;
           }
-          int globalDim = comm_dim(i) * field.full_dim(i);
+          int globalDim = comm_dim(i) * field.X(i);
           if (globalDim % blockDim[i] != 0) {
             if (verbose) printfQuda("blockDim[%d] = %d does not divide %d \n", i, blockDim[i], globalDim);
             return false;
@@ -123,6 +118,29 @@ namespace quda
 
       return true;
     }
+
+    // Checks if this is equal to given DDParam
+    inline bool operator==(const DDParam &dd) const
+    {
+      // if both are not in use we return true
+      if (not *this and not dd) return true;
+
+      // false if type does not match
+      if (type != dd.type) return false;
+
+      // checking all flags matches (note this should be actually type-wise)
+      for (int i = 0; i < (int)DD::size; i++)
+        if (flags[i] != dd.flags[i]) return false;
+
+      // checking blockDim matches when needed
+      if (type == QUDA_DD_RED_BLACK)
+        for (int i = 0; i < QUDA_MAX_DIM; i++)
+          if (blockDim[i] != dd.blockDim[i]) return false;
+
+      return true;
+    }
+
+    inline bool operator!=(const DDParam &dd) { return !(*this == dd); }
   };
 
 } // namespace quda
diff --git a/include/domain_decomposition_helper.cuh b/include/domain_decomposition_helper.cuh
index 489537115f..7b3418e9a9 100644
--- a/include/domain_decomposition_helper.cuh
+++ b/include/domain_decomposition_helper.cuh
@@ -10,9 +10,9 @@ namespace quda
   struct DDNo {
 
     // Initialization of input parameters from ColorSpinorField
-    DDNo(const ColorSpinorField &in)
+    DDNo(const DDParam &dd)
     {
-      if (in.dd.type != QUDA_DD_NO) { errorQuda("Unsupported type %d\n", in.dd.type); }
+      if (dd.type != QUDA_DD_NO) { errorQuda("Unsupported type %d\n", dd.type); }
     }
 
     // Only DDNo returns true. All others return false
@@ -39,15 +39,13 @@ namespace quda
     const bool black_active;       // if black blocks are active
     const bool block_hopping;      // if hopping between red and black is allowed
 
-    DDRedBlack(const ColorSpinorField &in) :
-      blockDim {in.dd.blockDim[0], in.dd.blockDim[1], in.dd.blockDim[2], in.dd.blockDim[3]},
-      red_active(in.dd.type == QUDA_DD_NO or in.dd.is(DD::red_active)),
-      black_active(in.dd.type == QUDA_DD_NO or in.dd.is(DD::black_active)),
-      block_hopping(in.dd.type == QUDA_DD_NO or not in.dd.is(DD::no_block_hopping))
+    DDRedBlack(const DDParam &dd) :
+      blockDim {dd.blockDim[0], dd.blockDim[1], dd.blockDim[2], dd.blockDim[3]},
+      red_active(dd.type == QUDA_DD_NO or dd.is(DD::red_active)),
+      black_active(dd.type == QUDA_DD_NO or dd.is(DD::black_active)),
+      block_hopping(dd.type == QUDA_DD_NO or not dd.is(DD::no_block_hopping))
     {
-      if (in.dd.type != QUDA_DD_NO and in.dd.type != QUDA_DD_RED_BLACK) {
-        errorQuda("Unsupported type %d\n", in.dd.type);
-      }
+      if (dd.type != QUDA_DD_NO and dd.type != QUDA_DD_RED_BLACK) { errorQuda("Unsupported type %d\n", dd.type); }
     }
 
     constexpr bool operator!() const { return false; }
diff --git a/include/dslash_helper.cuh b/include/dslash_helper.cuh
index 77259209fc..e728620c1d 100644
--- a/include/dslash_helper.cuh
+++ b/include/dslash_helper.cuh
@@ -350,9 +350,9 @@ namespace quda
       pack_blocks(0),
       exterior_dims(0),
       exterior_blocks(0),
-      dd_out(out),
-      dd_in(in),
-      dd_x(x),
+      dd_out(out.DD()),
+      dd_in(in.DD()),
+      dd_x(x.DD()),
 #ifndef NVSHMEM_COMMS
       counter(0)
 #else
diff --git a/include/instantiate_dslash.h b/include/instantiate_dslash.h
index 4808d67259..bc248a7865 100644
--- a/include/instantiate_dslash.h
+++ b/include/instantiate_dslash.h
@@ -51,16 +51,17 @@ namespace quda
   */
   template <template <typename, int, typename, QudaReconstructType> class Apply, typename Recon, typename Float,
             int nColor, typename... Args>
-  inline void instantiate(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, Args &&...args)
+  inline void instantiate(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+                          cvector_ref<const ColorSpinorField> &x, const GaugeField &U, Args &&...args)
   {
-    if (out.dd.type == QUDA_DD_NO and in.dd.type == QUDA_DD_NO) {
-      instantiate<Apply, Recon, Float, 3, DDNo>(out, in, U, args...);
+    if (out.DD().type == QUDA_DD_NO and in.DD().type == QUDA_DD_NO) {
+      instantiate<Apply, Recon, Float, 3, DDNo>(out, in, x, U, args...);
 #ifdef GPU_DD_DIRAC
-    } else if (out.dd.type == QUDA_DD_RED_BLACK or in.dd.type == QUDA_DD_RED_BLACK) {
-      instantiate<Apply, Recon, Float, 3, DDRedBlack>(out, in, U, args...);
+    } else if (out.DD().type == QUDA_DD_RED_BLACK or in.DD().type == QUDA_DD_RED_BLACK) {
+      instantiate<Apply, Recon, Float, 3, DDRedBlack>(out, in, x, U, args...);
 #endif
     } else {
-      errorQuda("Unsupported DD type %d\n", out.dd.type);
+      errorQuda("Unsupported DD type %d\n", out.DD().type);
     }
   }
 
diff --git a/include/kernels/color_spinor_project_dd.cuh b/include/kernels/color_spinor_project_dd.cuh
index 30bb7b0221..064c0b90cb 100644
--- a/include/kernels/color_spinor_project_dd.cuh
+++ b/include/kernels/color_spinor_project_dd.cuh
@@ -23,7 +23,7 @@ namespace quda
     ProjectDDArg(ColorSpinorField &out) :
       kernel_param(dim3(out.VolumeCB(), out.SiteSubset(), 1)),
       out(out),
-      dd(out),
+      dd(out.DD()),
       parity(out.SiteOrder() == QUDA_ODD_EVEN_SITE_ORDER ? 1 : 0),
       nParity(out.SiteSubset()),
       X0h(nParity == 2 ? out.X(0) / 2 : out.X(0)),
diff --git a/include/lattice_field.h b/include/lattice_field.h
index 35724e986e..2b9935481b 100644
--- a/include/lattice_field.h
+++ b/include/lattice_field.h
@@ -940,9 +940,9 @@ namespace quda {
   {
     const unwrap_t<T1> &a(a_);
     const unwrap_t<T2> &b(b_);
-    if (!a.dd.check(a, true)) errorQuda("DD checks not passed (%s:%d in %s())", file, line, func);
-    if (!b.dd.check(b, true)) errorQuda("DD checks not passed (%s:%d in %s())", file, line, func);
-    if (!a.dd.match(b.dd, true)) errorQuda("DD not match (%s:%d in %s())", file, line, func);
+    if (!a.DD().check(a, true)) errorQuda("DD checks not passed (%s:%d in %s())", file, line, func);
+    if (!b.DD().check(b, true)) errorQuda("DD checks not passed (%s:%d in %s())", file, line, func);
+    if (!a.DD().match(b.DD(), true)) errorQuda("DD not match (%s:%d in %s())", file, line, func);
     return true;
   }
 
diff --git a/include/reference_wrapper_helper.h b/include/reference_wrapper_helper.h
index a9eccf3648..5b2fe35828 100644
--- a/include/reference_wrapper_helper.h
+++ b/include/reference_wrapper_helper.h
@@ -7,6 +7,7 @@
 #include <enum_quda.h>
 #include <util_quda.h>
 #include <quda_internal.h>
+#include <domain_decomposition.h>
 
 namespace quda
 {
@@ -462,6 +463,21 @@ namespace quda
     {
       return operator[](0).AuxString();
     }
+
+    template <class U = T>
+    std::enable_if_t<std::is_same_v<std::remove_const_t<U>, ColorSpinorField>, DDParam &> DD() const
+    {
+      for (auto i = 1u; i < vector::size(); i++)
+        if (operator[](i - 1).DD() != operator[](i).DD()) errorQuda("DD do not match %d != %d", i - 1, i);
+      return operator[](0).DD();
+    }
+
+    template <class U = T, typename... Args>
+    std::enable_if_t<std::is_same_v<std::remove_const_t<U>, ColorSpinorField>, void> DD(const quda::DD &flag,
+                                                                                        const Args &...args)
+    {
+      for (auto i = 0u; i < vector::size(); i++) operator[](i).DD(flag, args...);
+    }
   };
 
   template <class T> using cvector_ref = const vector_ref<T>;
diff --git a/lib/color_spinor_project_dd.cu b/lib/color_spinor_project_dd.cu
index 6e4459fd1a..8843a1d2ea 100644
--- a/lib/color_spinor_project_dd.cu
+++ b/lib/color_spinor_project_dd.cu
@@ -25,10 +25,10 @@ namespace quda
     ProjectDD(ColorSpinorField &out) : TunableKernel2D(out, out.SiteSubset()), out(out)
     {
       strcat(aux, out.AuxString().c_str());
-      switch (out.dd.type) {
+      switch (out.DD().type) {
       case QUDA_DD_NO: strcat(aux, ",DDNo"); break;
       case QUDA_DD_RED_BLACK: strcat(aux, ",DDRedBlack"); break;
-      default: errorQuda("DD type %d not implemented", out.dd.type);
+      default: errorQuda("DD type %d not implemented", out.DD().type);
       }
 
       apply(device::get_default_stream());
@@ -124,10 +124,10 @@ namespace quda
 
   template <typename Float> void genericProjectDD(ColorSpinorField &a)
   {
-    switch (a.dd.type) {
+    switch (a.DD().type) {
     case QUDA_DD_NO: genericProjectDD<Float, DDNo>(a); break;
     case QUDA_DD_RED_BLACK: genericProjectDD<Float, DDRedBlack>(a); break;
-    default: errorQuda("DD type %d not implemented", a.dd.type);
+    default: errorQuda("DD type %d not implemented", a.DD().type);
     }
   }
 
diff --git a/lib/inv_mr_quda.cpp b/lib/inv_mr_quda.cpp
index 32ad1d66c7..324f3befc1 100644
--- a/lib/inv_mr_quda.cpp
+++ b/lib/inv_mr_quda.cpp
@@ -107,12 +107,12 @@ namespace quda
         if (param.do_block_schwarz()) {
           if (param.schwarz_type == QUDA_MULTIPLICATIVE_SCHWARZ) {
             // Red or black active
-            Ar.dd.reset(DD::red_black_type, step % 2 == 0 ? DD::red_active : DD::black_active);
-            r_sloppy.dd.reset(DD::red_black_type, step % 2 == 0 ? DD::red_active : DD::black_active);
+            Ar.DD(DD::reset, DD::red_black_type, step % 2 == 0 ? DD::red_active : DD::black_active);
+            r_sloppy.DD(DD::red_black_type, step % 2 == 0 ? DD::red_active : DD::black_active);
           } else {
             // Both red and black active but no hopping
-            Ar.dd.reset(DD::red_black_type, DD::red_active, DD::black_active, DD::no_block_hopping);
-            r_sloppy.dd.reset(DD::red_black_type, DD::red_active, DD::black_active, DD::no_block_hopping);
+            Ar.DD(DD::reset, DD::red_black_type, DD::red_active, DD::black_active, DD::no_block_hopping);
+            r_sloppy.DD(DD::reset, DD::red_black_type, DD::red_active, DD::black_active, DD::no_block_hopping);
           }
         }
 
@@ -157,8 +157,8 @@ namespace quda
 
         if (param.do_block_schwarz()) {
           // Disable domain decomposition
-          Ar.dd.reset();
-          r_sloppy.dd.reset();
+          Ar.DD(DD::reset);
+          r_sloppy.DD(DD::reset);
         }
 
         commGlobalReductionPop(); // renable global reductions for outer solver
diff --git a/tests/dslash_test_utils.h b/tests/dslash_test_utils.h
index c53daa9a1a..05ee37e69c 100644
--- a/tests/dslash_test_utils.h
+++ b/tests/dslash_test_utils.h
@@ -852,10 +852,10 @@ struct DslashTestWrapper {
 
       if (dd_red_black) {
         for (auto i = 0u; i < 4; i++) {
-          cudaSpinor.dd.blockDim[i] = dd_block_size[i];
-          cudaSpinorTmp.dd.blockDim[i] = dd_block_size[i];
-          cudaSpinorTmp2.dd.blockDim[i] = dd_block_size[i];
-          cudaSpinorTmp3.dd.blockDim[i] = dd_block_size[i];
+          cudaSpinor.DD().blockDim[i] = dd_block_size[i];
+          cudaSpinorTmp.DD().blockDim[i] = dd_block_size[i];
+          cudaSpinorTmp2.DD().blockDim[i] = dd_block_size[i];
+          cudaSpinorTmp3.DD().blockDim[i] = dd_block_size[i];
         }
 
         blas::zero(cudaSpinorOut);
@@ -863,8 +863,8 @@ struct DslashTestWrapper {
         // We test that applying all 4 possible operators Drr, Drb, Dbr, Dbb gived D
         for (int col = 0; col < 4; col++) {
 
-          cudaSpinor.dd.reset(DD::red_black_type, col % 2 == 0 ? DD::red_active : DD::black_active);
-          cudaSpinorTmp.dd.reset(DD::red_black_type, col / 2 == 0 ? DD::red_active : DD::black_active);
+          cudaSpinor.DD(DD::reset, DD::red_black_type, col % 2 == 0 ? DD::red_active : DD::black_active);
+          cudaSpinorTmp.DD(DD::reset, DD::red_black_type, col / 2 == 0 ? DD::red_active : DD::black_active);
 
           switch (dtest_type) {
           case dslash_test_type::Dslash: dirac->Dslash(cudaSpinorTmp, cudaSpinor, parity); break;
@@ -876,14 +876,14 @@ struct DslashTestWrapper {
             errorQuda("Test type %s not support for current Dslash", get_string(dtest_type_map, dtest_type).c_str());
           }
 
-          cudaSpinor.dd.reset();
-          cudaSpinorTmp.dd.reset();
+          cudaSpinor.DD(DD::reset);
+          cudaSpinorTmp.DD(DD::reset);
 
           // We also test that Dyx is same as D applied to projected in and out spinors
           cudaSpinorTmp2 = cudaSpinor;
-          cudaSpinorTmp2.dd.reset(DD::red_black_type, col % 2 == 0 ? DD::red_active : DD::black_active);
+          cudaSpinorTmp2.DD(DD::reset, DD::red_black_type, col % 2 == 0 ? DD::red_active : DD::black_active);
           cudaSpinorTmp2.projectDD();
-          cudaSpinorTmp2.dd.reset();
+          cudaSpinorTmp2.DD(DD::reset);
 
           switch (dtest_type) {
           case dslash_test_type::Dslash: dirac->Dslash(cudaSpinorTmp3, cudaSpinorTmp2, parity); break;
@@ -895,9 +895,9 @@ struct DslashTestWrapper {
             errorQuda("Test type %s not support for current Dslash", get_string(dtest_type_map, dtest_type).c_str());
           }
 
-          cudaSpinorTmp3.dd.reset(DD::red_black_type, col / 2 == 0 ? DD::red_active : DD::black_active);
+          cudaSpinorTmp3.DD(DD::reset, DD::red_black_type, col / 2 == 0 ? DD::red_active : DD::black_active);
           cudaSpinorTmp3.projectDD();
-          cudaSpinorTmp3.dd.reset();
+          cudaSpinorTmp3.DD(DD::reset);
 
           if (dd_test_projection)
             blas::xpy(cudaSpinorTmp3, cudaSpinorOut);

From 34d0305d5bcf8601b0d28b5d7ca87a43260edea1 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Mon, 17 Jun 2024 17:02:56 +0200
Subject: [PATCH 054/108] Restoring develop version of lib/dslash_policy.hpp

---
 lib/dslash_policy.hpp | 3231 ++++++++++++++++++++---------------------
 1 file changed, 1594 insertions(+), 1637 deletions(-)

diff --git a/lib/dslash_policy.hpp b/lib/dslash_policy.hpp
index 49e0ee8822..dce21f1be3 100644
--- a/lib/dslash_policy.hpp
+++ b/lib/dslash_policy.hpp
@@ -50,7 +50,9 @@ namespace quda
       int completeSum;
 
       inline DslashCommsPattern(const int commDim[], bool gdr_send = false) :
-        commsCompleted {}, dslashCompleted {}, completeSum(0)
+        commsCompleted {},
+        dslashCompleted {},
+        completeSum(0)
       {
 
         for (int i = 0; i < nStream - 1; i++) gatherCompleted[i] = gdr_send ? 1 : 0;
@@ -103,9 +105,9 @@ namespace quda
 
 #undef DSLASH_PROFILE
 #ifdef DSLASH_PROFILE
-#define PROFILE(f, profile, idx)                                                                                       \
-  profile.TPSTART(idx);                                                                                                \
-  f;                                                                                                                   \
+#define PROFILE(f, profile, idx)		\
+  profile.TPSTART(idx);				\
+  f;						\
   profile.TPSTOP(idx);
 #define PROFILE_START(idx) profile.TPSTART(idx);
 #define PROFILE_STOP(idx) profile.TPSTOP(idx);
@@ -115,13 +117,14 @@ namespace quda
 #define PROFILE_STOP(profile, idx)
 #endif
 
-    /**
-       @brief This helper function simply posts all receives in all directions
-       @param[out] input Field that we are doing halo exchange
-       @param[in] dslash The dslash object
-       @param[in] stream Stream were the receive is being posted (effectively ignored)
-       @param[in] gdr Whether we are using GPU Direct RDMA or not
-    */
+
+  /**
+     @brief This helper function simply posts all receives in all directions
+     @param[out] input Field that we are doing halo exchange
+     @param[in] dslash The dslash object
+     @param[in] stream Stream were the receive is being posted (effectively ignored)
+     @param[in] gdr Whether we are using GPU Direct RDMA or not
+  */
     template <typename Dslash> inline void issueRecv(const ColorSpinorField &halo, const Dslash &dslash, bool gdr)
     {
       for (int i = 3; i >= 0; i--) {
@@ -131,1976 +134,1930 @@ namespace quda
                   QUDA_PROFILE_COMMS_START);
         }
       }
-    }
-
-    /**
-       @brief This helper function simply posts the packing kernel needed for halo exchange
-       @param[out] in Field that we are packing
-       @param[in] dslash The dslash object
-       @param[in] parity Field parity
-       @param[in] location Memory location where we are packing to
-       - if Host is requested, the non-p2p halos will be sent to host
-       - if Remote is requested, the p2p halos will be written directly
-       @param[in] packIndex Stream index where the packing kernel will run
-    */
-    template <typename Dslash>
-    inline void issuePack(const ColorSpinorField &halo, cvector_ref<const ColorSpinorField> &in, const Dslash &dslash,
-                          int parity, MemoryLocation location, int packIndex, int shmem = 0)
-    {
-      auto &arg = dslash.dslashParam;
-      if ((location & Device) & Host) errorQuda("MemoryLocation cannot be both Device and Host");
-
-      bool pack = false;
-      for (int i = 3; i >= 0; i--)
-        if (arg.commDim[i]) {
-          pack = true;
-          break;
-        }
+  }
+
+  /**
+     @brief This helper function simply posts the packing kernel needed for halo exchange
+     @param[out] in Field that we are packing
+     @param[in] dslash The dslash object
+     @param[in] parity Field parity
+     @param[in] location Memory location where we are packing to
+     - if Host is requested, the non-p2p halos will be sent to host
+     - if Remote is requested, the p2p halos will be written directly
+     @param[in] packIndex Stream index where the packing kernel will run
+  */
+  template <typename Dslash>
+  inline void issuePack(const ColorSpinorField &halo, cvector_ref<const ColorSpinorField> &in, const Dslash &dslash,
+                        int parity, MemoryLocation location, int packIndex, int shmem = 0)
+  {
+    auto &arg = dslash.dslashParam;
+    if ( (location & Device) & Host) errorQuda("MemoryLocation cannot be both Device and Host");
+
+    bool pack = false;
+    for (int i=3; i>=0; i--)
+      if (arg.commDim[i]) {
+        pack = true;
+        break;
+      }
 
-      MemoryLocation pack_dest[2 * QUDA_MAX_DIM];
-      for (int dim = 0; dim < 4; dim++) {
-        for (int dir = 0; dir < 2; dir++) {
-          if ((location & Shmem)) {
-            pack_dest[2 * dim + dir] = Shmem; // pack to p2p remote
-          } else if ((location & Remote) && comm_peer2peer_enabled(dir, dim)) {
-            pack_dest[2 * dim + dir] = Remote; // pack to p2p remote
-          } else if (location & Host && !comm_peer2peer_enabled(dir, dim)) {
-            pack_dest[2 * dim + dir] = Host; // pack to cpu memory
-          } else {
-            pack_dest[2 * dim + dir] = Device; // pack to local gpu memory
-          }
+    MemoryLocation pack_dest[2*QUDA_MAX_DIM];
+    for (int dim=0; dim<4; dim++) {
+      for (int dir=0; dir<2; dir++) {
+        if ((location & Shmem)) {
+          pack_dest[2 * dim + dir] = Shmem; // pack to p2p remote
+        } else if ((location & Remote) && comm_peer2peer_enabled(dir, dim)) {
+          pack_dest[2*dim+dir] = Remote; // pack to p2p remote
+        } else if (location & Host && !comm_peer2peer_enabled(dir, dim)) {
+          pack_dest[2*dim+dir] = Host;   // pack to cpu memory
+        } else {
+          pack_dest[2*dim+dir] = Device; // pack to local gpu memory
         }
       }
-      if (pack) {
-        PROFILE(if (dslash_pack_compute)
-                  halo.pack(dslash.Nface() / 2, parity, dslash.Dagger(), device::get_stream(packIndex), pack_dest,
-                            location, arg.spin_project, arg.twist_a, arg.twist_b, arg.twist_c, shmem, in),
-                profile, QUDA_PROFILE_PACK_KERNEL);
-
-        // Record the end of the packing
-        PROFILE(if (location != Host) qudaEventRecord(packEnd[halo.bufferIndex], device::get_stream(packIndex)),
-                profile, QUDA_PROFILE_EVENT_RECORD);
-      }
     }
+    if (pack) {
+      PROFILE(if (dslash_pack_compute)
+                halo.pack(dslash.Nface() / 2, parity, dslash.Dagger(), device::get_stream(packIndex), pack_dest,
+                          location, arg.spin_project, arg.twist_a, arg.twist_b, arg.twist_c, shmem, in),
+              profile, QUDA_PROFILE_PACK_KERNEL);
+
+      // Record the end of the packing
+      PROFILE(if (location != Host) qudaEventRecord(packEnd[halo.bufferIndex], device::get_stream(packIndex)), profile,
+              QUDA_PROFILE_EVENT_RECORD);
+    }
+  }
+
+  /**
+     @brief This helper function simply posts the device-host memory
+     copies of all halos in all dimensions and directions
+     @param[out] in Field that whose halos we are communicating
+     @param[in] dslash The dslash object
+  */
+  template <typename Dslash> inline void issueGather(const ColorSpinorField &halo, const Dslash &dslash)
+  {
 
-    /**
-       @brief This helper function simply posts the device-host memory
-       copies of all halos in all dimensions and directions
-       @param[out] in Field that whose halos we are communicating
-       @param[in] dslash The dslash object
-    */
-    template <typename Dslash> inline void issueGather(const ColorSpinorField &halo, const Dslash &dslash)
-    {
-
-      for (int i = 3; i >= 0; i--) {
-        if (!dslash.dslashParam.commDim[i]) continue;
+    for (int i = 3; i >=0; i--) {
+      if (!dslash.dslashParam.commDim[i]) continue;
 
-        for (int dir = 1; dir >= 0; dir--) { // forwards gather
-          auto &event = packEnd[halo.bufferIndex];
+      for (int dir=1; dir>=0; dir--) { // forwards gather
+        auto &event = packEnd[halo.bufferIndex];
 
-          PROFILE(qudaStreamWaitEvent(device::get_stream(2 * i + dir), event, 0), profile,
-                  QUDA_PROFILE_STREAM_WAIT_EVENT);
+        PROFILE(qudaStreamWaitEvent(device::get_stream(2*i+dir), event, 0), profile, QUDA_PROFILE_STREAM_WAIT_EVENT);
 
-          // Initialize host transfer from source spinor
-          PROFILE(if (dslash_copy) halo.gather(2 * i + dir, device::get_stream(2 * i + dir)), profile,
-                  QUDA_PROFILE_GATHER);
+        // Initialize host transfer from source spinor
+        PROFILE(if (dslash_copy) halo.gather(2 * i + dir, device::get_stream(2 * i + dir)), profile, QUDA_PROFILE_GATHER);
 
-          // Record the end of the gathering if not peer-to-peer
-          if (!comm_peer2peer_enabled(dir, i)) {
-            PROFILE(qudaEventRecord(gatherEnd[2 * i + dir], device::get_stream(2 * i + dir)), profile,
-                    QUDA_PROFILE_EVENT_RECORD);
-          }
-        }
+        // Record the end of the gathering if not peer-to-peer
+	if (!comm_peer2peer_enabled(dir,i)) {
+	  PROFILE(qudaEventRecord(gatherEnd[2*i+dir], device::get_stream(2*i+dir)), profile, QUDA_PROFILE_EVENT_RECORD);
+	}
       }
     }
-
-    /**
-       @brief Returns a stream index for posting the pack/scatters to.
-       We desire a stream index that is not being used for peer-to-peer
-       communication.  This is used by the fused halo dslash kernels
-       where we post all scatters to the same stream so we only have a
-       single event to wait on before the exterior kernel is applied,
-       and by the zero-copy dslash kernels where we want to post the
-       packing kernel to an unused stream.
-       @return stream index
-    */
-    template <typename T> inline int getStreamIndex(const T &dslashParam)
-    {
-      // set index to a stream index not being used for p2p
-      int index = -1;
-      for (int i = 3; i >= 0; i--) {
-        if (!dslashParam.commDim[i]) continue;
-        if (!comm_peer2peer_enabled(0, i))
-          index = 2 * i + 0;
-        else if (!comm_peer2peer_enabled(1, i))
-          index = 2 * i + 1;
-      }
-      // make sure we pick a valid index, in case we are fully p2p connected
-      if (index == -1) index = 0;
-      return index;
+  }
+
+  /**
+     @brief Returns a stream index for posting the pack/scatters to.
+     We desire a stream index that is not being used for peer-to-peer
+     communication.  This is used by the fused halo dslash kernels
+     where we post all scatters to the same stream so we only have a
+     single event to wait on before the exterior kernel is applied,
+     and by the zero-copy dslash kernels where we want to post the
+     packing kernel to an unused stream.
+     @return stream index
+  */
+  template <typename T>
+  inline int getStreamIndex(const T &dslashParam) {
+    // set index to a stream index not being used for p2p
+    int index = -1;
+    for (int i = 3; i >=0; i--) {
+      if (!dslashParam.commDim[i]) continue;
+      if (!comm_peer2peer_enabled(0,i)) index = 2*i+0;
+      else if (!comm_peer2peer_enabled(1,i)) index = 2*i+1;
     }
+    // make sure we pick a valid index, in case we are fully p2p connected
+    if (index == -1) index = 0;
+    return index;
+  }
+
+  /**
+     @brief Wrapper for querying if communication is finished in the
+     dslash, and if it is take the appropriate action:
+
+     - if peer-to-peer then we now know that the peer-to-peer copy is
+       now in flight and we are safe to post the cudaStreamWaitEvent
+       in our GPU context
+
+     - if gdr or zero-copy for the receive buffer then we have nothing
+       else to do, it is now safe to post halo kernel
+
+     - if basic staging, we post the scatter (host to device memory copy)
+
+     @param[in,out] in Field being commicated
+     @param[in] dslash The dslash object (unused)
+     @param[in] dim Dimension we are working on
+     @param[in] dir Direction we are working on
+     @param[in] gdr_send Whether GPU Direct RDMA is being used for sending
+     @param[in] gdr_recv Whether GPU Direct RDMA is being used for receiving
+     @param[in] zero_copy_recv Whether we are using zero-copy on the
+     receive end (and hence do not need to do CPU->GPU copy)
+     @param[in] scatterIndex The stream index used for posting the host-to-device memory copy in
+   */
+  template <typename Dslash>
+  inline bool commsComplete(const ColorSpinorField &halo, const Dslash &, int dim, int dir, bool gdr_send,
+                            bool gdr_recv, bool zero_copy_recv, int scatterIndex = -1)
+  {
+    PROFILE(int comms_test
+            = dslash_comms ? halo.commsQuery(2 * dim + dir, device::get_stream(2 * dim + dir), gdr_send, gdr_recv) : 1,
+            profile, QUDA_PROFILE_COMMS_QUERY);
+    if (comms_test) {
+      // now we are receive centric
+      int dir2 = 1-dir;
+
+      // if peer-2-peer in a given direction then we need to insert a wait on that copy event
+      if (comm_peer2peer_enabled(dir2,dim)) {
+        PROFILE(qudaStreamWaitEvent(device::get_default_stream(), halo.getIPCRemoteCopyEvent(dir2, dim), 0), profile,
+                QUDA_PROFILE_STREAM_WAIT_EVENT);
+      } else {
 
-    /**
-       @brief Wrapper for querying if communication is finished in the
-       dslash, and if it is take the appropriate action:
-
-       - if peer-to-peer then we now know that the peer-to-peer copy is
-         now in flight and we are safe to post the cudaStreamWaitEvent
-         in our GPU context
-
-       - if gdr or zero-copy for the receive buffer then we have nothing
-         else to do, it is now safe to post halo kernel
-
-       - if basic staging, we post the scatter (host to device memory copy)
-
-       @param[in,out] in Field being commicated
-       @param[in] dslash The dslash object (unused)
-       @param[in] dim Dimension we are working on
-       @param[in] dir Direction we are working on
-       @param[in] gdr_send Whether GPU Direct RDMA is being used for sending
-       @param[in] gdr_recv Whether GPU Direct RDMA is being used for receiving
-       @param[in] zero_copy_recv Whether we are using zero-copy on the
-       receive end (and hence do not need to do CPU->GPU copy)
-       @param[in] scatterIndex The stream index used for posting the host-to-device memory copy in
-     */
-    template <typename Dslash>
-    inline bool commsComplete(const ColorSpinorField &halo, const Dslash &, int dim, int dir, bool gdr_send,
-                              bool gdr_recv, bool zero_copy_recv, int scatterIndex = -1)
-    {
-      PROFILE(int comms_test
-              = dslash_comms ? halo.commsQuery(2 * dim + dir, device::get_stream(2 * dim + dir), gdr_send, gdr_recv) : 1,
-              profile, QUDA_PROFILE_COMMS_QUERY);
-      if (comms_test) {
-        // now we are receive centric
-        int dir2 = 1 - dir;
-
-        // if peer-2-peer in a given direction then we need to insert a wait on that copy event
-        if (comm_peer2peer_enabled(dir2, dim)) {
-          PROFILE(qudaStreamWaitEvent(device::get_default_stream(), halo.getIPCRemoteCopyEvent(dir2, dim), 0), profile,
-                  QUDA_PROFILE_STREAM_WAIT_EVENT);
-        } else {
-
-          if (scatterIndex == -1) scatterIndex = 2 * dim + dir;
+        if (scatterIndex == -1) scatterIndex = 2 * dim + dir;
 
-          if (!gdr_recv && !zero_copy_recv) { // Issue CPU->GPU copy if not GDR
-            // note the ColorSpinorField::scatter transforms from
-            // scatter centric to gather centric (e.g., flips
-            // direction) so here just use dir not dir2
-            PROFILE(if (dslash_copy) halo.scatter(2 * dim + dir, device::get_stream(scatterIndex)), profile,
-                    QUDA_PROFILE_SCATTER);
-          }
+        if (!gdr_recv && !zero_copy_recv) { // Issue CPU->GPU copy if not GDR
+          // note the ColorSpinorField::scatter transforms from
+          // scatter centric to gather centric (e.g., flips
+          // direction) so here just use dir not dir2
+          PROFILE(if (dslash_copy) halo.scatter(2 * dim + dir, device::get_stream(scatterIndex)), profile,
+                  QUDA_PROFILE_SCATTER);
         }
       }
-      return comms_test;
-    }
 
-    /**
-       @brief Ensure that the dslash is complete.  By construction, the
-       dslash will have completed (or is in flight) on this process,
-       however, we must also ensure that no local work begins until any
-       communication in flight from this process to another has
-       completed.  This prevents a race condition where we could start
-       updating the local buffers on a subsequent computation before we
-       have finished sending.
-    */
-    template <typename T> inline void completeDslash(const ColorSpinorField &halo, const T &dslashParam)
-    {
-      // this ensures that the p2p sending is completed before any
-      // subsequent work is done on the compute stream
-      for (int dim = 3; dim >= 0; dim--) {
-        if (!dslashParam.commDim[dim]) continue;
-        for (int dir = 0; dir < 2; dir++) {
-          if (comm_peer2peer_enabled(dir, dim)) {
-            PROFILE(qudaStreamWaitEvent(device::get_default_stream(), halo.getIPCCopyEvent(dir, dim), 0), profile,
-                    QUDA_PROFILE_STREAM_WAIT_EVENT);
-          }
+    }
+    return comms_test;
+  }
+
+  /**
+     @brief Ensure that the dslash is complete.  By construction, the
+     dslash will have completed (or is in flight) on this process,
+     however, we must also ensure that no local work begins until any
+     communication in flight from this process to another has
+     completed.  This prevents a race condition where we could start
+     updating the local buffers on a subsequent computation before we
+     have finished sending.
+  */
+  template <typename T> inline void completeDslash(const ColorSpinorField &halo, const T &dslashParam)
+  {
+    // this ensures that the p2p sending is completed before any
+    // subsequent work is done on the compute stream
+    for (int dim=3; dim>=0; dim--) {
+      if (!dslashParam.commDim[dim]) continue;
+      for (int dir=0; dir<2; dir++) {
+	if (comm_peer2peer_enabled(dir,dim)) {
+          PROFILE(qudaStreamWaitEvent(device::get_default_stream(), halo.getIPCCopyEvent(dir, dim), 0), profile,
+                  QUDA_PROFILE_STREAM_WAIT_EVENT);
         }
       }
     }
-
-    /**
-       @brief Set the ghosts to the mapped CPU ghost buffer, or unsets
-       if already set.  Note this must not be called until after the
-       interior dslash has been called, since it sets the peer-to-peer
-       ghost pointers, and this need to be done without the mapped ghost
-       enabled.
-
-       @param[in,out] dslash The dslash object
-       @param[in,out] in The ColorSpinorField source
-       @param[in] to_mapped Whether we are switching to mapped ghosts or not
-     */
-    template <typename Dslash> inline void setMappedGhost(Dslash &dslash, const ColorSpinorField &halo, bool to_mapped)
-    {
-      static char aux_copy[TuneKey::aux_n];
-      static bool set_mapped = false;
-
-      if (to_mapped) {
-        if (set_mapped) errorQuda("set_mapped already set");
-        // in the below we switch to the mapped ghost buffer and update the tuneKey to reflect this
-        halo.bufferIndex += 2;
-        strcpy(aux_copy, dslash.getAux(dslash.dslashParam.kernel_type));
-        if (comm_peer2peer_enabled_global())
-          dslash.augmentAux(dslash.dslashParam.kernel_type, ",zero_copy,p2p=1");
-        else
-          dslash.augmentAux(dslash.dslashParam.kernel_type, ",zero_copy,p2p=0");
-        set_mapped = true;
-      } else {
-        if (!set_mapped) errorQuda("set_mapped not set");
-        // reset to default
-        dslash.setAux(dslash.dslashParam.kernel_type, aux_copy);
-        halo.bufferIndex -= 2;
-        set_mapped = false;
-      }
+  }
+
+  /**
+     @brief Set the ghosts to the mapped CPU ghost buffer, or unsets
+     if already set.  Note this must not be called until after the
+     interior dslash has been called, since it sets the peer-to-peer
+     ghost pointers, and this need to be done without the mapped ghost
+     enabled.
+
+     @param[in,out] dslash The dslash object
+     @param[in,out] in The ColorSpinorField source
+     @param[in] to_mapped Whether we are switching to mapped ghosts or not
+   */
+  template <typename Dslash> inline void setMappedGhost(Dslash &dslash, const ColorSpinorField &halo, bool to_mapped)
+  {
+    static char aux_copy[TuneKey::aux_n];
+    static bool set_mapped = false;
+
+    if (to_mapped) {
+      if (set_mapped) errorQuda("set_mapped already set");
+      // in the below we switch to the mapped ghost buffer and update the tuneKey to reflect this
+      halo.bufferIndex += 2;
+      strcpy(aux_copy,dslash.getAux(dslash.dslashParam.kernel_type));
+      if (comm_peer2peer_enabled_global())
+        dslash.augmentAux(dslash.dslashParam.kernel_type, ",zero_copy,p2p=1");
+      else
+        dslash.augmentAux(dslash.dslashParam.kernel_type, ",zero_copy,p2p=0");
+      set_mapped = true;
+    } else {
+      if (!set_mapped) errorQuda("set_mapped not set");
+      // reset to default
+      dslash.setAux(dslash.dslashParam.kernel_type, aux_copy);
+      halo.bufferIndex -= 2;
+      set_mapped = false;
     }
+  }
 
-    template <typename Dslash> struct DslashPolicyImp {
-
-      virtual void operator()(Dslash &, cvector_ref<const ColorSpinorField> &, const ColorSpinorField &, TimeProfile &)
-      {
-      }
+  template <typename Dslash> struct DslashPolicyImp {
 
-      virtual ~DslashPolicyImp() = default;
-    };
+    virtual void operator()(Dslash &, cvector_ref<const ColorSpinorField> &, const ColorSpinorField &, TimeProfile &) { }
 
-    /**
-       Standard dslash parallelization with host staging for send and receive
-    */
-    template <typename Dslash> struct DslashBasic : DslashPolicyImp<Dslash> {
+    virtual ~DslashPolicyImp() = default;
+  };
 
-      void operator()(Dslash &dslash, cvector_ref<const ColorSpinorField> &in, const ColorSpinorField &halo,
-                      TimeProfile &profile)
-      {
-        PROFILE_START(profile, QUDA_PROFILE_TOTAL);
-        auto &dslashParam = dslash.dslashParam;
-        dslashParam.kernel_type = INTERIOR_KERNEL;
-        dslashParam.threads = halo.getDslashConstant().volume_4d_cb;
-        dslash.setShmem(0);
+  /**
+     Standard dslash parallelization with host staging for send and receive
+  */
+  template <typename Dslash> struct DslashBasic : DslashPolicyImp<Dslash> {
 
-        issueRecv(halo, dslash, false); // Prepost receives
+    void operator()(Dslash &dslash, cvector_ref<const ColorSpinorField> &in, const ColorSpinorField &halo,
+                    TimeProfile &profile)
+    {
+      PROFILE_START(profile, QUDA_PROFILE_TOTAL);
+      auto &dslashParam = dslash.dslashParam;
+      dslashParam.kernel_type = INTERIOR_KERNEL;
+      dslashParam.threads = halo.getDslashConstant().volume_4d_cb;
+      dslash.setShmem(0);
 
-        const int packIndex = device::get_default_stream_idx();
-        const int parity_src = (in.SiteSubset() == QUDA_PARITY_SITE_SUBSET ? 1 - dslashParam.parity : 0);
-        issuePack(halo, in, dslash, parity_src,
-                  static_cast<MemoryLocation>(Device | (Remote * dslashParam.remote_write)), packIndex);
+      issueRecv(halo, dslash, false); // Prepost receives
 
-        issueGather(halo, dslash);
+      const int packIndex = device::get_default_stream_idx();
+      const int parity_src = (in.SiteSubset() == QUDA_PARITY_SITE_SUBSET ? 1 - dslashParam.parity : 0);
+      issuePack(halo, in, dslash, parity_src, static_cast<MemoryLocation>(Device | (Remote * dslashParam.remote_write)),
+                packIndex);
 
-        PROFILE(if (dslash_interior_compute) dslash.apply(device::get_default_stream()), profile,
-                QUDA_PROFILE_DSLASH_KERNEL);
-        if (aux_worker) aux_worker->apply(device::get_default_stream());
+      issueGather(halo, dslash);
 
-        DslashCommsPattern pattern(dslashParam.commDim);
-        while (pattern.completeSum < pattern.commDimTotal) {
-          for (int i = 3; i >= 0; i--) {
-            if (!dslashParam.commDim[i]) continue;
+      PROFILE(if (dslash_interior_compute) dslash.apply(device::get_default_stream()), profile, QUDA_PROFILE_DSLASH_KERNEL);
+      if (aux_worker) aux_worker->apply(device::get_default_stream());
 
-            for (int dir = 1; dir >= 0; dir--) {
-              // Query if gather has completed
-              if (!pattern.gatherCompleted[2 * i + dir] && pattern.gatherCompleted[pattern.previousDir[2 * i + dir]]) {
-                bool event_test = comm_peer2peer_enabled(dir, i);
-                if (!event_test)
-                  PROFILE(event_test = qudaEventQuery(gatherEnd[2 * i + dir]), profile, QUDA_PROFILE_EVENT_QUERY);
-
-                if (event_test) {
-                  pattern.gatherCompleted[2 * i + dir] = 1;
-                  pattern.completeSum++;
-                  PROFILE(if (dslash_comms) halo.sendStart(
-                            2 * i + dir, device::get_stream(dslashParam.remote_write ? packIndex : 2 * i + dir), false,
-                            dslashParam.remote_write),
-                          profile, QUDA_PROFILE_COMMS_START);
-                }
+      DslashCommsPattern pattern(dslashParam.commDim);
+      while (pattern.completeSum < pattern.commDimTotal) {
+        for (int i = 3; i >= 0; i--) {
+          if (!dslashParam.commDim[i]) continue;
+
+          for (int dir = 1; dir >= 0; dir--) {
+            // Query if gather has completed
+            if (!pattern.gatherCompleted[2 * i + dir] && pattern.gatherCompleted[pattern.previousDir[2 * i + dir]]) {
+              bool event_test = comm_peer2peer_enabled(dir, i);
+              if (!event_test)
+                PROFILE(event_test = qudaEventQuery(gatherEnd[2 * i + dir]), profile, QUDA_PROFILE_EVENT_QUERY);
+
+              if (event_test) {
+                pattern.gatherCompleted[2 * i + dir] = 1;
+                pattern.completeSum++;
+                PROFILE(if (dslash_comms) halo.sendStart(
+                          2 * i + dir, device::get_stream(dslashParam.remote_write ? packIndex : 2 * i + dir), false,
+                          dslashParam.remote_write),
+                        profile, QUDA_PROFILE_COMMS_START);
               }
+            }
 
-              // Query if comms has finished
-              if (!pattern.commsCompleted[2 * i + dir] && pattern.gatherCompleted[2 * i + dir]) {
-                if (commsComplete(halo, dslash, i, dir, false, false, false)) {
-                  pattern.commsCompleted[2 * i + dir] = 1;
-                  pattern.completeSum++;
-                }
+            // Query if comms has finished
+            if (!pattern.commsCompleted[2 * i + dir] && pattern.gatherCompleted[2 * i + dir]) {
+              if (commsComplete(halo, dslash, i, dir, false, false, false)) {
+                pattern.commsCompleted[2 * i + dir] = 1;
+                pattern.completeSum++;
               }
+            }
 
-            } // dir=0,1
+          } // dir=0,1
 
-            if (!pattern.dslashCompleted[2 * i] && pattern.dslashCompleted[pattern.previousDir[2 * i + 1]]
-                && pattern.commsCompleted[2 * i] && pattern.commsCompleted[2 * i + 1]) {
+          if (!pattern.dslashCompleted[2 * i] && pattern.dslashCompleted[pattern.previousDir[2 * i + 1]]
+              && pattern.commsCompleted[2 * i] && pattern.commsCompleted[2 * i + 1]) {
 
-              for (int dir = 1; dir >= 0; dir--) {
-                if (!comm_peer2peer_enabled(
+            for (int dir = 1; dir >= 0; dir--) {
+              if (!comm_peer2peer_enabled(
                       1 - dir, i)) { // if not peer-to-peer we post an event in the scatter stream and wait on that
-                  // Record the end of the scattering
-                  PROFILE(qudaEventRecord(scatterEnd[2 * i + dir], device::get_stream(2 * i + dir)), profile,
-                          QUDA_PROFILE_EVENT_RECORD);
-                  // wait for scattering to finish and then launch dslash
-                  PROFILE(qudaStreamWaitEvent(device::get_default_stream(), scatterEnd[2 * i + dir], 0), profile,
-                          QUDA_PROFILE_STREAM_WAIT_EVENT);
-                }
+                // Record the end of the scattering
+                PROFILE(qudaEventRecord(scatterEnd[2 * i + dir], device::get_stream(2 * i + dir)), profile, QUDA_PROFILE_EVENT_RECORD);
+                // wait for scattering to finish and then launch dslash
+                PROFILE(qudaStreamWaitEvent(device::get_default_stream(), scatterEnd[2 * i + dir], 0), profile,
+                    QUDA_PROFILE_STREAM_WAIT_EVENT);
               }
+            }
 
-              dslashParam.kernel_type = static_cast<KernelType>(i);
-              dslashParam.threads = dslash.Nface() * halo.getDslashConstant().ghostFaceCB[i]; // updating 2 or 6 faces
+            dslashParam.kernel_type = static_cast<KernelType>(i);
+            dslashParam.threads = dslash.Nface() * halo.getDslashConstant().ghostFaceCB[i]; // updating 2 or 6 faces
 
-              // all faces use this stream
-              PROFILE(if (dslash_exterior_compute) dslash.apply(device::get_default_stream()), profile,
-                      QUDA_PROFILE_DSLASH_KERNEL);
+            // all faces use this stream
+            PROFILE(if (dslash_exterior_compute) dslash.apply(device::get_default_stream()), profile, QUDA_PROFILE_DSLASH_KERNEL);
 
-              pattern.dslashCompleted[2 * i] = 1;
-            }
+            pattern.dslashCompleted[2 * i] = 1;
           }
         }
-
-        completeDslash(halo, dslashParam);
-        halo.bufferIndex = (1 - halo.bufferIndex);
-        PROFILE_STOP(profile, QUDA_PROFILE_TOTAL);
       }
-    };
 
-    /**
-       Generic shmem dslash
-        // shmem bitfield encodes
-        // 0 - no shmem
-        // 1 - pack P2P (merged in interior)
-        // 2 - pack IB (merged in interior)
-        // 3 - pack P2P + IB (merged in interior)
-        // 8 - barrier part I (packing) (merged in interior, only useful if packing) -- currently required
-        // 16 - barrier part II (spin exterior) (merged in exterior) -- currently required
-        // 32 - use packstream -- not used
-        // 64 - use uber kernel (merge exterior)
-    */
-    template <typename Dslash, int shmem> struct DslashShmemGeneric : DslashPolicyImp<Dslash> {
+      completeDslash(halo, dslashParam);
+      halo.bufferIndex = (1 - halo.bufferIndex);
+      PROFILE_STOP(profile, QUDA_PROFILE_TOTAL);
+    }
+  };
+
+  /**
+     Generic shmem dslash
+      // shmem bitfield encodes
+      // 0 - no shmem
+      // 1 - pack P2P (merged in interior)
+      // 2 - pack IB (merged in interior)
+      // 3 - pack P2P + IB (merged in interior)
+      // 8 - barrier part I (packing) (merged in interior, only useful if packing) -- currently required
+      // 16 - barrier part II (spin exterior) (merged in exterior) -- currently required
+      // 32 - use packstream -- not used
+      // 64 - use uber kernel (merge exterior)
+  */
+  template <typename Dslash, int shmem> struct DslashShmemGeneric : DslashPolicyImp<Dslash> {
 
 #ifdef NVSHMEM_COMMS
-      void operator()(Dslash &dslash, cvector_ref<const ColorSpinorField> &in, const ColorSpinorField &halo,
-                      TimeProfile &profile)
-      {
-        PROFILE_START(profile, QUDA_PROFILE_TOTAL);
+    void operator()(Dslash &dslash, cvector_ref<const ColorSpinorField> &in, const ColorSpinorField &halo,
+                    TimeProfile &profile)
+    {
+      PROFILE_START(profile, QUDA_PROFILE_TOTAL);
 
-        auto &dslashParam = dslash.dslashParam;
-        setFusedParam(dslashParam, dslash, halo);
+      auto &dslashParam = dslash.dslashParam;
+      setFusedParam(dslashParam, dslash, halo);
 
-        DslashCommsPattern pattern(dslashParam.commDim);
-        dslashParam.kernel_type = (shmem & 64) ? UBER_KERNEL : INTERIOR_KERNEL;
-        dslashParam.threads = halo.getDslashConstant().volume_4d_cb;
-        dslash.setShmem(shmem);
-        dslashParam.setExteriorDims(shmem & 64);
+      DslashCommsPattern pattern(dslashParam.commDim);
+      dslashParam.kernel_type = (shmem & 64) ? UBER_KERNEL : INTERIOR_KERNEL;
+      dslashParam.threads = halo.getDslashConstant().volume_4d_cb;
+      dslash.setShmem(shmem);
+      dslashParam.setExteriorDims(shmem & 64);
 
-        // record start of the dslash
-        const int packIndex = device::get_default_stream_idx();
-        constexpr MemoryLocation location = static_cast<MemoryLocation>(Shmem);
+      // record start of the dslash
+      const int packIndex = device::get_default_stream_idx();
+      constexpr MemoryLocation location = static_cast<MemoryLocation>(Shmem);
 
-        if (!((shmem & 2) and (shmem & 1))) {
-          issuePack(halo, in, dslash, 1 - dslashParam.parity, location, packIndex, shmem);
-        }
+      if (!((shmem & 2) and (shmem & 1))) {
+        issuePack(halo, in, dslash, 1 - dslashParam.parity, location, packIndex, shmem);
+      }
 
-        dslash.setPack(((shmem & 2) or (shmem & 1)), location); // enable fused kernel packing
+      dslash.setPack(((shmem & 2) or (shmem & 1)), location); // enable fused kernel packing
 
-        PROFILE(if (dslash_interior_compute) dslash.apply(device::get_default_stream()), profile,
-                QUDA_PROFILE_DSLASH_KERNEL);
+      PROFILE(if (dslash_interior_compute) dslash.apply(device::get_default_stream()), profile, QUDA_PROFILE_DSLASH_KERNEL);
 
-        dslash.setPack(false, location); // disable fused kernel packing
-        if (aux_worker) aux_worker->apply(device::get_default_stream());
+      dslash.setPack(false, location); // disable fused kernel packing
+      if (aux_worker) aux_worker->apply(device::get_default_stream());
 
-        if (pattern.commDimTotal) {
-          setFusedParam(dslashParam, dslash, halo); // setup for fused exterior kernel
-          if (!(shmem & 64)) {
-            PROFILE(if (dslash_exterior_compute) dslash.apply(device::get_default_stream()), profile,
-                    QUDA_PROFILE_DSLASH_KERNEL);
-          }
+      if (pattern.commDimTotal) {
+        setFusedParam(dslashParam, dslash, halo); // setup for fused exterior kernel
+        if (!(shmem & 64)) {
+          PROFILE(if (dslash_exterior_compute) dslash.apply(device::get_default_stream()), profile, QUDA_PROFILE_DSLASH_KERNEL);
         }
-
-        dslash::inc_dslash_shmem_sync_counter();
-        halo.bufferIndex = (1 - halo.bufferIndex);
-        PROFILE_STOP(profile, QUDA_PROFILE_TOTAL);
       }
+
+      dslash::inc_dslash_shmem_sync_counter();
+      halo.bufferIndex = (1 - halo.bufferIndex);
+      PROFILE_STOP(profile, QUDA_PROFILE_TOTAL);
+    }
 #else
-      void operator()(Dslash &, cvector_ref<const ColorSpinorField> &, const ColorSpinorField &, TimeProfile &)
-      {
-        errorQuda("NVSHMEM Dslash policies not built");
-      }
+    void operator()(Dslash &, cvector_ref<const ColorSpinorField> &, const ColorSpinorField &, TimeProfile &)
+    {
+      errorQuda("NVSHMEM Dslash policies not built");
+    }
 #endif
-    };
-
-    template <typename Dslash> using DslashShmemUberPackIntra = DslashShmemGeneric<Dslash, 64 + 16 + 8 + 1>;
-    template <typename Dslash> using DslashShmemUberPackFull = DslashShmemGeneric<Dslash, 64 + 16 + 8 + 2 + 1>;
-    template <typename Dslash> using DslashShmemPackIntra = DslashShmemGeneric<Dslash, 16 + 8 + 1>;
-    template <typename Dslash> using DslashShmemPackFull = DslashShmemGeneric<Dslash, 16 + 8 + 2 + 1>;
+  };
 
-    /**
-     Standard dslash parallelization with host staging for send and receive, and fused halo update kernel
-   */
-    template <typename Dslash> struct DslashFusedExterior : DslashPolicyImp<Dslash> {
+  template <typename Dslash> using DslashShmemUberPackIntra = DslashShmemGeneric<Dslash, 64 + 16 + 8 + 1>;
+  template <typename Dslash> using DslashShmemUberPackFull = DslashShmemGeneric<Dslash, 64 + 16 + 8 + 2 + 1>;
+  template <typename Dslash> using DslashShmemPackIntra = DslashShmemGeneric<Dslash, 16 + 8 + 1>;
+  template <typename Dslash> using DslashShmemPackFull = DslashShmemGeneric<Dslash, 16 + 8 + 2 + 1>;
 
-      void operator()(Dslash &dslash, cvector_ref<const ColorSpinorField> &in, const ColorSpinorField &halo,
-                      TimeProfile &profile)
-      {
-        PROFILE_START(profile, QUDA_PROFILE_TOTAL);
+  /**
+   Standard dslash parallelization with host staging for send and receive, and fused halo update kernel
+ */
+  template <typename Dslash> struct DslashFusedExterior : DslashPolicyImp<Dslash> {
 
-        auto &dslashParam = dslash.dslashParam;
-        dslashParam.kernel_type = INTERIOR_KERNEL;
-        dslashParam.threads = halo.getDslashConstant().volume_4d_cb;
-        dslash.setShmem(0);
+    void operator()(Dslash &dslash, cvector_ref<const ColorSpinorField> &in, const ColorSpinorField &halo,
+                    TimeProfile &profile)
+    {
+      PROFILE_START(profile, QUDA_PROFILE_TOTAL);
 
-        issueRecv(halo, dslash, false); // Prepost receives
+      auto &dslashParam = dslash.dslashParam;
+      dslashParam.kernel_type = INTERIOR_KERNEL;
+      dslashParam.threads = halo.getDslashConstant().volume_4d_cb;
+      dslash.setShmem(0);
 
-        const int packIndex = device::get_default_stream_idx();
-        const int parity_src = (in.SiteSubset() == QUDA_PARITY_SITE_SUBSET ? 1 - dslashParam.parity : 0);
-        issuePack(halo, in, dslash, parity_src,
-                  static_cast<MemoryLocation>(Device | (Remote * dslashParam.remote_write)), packIndex);
+      issueRecv(halo, dslash, false); // Prepost receives
 
-        issueGather(halo, dslash);
+      const int packIndex = device::get_default_stream_idx();
+      const int parity_src = (in.SiteSubset() == QUDA_PARITY_SITE_SUBSET ? 1 - dslashParam.parity : 0);
+      issuePack(halo, in, dslash, parity_src, static_cast<MemoryLocation>(Device | (Remote * dslashParam.remote_write)),
+                packIndex);
 
-        PROFILE(if (dslash_interior_compute) dslash.apply(device::get_default_stream()), profile,
-                QUDA_PROFILE_DSLASH_KERNEL);
-        if (aux_worker) aux_worker->apply(device::get_default_stream());
+      issueGather(halo, dslash);
 
-        const int scatterIndex = getStreamIndex(dslashParam);
-        DslashCommsPattern pattern(dslashParam.commDim);
-        while (pattern.completeSum < pattern.commDimTotal) {
-          for (int i = 3; i >= 0; i--) {
-            if (!dslashParam.commDim[i]) continue;
+      PROFILE(if (dslash_interior_compute) dslash.apply(device::get_default_stream()), profile, QUDA_PROFILE_DSLASH_KERNEL);
+      if (aux_worker) aux_worker->apply(device::get_default_stream());
 
-            for (int dir = 1; dir >= 0; dir--) {
-              // Query if gather has completed
-              if (!pattern.gatherCompleted[2 * i + dir] && pattern.gatherCompleted[pattern.previousDir[2 * i + dir]]) {
-                bool event_test = comm_peer2peer_enabled(dir, i);
-                if (!event_test)
-                  PROFILE(event_test = qudaEventQuery(gatherEnd[2 * i + dir]), profile, QUDA_PROFILE_EVENT_QUERY);
-
-                if (event_test) {
-                  pattern.gatherCompleted[2 * i + dir] = 1;
-                  pattern.completeSum++;
-                  PROFILE(if (dslash_comms) halo.sendStart(
-                            2 * i + dir, device::get_stream(dslashParam.remote_write ? packIndex : 2 * i + dir), false,
-                            dslashParam.remote_write),
-                          profile, QUDA_PROFILE_COMMS_START);
-                }
+      const int scatterIndex = getStreamIndex(dslashParam);
+      DslashCommsPattern pattern(dslashParam.commDim);
+      while (pattern.completeSum < pattern.commDimTotal) {
+        for (int i = 3; i >= 0; i--) {
+          if (!dslashParam.commDim[i]) continue;
+
+          for (int dir = 1; dir >= 0; dir--) {
+            // Query if gather has completed
+            if (!pattern.gatherCompleted[2 * i + dir] && pattern.gatherCompleted[pattern.previousDir[2 * i + dir]]) {
+              bool event_test = comm_peer2peer_enabled(dir, i);
+              if (!event_test)
+                PROFILE(event_test = qudaEventQuery(gatherEnd[2 * i + dir]), profile, QUDA_PROFILE_EVENT_QUERY);
+
+              if (event_test) {
+                pattern.gatherCompleted[2 * i + dir] = 1;
+                pattern.completeSum++;
+                PROFILE(if (dslash_comms) halo.sendStart(
+                          2 * i + dir, device::get_stream(dslashParam.remote_write ? packIndex : 2 * i + dir), false,
+                          dslashParam.remote_write),
+                        profile, QUDA_PROFILE_COMMS_START);
               }
+            }
 
-              // Query if comms has finished
-              if (!pattern.commsCompleted[2 * i + dir] && pattern.gatherCompleted[2 * i + dir]) {
-                if (commsComplete(halo, dslash, i, dir, false, false, false, scatterIndex)) {
-                  pattern.commsCompleted[2 * i + dir] = 1;
-                  pattern.completeSum++;
-                }
+            // Query if comms has finished
+            if (!pattern.commsCompleted[2 * i + dir] && pattern.gatherCompleted[2 * i + dir]) {
+              if (commsComplete(halo, dslash, i, dir, false, false, false, scatterIndex)) {
+                pattern.commsCompleted[2 * i + dir] = 1;
+                pattern.completeSum++;
               }
-            } // dir=0,1
-          } // i
-        } // while(pattern.completeSum < commDimTotal)
+            }
+          } // dir=0,1
+        }   // i
+      }     // while(pattern.completeSum < commDimTotal)
 
-        for (int i = 3; i >= 0; i--) {
-          if (dslashParam.commDim[i]
-              && (!comm_peer2peer_enabled(0, i)
-                  || !comm_peer2peer_enabled(
+      for (int i = 3; i >= 0; i--) {
+        if (dslashParam.commDim[i]
+            && (!comm_peer2peer_enabled(0, i)
+                || !comm_peer2peer_enabled(
                     1, i))) { // if not peer-to-peer we post an event in the scatter stream and wait on that
-            PROFILE(qudaEventRecord(scatterEnd[0], device::get_stream(scatterIndex)), profile, QUDA_PROFILE_EVENT_RECORD);
-            PROFILE(qudaStreamWaitEvent(device::get_default_stream(), scatterEnd[0], 0), profile,
-                    QUDA_PROFILE_STREAM_WAIT_EVENT);
-            break;
-          }
-        }
-
-        // Launch exterior kernel
-        if (pattern.commDimTotal) {
-          setFusedParam(dslashParam, dslash, halo); // setup for exterior kernel
-          PROFILE(if (dslash_exterior_compute) dslash.apply(device::get_default_stream()), profile,
-                  QUDA_PROFILE_DSLASH_KERNEL);
+          PROFILE(qudaEventRecord(scatterEnd[0], device::get_stream(scatterIndex)), profile, QUDA_PROFILE_EVENT_RECORD);
+          PROFILE(qudaStreamWaitEvent(device::get_default_stream(), scatterEnd[0], 0), profile, QUDA_PROFILE_STREAM_WAIT_EVENT);
+          break;
         }
+      }
 
-        completeDslash(halo, dslashParam);
-        halo.bufferIndex = (1 - halo.bufferIndex);
-        PROFILE_STOP(profile, QUDA_PROFILE_TOTAL);
+      // Launch exterior kernel
+      if (pattern.commDimTotal) {
+        setFusedParam(dslashParam, dslash, halo); // setup for exterior kernel
+        PROFILE(if (dslash_exterior_compute) dslash.apply(device::get_default_stream()), profile, QUDA_PROFILE_DSLASH_KERNEL);
       }
-    };
 
-    /**
-       Dslash parallelization with GDR for send and receive
-     */
-    template <typename Dslash> struct DslashGDR : DslashPolicyImp<Dslash> {
+      completeDslash(halo, dslashParam);
+      halo.bufferIndex = (1 - halo.bufferIndex);
+      PROFILE_STOP(profile, QUDA_PROFILE_TOTAL);
+    }
+  };
 
-      void operator()(Dslash &dslash, cvector_ref<const ColorSpinorField> &in, const ColorSpinorField &halo,
-                      TimeProfile &profile)
-      {
-        PROFILE_START(profile, QUDA_PROFILE_TOTAL);
+/**
+   Dslash parallelization with GDR for send and receive
+ */
+  template <typename Dslash> struct DslashGDR : DslashPolicyImp<Dslash> {
 
-        auto &dslashParam = dslash.dslashParam;
-        dslashParam.kernel_type = INTERIOR_KERNEL;
-        dslashParam.threads = halo.getDslashConstant().volume_4d_cb;
-        dslash.setShmem(0);
+    void operator()(Dslash &dslash, cvector_ref<const ColorSpinorField> &in, const ColorSpinorField &halo,
+                    TimeProfile &profile)
+    {
+      PROFILE_START(profile, QUDA_PROFILE_TOTAL);
 
-        issueRecv(halo, dslash, true); // Prepost receives
+      auto &dslashParam = dslash.dslashParam;
+      dslashParam.kernel_type = INTERIOR_KERNEL;
+      dslashParam.threads = halo.getDslashConstant().volume_4d_cb;
+      dslash.setShmem(0);
 
-        const int packIndex = device::get_default_stream_idx();
-        const int parity_src = (in.SiteSubset() == QUDA_PARITY_SITE_SUBSET ? 1 - dslashParam.parity : 0);
-        issuePack(halo, in, dslash, parity_src,
-                  static_cast<MemoryLocation>(Device | (Remote * dslashParam.remote_write)), packIndex);
+      issueRecv(halo, dslash, true); // Prepost receives
 
-        PROFILE(if (dslash_interior_compute) dslash.apply(device::get_default_stream()), profile,
-                QUDA_PROFILE_DSLASH_KERNEL);
-        if (aux_worker) aux_worker->apply(device::get_default_stream());
+      const int packIndex = device::get_default_stream_idx();
+      const int parity_src = (in.SiteSubset() == QUDA_PARITY_SITE_SUBSET ? 1 - dslashParam.parity : 0);
+      issuePack(halo, in, dslash, parity_src, static_cast<MemoryLocation>(Device | (Remote * dslashParam.remote_write)),
+                packIndex);
 
-        bool pack_event = false;
-        for (int p2p = 0; p2p < 2; p2p++) { // schedule non-p2p traffic first, then do p2p
-          for (int i = 3; i >= 0; i--) {
-            if (!dslashParam.commDim[i]) continue;
+      PROFILE(if (dslash_interior_compute) dslash.apply(device::get_default_stream()), profile, QUDA_PROFILE_DSLASH_KERNEL);
+      if (aux_worker) aux_worker->apply(device::get_default_stream());
 
-            if (!pack_event) {
-              qudaEventSynchronize(packEnd[halo.bufferIndex]);
-              pack_event = true;
-            }
+      bool pack_event = false;
+      for (int p2p = 0; p2p < 2; p2p++) { // schedule non-p2p traffic first, then do p2p
+        for (int i = 3; i >= 0; i--) {
+          if (!dslashParam.commDim[i]) continue;
 
-            for (int dir = 1; dir >= 0; dir--) {
-              if ((comm_peer2peer_enabled(dir, i) + p2p) % 2 == 0) {
-                PROFILE(if (dslash_comms) halo.sendStart(
-                          2 * i + dir, device::get_stream(dslashParam.remote_write ? packIndex : 2 * i + dir), true,
-                          dslashParam.remote_write),
-                        profile, QUDA_PROFILE_COMMS_START);
-              } // is p2p?
-            } // dir
-          } // i
-        } // p2p
+          if (!pack_event) {
+            qudaEventSynchronize(packEnd[halo.bufferIndex]);
+            pack_event = true;
+          }
 
-        DslashCommsPattern pattern(dslashParam.commDim, true);
-        while (pattern.completeSum < pattern.commDimTotal) {
-          for (int i = 3; i >= 0; i--) {
-            if (!dslashParam.commDim[i]) continue;
+          for (int dir = 1; dir >= 0; dir--) {
+            if ((comm_peer2peer_enabled(dir, i) + p2p) % 2 == 0) {
+              PROFILE(if (dslash_comms) halo.sendStart(
+                        2 * i + dir, device::get_stream(dslashParam.remote_write ? packIndex : 2 * i + dir), true,
+                        dslashParam.remote_write),
+                      profile, QUDA_PROFILE_COMMS_START);
+            } // is p2p?
+          }   // dir
+        }     // i
+      }       // p2p
+
+      DslashCommsPattern pattern(dslashParam.commDim, true);
+      while (pattern.completeSum < pattern.commDimTotal) {
+        for (int i = 3; i >= 0; i--) {
+          if (!dslashParam.commDim[i]) continue;
 
-            for (int dir = 1; dir >= 0; dir--) {
+          for (int dir = 1; dir >= 0; dir--) {
 
-              // Query if comms has finished
-              if (!pattern.commsCompleted[2 * i + dir]) {
-                if (commsComplete(halo, dslash, i, dir, true, true, false)) {
-                  pattern.commsCompleted[2 * i + dir] = 1;
-                  pattern.completeSum++;
-                }
+            // Query if comms has finished
+            if (!pattern.commsCompleted[2 * i + dir]) {
+              if (commsComplete(halo, dslash, i, dir, true, true, false)) {
+                pattern.commsCompleted[2 * i + dir] = 1;
+                pattern.completeSum++;
               }
+            }
 
-            } // dir=0,1
+          } // dir=0,1
 
-            if (!pattern.dslashCompleted[2 * i] && pattern.dslashCompleted[pattern.previousDir[2 * i + 1]]
-                && pattern.commsCompleted[2 * i] && pattern.commsCompleted[2 * i + 1]) {
-              dslashParam.kernel_type = static_cast<KernelType>(i);
-              dslashParam.threads = dslash.Nface() * halo.getDslashConstant().ghostFaceCB[i]; // updating 2 or 6 faces
+          if (!pattern.dslashCompleted[2 * i] && pattern.dslashCompleted[pattern.previousDir[2 * i + 1]]
+              && pattern.commsCompleted[2 * i] && pattern.commsCompleted[2 * i + 1]) {
+            dslashParam.kernel_type = static_cast<KernelType>(i);
+            dslashParam.threads = dslash.Nface() * halo.getDslashConstant().ghostFaceCB[i]; // updating 2 or 6 faces
 
-              // all faces use this stream
-              PROFILE(if (dslash_exterior_compute) dslash.apply(device::get_default_stream()), profile,
-                      QUDA_PROFILE_DSLASH_KERNEL);
+            // all faces use this stream
+            PROFILE(if (dslash_exterior_compute) dslash.apply(device::get_default_stream()), profile, QUDA_PROFILE_DSLASH_KERNEL);
 
-              pattern.dslashCompleted[2 * i] = 1;
-            }
+            pattern.dslashCompleted[2 * i] = 1;
           }
         }
-
-        completeDslash(halo, dslashParam);
-        halo.bufferIndex = (1 - halo.bufferIndex);
-        PROFILE_STOP(profile, QUDA_PROFILE_TOTAL);
       }
-    };
 
-    /**
-       Dslash parallelization with GDR for send and receive with fused halo update kernel
-     */
-    template <typename Dslash> struct DslashFusedGDR : DslashPolicyImp<Dslash> {
+      completeDslash(halo, dslashParam);
+      halo.bufferIndex = (1 - halo.bufferIndex);
+      PROFILE_STOP(profile, QUDA_PROFILE_TOTAL);
+    }
+  };
 
-      void operator()(Dslash &dslash, cvector_ref<const ColorSpinorField> &in, const ColorSpinorField &halo,
-                      TimeProfile &profile)
-      {
-        PROFILE_START(profile, QUDA_PROFILE_TOTAL);
+/**
+   Dslash parallelization with GDR for send and receive with fused halo update kernel
+ */
+  template <typename Dslash> struct DslashFusedGDR : DslashPolicyImp<Dslash> {
 
-        auto &dslashParam = dslash.dslashParam;
-        dslashParam.kernel_type = INTERIOR_KERNEL;
-        dslashParam.threads = halo.getDslashConstant().volume_4d_cb;
-        dslash.setShmem(0);
+    void operator()(Dslash &dslash, cvector_ref<const ColorSpinorField> &in, const ColorSpinorField &halo,
+                    TimeProfile &profile)
+    {
+      PROFILE_START(profile, QUDA_PROFILE_TOTAL);
 
-        issueRecv(halo, dslash, true); // Prepost receives
+      auto &dslashParam = dslash.dslashParam;
+      dslashParam.kernel_type = INTERIOR_KERNEL;
+      dslashParam.threads = halo.getDslashConstant().volume_4d_cb;
+      dslash.setShmem(0);
 
-        const int packIndex = device::get_default_stream_idx();
-        const int parity_src = (in.SiteSubset() == QUDA_PARITY_SITE_SUBSET ? 1 - dslashParam.parity : 0);
-        issuePack(halo, in, dslash, parity_src,
-                  static_cast<MemoryLocation>(Device | (Remote * dslashParam.remote_write)), packIndex);
+      issueRecv(halo, dslash, true); // Prepost receives
 
-        PROFILE(if (dslash_interior_compute) dslash.apply(device::get_default_stream()), profile,
-                QUDA_PROFILE_DSLASH_KERNEL);
-        if (aux_worker) aux_worker->apply(device::get_default_stream());
+      const int packIndex = device::get_default_stream_idx();
+      const int parity_src = (in.SiteSubset() == QUDA_PARITY_SITE_SUBSET ? 1 - dslashParam.parity : 0);
+      issuePack(halo, in, dslash, parity_src, static_cast<MemoryLocation>(Device | (Remote * dslashParam.remote_write)),
+                packIndex);
 
-        bool pack_event = false;
-        for (int p2p = 0; p2p < 2; p2p++) { // schedule non-p2p traffic first, then do p2p
-          for (int i = 3; i >= 0; i--) {
-            if (!dslashParam.commDim[i]) continue;
+      PROFILE(if (dslash_interior_compute) dslash.apply(device::get_default_stream()), profile, QUDA_PROFILE_DSLASH_KERNEL);
+      if (aux_worker) aux_worker->apply(device::get_default_stream());
 
-            if (!pack_event) {
-              qudaEventSynchronize(packEnd[halo.bufferIndex]);
-              pack_event = true;
-            }
+      bool pack_event = false;
+      for (int p2p = 0; p2p < 2; p2p++) { // schedule non-p2p traffic first, then do p2p
+        for (int i = 3; i >= 0; i--) {
+          if (!dslashParam.commDim[i]) continue;
 
-            for (int dir = 1; dir >= 0; dir--) {
-              if ((comm_peer2peer_enabled(dir, i) + p2p) % 2 == 0) {
-                PROFILE(if (dslash_comms) halo.sendStart(
-                          2 * i + dir, device::get_stream(dslashParam.remote_write ? packIndex : 2 * i + dir), true,
-                          dslashParam.remote_write),
-                        profile, QUDA_PROFILE_COMMS_START);
-              } // is p2p?
-            }
+          if (!pack_event) {
+            qudaEventSynchronize(packEnd[halo.bufferIndex]);
+            pack_event = true;
           }
-        } // p2p
 
-        DslashCommsPattern pattern(dslashParam.commDim, true);
-        while (pattern.completeSum < pattern.commDimTotal) {
-          for (int i = 3; i >= 0; i--) {
-            if (!dslashParam.commDim[i]) continue;
+          for (int dir = 1; dir >= 0; dir--) {
+            if ((comm_peer2peer_enabled(dir, i) + p2p) % 2 == 0) {
+              PROFILE(if (dslash_comms) halo.sendStart(
+                        2 * i + dir, device::get_stream(dslashParam.remote_write ? packIndex : 2 * i + dir), true,
+                        dslashParam.remote_write),
+                      profile, QUDA_PROFILE_COMMS_START);
+            } // is p2p?
+          }
+        }
+      } // p2p
 
-            for (int dir = 1; dir >= 0; dir--) {
+      DslashCommsPattern pattern(dslashParam.commDim, true);
+      while (pattern.completeSum < pattern.commDimTotal) {
+        for (int i = 3; i >= 0; i--) {
+          if (!dslashParam.commDim[i]) continue;
 
-              // Query if comms has finished
-              if (!pattern.commsCompleted[2 * i + dir]) {
-                if (commsComplete(halo, dslash, i, dir, true, true, false)) {
-                  pattern.commsCompleted[2 * i + dir] = 1;
-                  pattern.completeSum++;
-                }
-              }
-            } // dir=0,1
-          } // i
-        } // pattern.completeSum < pattern.CommDimTotal
-
-        // Launch exterior kernel
-        if (pattern.commDimTotal) {
-          setFusedParam(dslashParam, dslash, halo); // setup for fused exterior kernel
-          PROFILE(if (dslash_exterior_compute) dslash.apply(device::get_default_stream()), profile,
-                  QUDA_PROFILE_DSLASH_KERNEL);
-        }
+          for (int dir = 1; dir >= 0; dir--) {
 
-        completeDslash(halo, dslashParam);
-        halo.bufferIndex = (1 - halo.bufferIndex);
-        PROFILE_STOP(profile, QUDA_PROFILE_TOTAL);
+            // Query if comms has finished
+            if (!pattern.commsCompleted[2 * i + dir]) {
+              if (commsComplete(halo, dslash, i, dir, true, true, false)) {
+                pattern.commsCompleted[2 * i + dir] = 1;
+                pattern.completeSum++;
+              }
+            }
+          } // dir=0,1
+        }   // i
+      }     // pattern.completeSum < pattern.CommDimTotal
+
+      // Launch exterior kernel
+      if (pattern.commDimTotal) {
+        setFusedParam(dslashParam, dslash, halo); // setup for fused exterior kernel
+        PROFILE(if (dslash_exterior_compute) dslash.apply(device::get_default_stream()), profile, QUDA_PROFILE_DSLASH_KERNEL);
       }
-    };
 
-    /**
-       Dslash parallelization with host staging for send and GDR for receive
-     */
-    template <typename Dslash> struct DslashGDRRecv : DslashPolicyImp<Dslash> {
+      completeDslash(halo, dslashParam);
+      halo.bufferIndex = (1 - halo.bufferIndex);
+      PROFILE_STOP(profile, QUDA_PROFILE_TOTAL);
+    }
+  };
 
-      void operator()(Dslash &dslash, cvector_ref<const ColorSpinorField> &in, const ColorSpinorField &halo,
-                      TimeProfile &profile)
-      {
-        PROFILE_START(profile, QUDA_PROFILE_TOTAL);
+/**
+   Dslash parallelization with host staging for send and GDR for receive
+ */
+  template <typename Dslash> struct DslashGDRRecv : DslashPolicyImp<Dslash> {
 
-        auto &dslashParam = dslash.dslashParam;
-        dslashParam.kernel_type = INTERIOR_KERNEL;
-        dslashParam.threads = halo.getDslashConstant().volume_4d_cb;
-        dslash.setShmem(0);
+    void operator()(Dslash &dslash, cvector_ref<const ColorSpinorField> &in, const ColorSpinorField &halo,
+                    TimeProfile &profile)
+    {
+      PROFILE_START(profile, QUDA_PROFILE_TOTAL);
 
-        issueRecv(halo, dslash, true); // Prepost receives
+      auto &dslashParam = dslash.dslashParam;
+      dslashParam.kernel_type = INTERIOR_KERNEL;
+      dslashParam.threads = halo.getDslashConstant().volume_4d_cb;
+      dslash.setShmem(0);
 
-        const int packIndex = device::get_default_stream_idx();
-        const int parity_src = (in.SiteSubset() == QUDA_PARITY_SITE_SUBSET ? 1 - dslashParam.parity : 0);
-        issuePack(halo, in, dslash, parity_src,
-                  static_cast<MemoryLocation>(Device | (Remote * dslashParam.remote_write)), packIndex);
+      issueRecv(halo, dslash, true); // Prepost receives
 
-        issueGather(halo, dslash);
+      const int packIndex = device::get_default_stream_idx();
+      const int parity_src = (in.SiteSubset() == QUDA_PARITY_SITE_SUBSET ? 1 - dslashParam.parity : 0);
+      issuePack(halo, in, dslash, parity_src, static_cast<MemoryLocation>(Device | (Remote * dslashParam.remote_write)),
+                packIndex);
 
-        PROFILE(if (dslash_interior_compute) dslash.apply(device::get_default_stream()), profile,
-                QUDA_PROFILE_DSLASH_KERNEL);
-        if (aux_worker) aux_worker->apply(device::get_default_stream());
+      issueGather(halo, dslash);
 
-        DslashCommsPattern pattern(dslashParam.commDim);
-        while (pattern.completeSum < pattern.commDimTotal) {
-          for (int i = 3; i >= 0; i--) {
-            if (!dslashParam.commDim[i]) continue;
+      PROFILE(if (dslash_interior_compute) dslash.apply(device::get_default_stream()), profile, QUDA_PROFILE_DSLASH_KERNEL);
+      if (aux_worker) aux_worker->apply(device::get_default_stream());
 
-            for (int dir = 1; dir >= 0; dir--) {
-              // Query if gather has completed
-              if (!pattern.gatherCompleted[2 * i + dir] && pattern.gatherCompleted[pattern.previousDir[2 * i + dir]]) {
-                bool event_test = comm_peer2peer_enabled(dir, i);
-                if (!event_test)
-                  PROFILE(event_test = qudaEventQuery(gatherEnd[2 * i + dir]), profile, QUDA_PROFILE_EVENT_QUERY);
-
-                if (event_test) {
-                  pattern.gatherCompleted[2 * i + dir] = 1;
-                  pattern.completeSum++;
-                  PROFILE(if (dslash_comms) halo.sendStart(
-                            2 * i + dir, device::get_stream(dslashParam.remote_write ? packIndex : 2 * i + dir), false,
-                            dslashParam.remote_write),
-                          profile, QUDA_PROFILE_COMMS_START);
-                }
+      DslashCommsPattern pattern(dslashParam.commDim);
+      while (pattern.completeSum < pattern.commDimTotal) {
+        for (int i = 3; i >= 0; i--) {
+          if (!dslashParam.commDim[i]) continue;
+
+          for (int dir = 1; dir >= 0; dir--) {
+            // Query if gather has completed
+            if (!pattern.gatherCompleted[2 * i + dir] && pattern.gatherCompleted[pattern.previousDir[2 * i + dir]]) {
+              bool event_test = comm_peer2peer_enabled(dir, i);
+              if (!event_test)
+                PROFILE(event_test = qudaEventQuery(gatherEnd[2 * i + dir]), profile, QUDA_PROFILE_EVENT_QUERY);
+
+              if (event_test) {
+                pattern.gatherCompleted[2 * i + dir] = 1;
+                pattern.completeSum++;
+                PROFILE(if (dslash_comms) halo.sendStart(
+                          2 * i + dir, device::get_stream(dslashParam.remote_write ? packIndex : 2 * i + dir), false,
+                          dslashParam.remote_write),
+                        profile, QUDA_PROFILE_COMMS_START);
               }
+            }
 
-              // Query if comms has finished
-              if (!pattern.commsCompleted[2 * i + dir] && pattern.gatherCompleted[2 * i + dir]) {
-                if (commsComplete(halo, dslash, i, dir, false, true, false)) {
-                  pattern.commsCompleted[2 * i + dir] = 1;
-                  pattern.completeSum++;
-                }
+            // Query if comms has finished
+            if (!pattern.commsCompleted[2 * i + dir] && pattern.gatherCompleted[2 * i + dir]) {
+              if (commsComplete(halo, dslash, i, dir, false, true, false)) {
+                pattern.commsCompleted[2 * i + dir] = 1;
+                pattern.completeSum++;
               }
+            }
 
-            } // dir=0,1
+          } // dir=0,1
 
-            if (!pattern.dslashCompleted[2 * i] && pattern.dslashCompleted[pattern.previousDir[2 * i + 1]]
-                && pattern.commsCompleted[2 * i] && pattern.commsCompleted[2 * i + 1]) {
-              dslashParam.kernel_type = static_cast<KernelType>(i);
-              dslashParam.threads = dslash.Nface() * halo.getDslashConstant().ghostFaceCB[i]; // updating 2 or 6 faces
+          if (!pattern.dslashCompleted[2 * i] && pattern.dslashCompleted[pattern.previousDir[2 * i + 1]]
+              && pattern.commsCompleted[2 * i] && pattern.commsCompleted[2 * i + 1]) {
+            dslashParam.kernel_type = static_cast<KernelType>(i);
+            dslashParam.threads = dslash.Nface() * halo.getDslashConstant().ghostFaceCB[i]; // updating 2 or 6 faces
 
-              // all faces use this stream
-              PROFILE(if (dslash_exterior_compute) dslash.apply(device::get_default_stream()), profile,
-                      QUDA_PROFILE_DSLASH_KERNEL);
+            // all faces use this stream
+            PROFILE(if (dslash_exterior_compute) dslash.apply(device::get_default_stream()), profile, QUDA_PROFILE_DSLASH_KERNEL);
 
-              pattern.dslashCompleted[2 * i] = 1;
-            }
+            pattern.dslashCompleted[2 * i] = 1;
           }
         }
-
-        completeDslash(halo, dslashParam);
-        halo.bufferIndex = (1 - halo.bufferIndex);
-        PROFILE_STOP(profile, QUDA_PROFILE_TOTAL);
       }
-    };
 
-    /**
-       Dslash parallelization with host staging for send and GDR for receive, with fused halo update kernel
-     */
-    template <typename Dslash> struct DslashFusedGDRRecv : DslashPolicyImp<Dslash> {
+      completeDslash(halo, dslashParam);
+      halo.bufferIndex = (1 - halo.bufferIndex);
+      PROFILE_STOP(profile, QUDA_PROFILE_TOTAL);
+    }
+  };
 
-      void operator()(Dslash &dslash, cvector_ref<const ColorSpinorField> &in, const ColorSpinorField &halo,
-                      TimeProfile &profile)
-      {
-        PROFILE_START(profile, QUDA_PROFILE_TOTAL);
+/**
+   Dslash parallelization with host staging for send and GDR for receive, with fused halo update kernel
+ */
+  template <typename Dslash> struct DslashFusedGDRRecv : DslashPolicyImp<Dslash> {
 
-        auto &dslashParam = dslash.dslashParam;
-        dslashParam.kernel_type = INTERIOR_KERNEL;
-        dslashParam.threads = halo.getDslashConstant().volume_4d_cb;
-        dslash.setShmem(0);
+    void operator()(Dslash &dslash, cvector_ref<const ColorSpinorField> &in, const ColorSpinorField &halo,
+                    TimeProfile &profile)
+    {
+      PROFILE_START(profile, QUDA_PROFILE_TOTAL);
 
-        issueRecv(halo, dslash, true); // Prepost receives
+      auto &dslashParam = dslash.dslashParam;
+      dslashParam.kernel_type = INTERIOR_KERNEL;
+      dslashParam.threads = halo.getDslashConstant().volume_4d_cb;
+      dslash.setShmem(0);
 
-        const int packIndex = device::get_default_stream_idx();
-        const int parity_src = (in.SiteSubset() == QUDA_PARITY_SITE_SUBSET ? 1 - dslashParam.parity : 0);
-        issuePack(halo, in, dslash, parity_src,
-                  static_cast<MemoryLocation>(Device | (Remote * dslashParam.remote_write)), packIndex);
+      issueRecv(halo, dslash, true); // Prepost receives
 
-        issueGather(halo, dslash);
+      const int packIndex = device::get_default_stream_idx();
+      const int parity_src = (in.SiteSubset() == QUDA_PARITY_SITE_SUBSET ? 1 - dslashParam.parity : 0);
+      issuePack(halo, in, dslash, parity_src, static_cast<MemoryLocation>(Device | (Remote * dslashParam.remote_write)),
+                packIndex);
 
-        PROFILE(if (dslash_interior_compute) dslash.apply(device::get_default_stream()), profile,
-                QUDA_PROFILE_DSLASH_KERNEL);
-        if (aux_worker) aux_worker->apply(device::get_default_stream());
+      issueGather(halo, dslash);
 
-        DslashCommsPattern pattern(dslashParam.commDim);
-        while (pattern.completeSum < pattern.commDimTotal) {
-          for (int i = 3; i >= 0; i--) {
-            if (!dslashParam.commDim[i]) continue;
+      PROFILE(if (dslash_interior_compute) dslash.apply(device::get_default_stream()), profile, QUDA_PROFILE_DSLASH_KERNEL);
+      if (aux_worker) aux_worker->apply(device::get_default_stream());
 
-            for (int dir = 1; dir >= 0; dir--) {
-              // Query if gather has completed
-              if (!pattern.gatherCompleted[2 * i + dir] && pattern.gatherCompleted[pattern.previousDir[2 * i + dir]]) {
-                bool event_test = comm_peer2peer_enabled(dir, i);
-                if (!event_test)
-                  PROFILE(event_test = qudaEventQuery(gatherEnd[2 * i + dir]), profile, QUDA_PROFILE_EVENT_QUERY);
-
-                if (event_test) {
-                  pattern.gatherCompleted[2 * i + dir] = 1;
-                  pattern.completeSum++;
-                  PROFILE(if (dslash_comms) halo.sendStart(2 * i + dir,
-                                                           dslashParam.remote_write ? device::get_default_stream() :
-                                                                                      device::get_stream(2 * i + dir),
-                                                           false, dslashParam.remote_write),
-                          profile, QUDA_PROFILE_COMMS_START);
-                }
+      DslashCommsPattern pattern(dslashParam.commDim);
+      while (pattern.completeSum < pattern.commDimTotal) {
+        for (int i = 3; i >= 0; i--) {
+          if (!dslashParam.commDim[i]) continue;
+
+          for (int dir = 1; dir >= 0; dir--) {
+            // Query if gather has completed
+            if (!pattern.gatherCompleted[2 * i + dir] && pattern.gatherCompleted[pattern.previousDir[2 * i + dir]]) {
+              bool event_test = comm_peer2peer_enabled(dir, i);
+              if (!event_test)
+                PROFILE(event_test = qudaEventQuery(gatherEnd[2 * i + dir]), profile, QUDA_PROFILE_EVENT_QUERY);
+
+              if (event_test) {
+                pattern.gatherCompleted[2 * i + dir] = 1;
+                pattern.completeSum++;
+                PROFILE(if (dslash_comms) halo.sendStart(2 * i + dir,
+                                                         dslashParam.remote_write ? device::get_default_stream() :
+                                                                                    device::get_stream(2 * i + dir),
+                                                         false, dslashParam.remote_write),
+                        profile, QUDA_PROFILE_COMMS_START);
               }
+            }
 
-              // Query if comms has finished
-              if (!pattern.commsCompleted[2 * i + dir] && pattern.gatherCompleted[2 * i + dir]) {
-                if (commsComplete(halo, dslash, i, dir, false, true, false)) {
-                  pattern.commsCompleted[2 * i + dir] = 1;
-                  pattern.completeSum++;
-                }
+            // Query if comms has finished
+            if (!pattern.commsCompleted[2 * i + dir] && pattern.gatherCompleted[2 * i + dir]) {
+              if (commsComplete(halo, dslash, i, dir, false, true, false)) {
+                pattern.commsCompleted[2 * i + dir] = 1;
+                pattern.completeSum++;
               }
-            } // dir=0,1
-          } // i
-        } // while(pattern.completeSum < commDimTotal)
-
-        // Launch exterior kernel
-        if (pattern.commDimTotal) {
-          setFusedParam(dslashParam, dslash, halo); // setup for fused exterior kernel
-          PROFILE(if (dslash_exterior_compute) dslash.apply(device::get_default_stream()), profile,
-                  QUDA_PROFILE_DSLASH_KERNEL);
-        }
-
-        completeDslash(halo, dslashParam);
-        halo.bufferIndex = (1 - halo.bufferIndex);
-        PROFILE_STOP(profile, QUDA_PROFILE_TOTAL);
+            }
+          } // dir=0,1
+        }   // i
+      }     // while(pattern.completeSum < commDimTotal)
+
+      // Launch exterior kernel
+      if (pattern.commDimTotal) {
+        setFusedParam(dslashParam, dslash, halo); // setup for fused exterior kernel
+        PROFILE(if (dslash_exterior_compute) dslash.apply(device::get_default_stream()), profile, QUDA_PROFILE_DSLASH_KERNEL);
       }
-    };
 
-    /**
-       Variation of multi-gpu dslash where the packing kernel writes
-       buffers directly to host memory
-    */
-    template <typename Dslash> struct DslashZeroCopyPack : DslashPolicyImp<Dslash> {
+      completeDslash(halo, dslashParam);
+      halo.bufferIndex = (1 - halo.bufferIndex);
+      PROFILE_STOP(profile, QUDA_PROFILE_TOTAL);
+    }
+  };
 
-      void operator()(Dslash &dslash, cvector_ref<const ColorSpinorField> &in, const ColorSpinorField &halo,
-                      TimeProfile &profile)
-      {
-        PROFILE_START(profile, QUDA_PROFILE_TOTAL);
+  /**
+     Variation of multi-gpu dslash where the packing kernel writes
+     buffers directly to host memory
+  */
+  template <typename Dslash> struct DslashZeroCopyPack : DslashPolicyImp<Dslash> {
 
-        auto &dslashParam = dslash.dslashParam;
-        dslashParam.kernel_type = INTERIOR_KERNEL;
-        dslashParam.threads = halo.getDslashConstant().volume_4d_cb;
-        dslash.setShmem(0);
+    void operator()(Dslash &dslash, cvector_ref<const ColorSpinorField> &in, const ColorSpinorField &halo,
+                    TimeProfile &profile)
+    {
+      PROFILE_START(profile, QUDA_PROFILE_TOTAL);
 
-        // record start of the dslash
-        PROFILE(qudaEventRecord(dslashStart[halo.bufferIndex], device::get_default_stream()), profile,
-                QUDA_PROFILE_EVENT_RECORD);
+      auto &dslashParam = dslash.dslashParam;
+      dslashParam.kernel_type = INTERIOR_KERNEL;
+      dslashParam.threads = halo.getDslashConstant().volume_4d_cb;
+      dslash.setShmem(0);
 
-        issueRecv(halo, dslash, false); // Prepost receives
+      // record start of the dslash
+      PROFILE(qudaEventRecord(dslashStart[halo.bufferIndex], device::get_default_stream()), profile,
+              QUDA_PROFILE_EVENT_RECORD);
 
-        const int packIndex = getStreamIndex(dslashParam);
-        PROFILE(qudaStreamWaitEvent(device::get_stream(packIndex), dslashStart[halo.bufferIndex], 0), profile,
-                QUDA_PROFILE_STREAM_WAIT_EVENT);
-        const int parity_src = (in.SiteSubset() == QUDA_PARITY_SITE_SUBSET ? 1 - dslashParam.parity : 0);
-        issuePack(halo, in, dslash, parity_src, static_cast<MemoryLocation>(Host | (Remote * dslashParam.remote_write)),
-                  packIndex);
-
-        PROFILE(if (dslash_interior_compute) dslash.apply(device::get_default_stream()), profile,
-                QUDA_PROFILE_DSLASH_KERNEL);
-        if (aux_worker) aux_worker->apply(device::get_default_stream());
-
-        for (int i = 3; i >= 0; i--) { // only synchronize if we need to
-          if (!dslashParam.remote_write
-              || (dslashParam.commDim[i] && (!comm_peer2peer_enabled(0, i) || !comm_peer2peer_enabled(1, i)))) {
-            qudaStreamSynchronize(device::get_stream(packIndex));
-            break;
-          }
-        }
+      issueRecv(halo, dslash, false); // Prepost receives
 
-        for (int p2p = 0; p2p < 2; p2p++) { // schedule non-p2p traffic first, then do p2p
-          for (int i = 3; i >= 0; i--) {
-            if (!dslashParam.commDim[i]) continue;
+      const int packIndex = getStreamIndex(dslashParam);
+      PROFILE(qudaStreamWaitEvent(device::get_stream(packIndex), dslashStart[halo.bufferIndex], 0), profile,
+              QUDA_PROFILE_STREAM_WAIT_EVENT);
+      const int parity_src = (in.SiteSubset() == QUDA_PARITY_SITE_SUBSET ? 1 - dslashParam.parity : 0);
+      issuePack(halo, in, dslash, parity_src, static_cast<MemoryLocation>(Host | (Remote * dslashParam.remote_write)),
+                packIndex);
 
-            for (int dir = 1; dir >= 0; dir--) {
-              if ((comm_peer2peer_enabled(dir, i) + p2p) % 2 == 0) {
-                PROFILE(if (dslash_comms) halo.sendStart(
-                          2 * i + dir, device::get_stream(dslashParam.remote_write ? packIndex : 2 * i + dir), false,
-                          dslashParam.remote_write),
-                        profile, QUDA_PROFILE_COMMS_START);
-              } // is p2p?
-            } // dir
-          } // i
-        } // p2p
+      PROFILE(if (dslash_interior_compute) dslash.apply(device::get_default_stream()), profile, QUDA_PROFILE_DSLASH_KERNEL);
+      if (aux_worker) aux_worker->apply(device::get_default_stream());
+
+      for (int i = 3; i >= 0; i--) { // only synchronize if we need to
+        if (!dslashParam.remote_write
+            || (dslashParam.commDim[i] && (!comm_peer2peer_enabled(0, i) || !comm_peer2peer_enabled(1, i)))) {
+          qudaStreamSynchronize(device::get_stream(packIndex));
+          break;
+        }
+      }
 
-        DslashCommsPattern pattern(dslashParam.commDim, true);
-        while (pattern.completeSum < pattern.commDimTotal) {
+      for (int p2p = 0; p2p < 2; p2p++) { // schedule non-p2p traffic first, then do p2p
+        for (int i = 3; i >= 0; i--) {
+          if (!dslashParam.commDim[i]) continue;
+
+          for (int dir = 1; dir >= 0; dir--) {
+            if ((comm_peer2peer_enabled(dir, i) + p2p) % 2 == 0) {
+              PROFILE(if (dslash_comms) halo.sendStart(
+                        2 * i + dir, device::get_stream(dslashParam.remote_write ? packIndex : 2 * i + dir), false,
+                        dslashParam.remote_write),
+                      profile, QUDA_PROFILE_COMMS_START);
+            } // is p2p?
+          }   // dir
+        }     // i
+      }       // p2p
+
+      DslashCommsPattern pattern(dslashParam.commDim, true);
+      while (pattern.completeSum < pattern.commDimTotal) {
 
-          for (int i = 3; i >= 0; i--) {
-            if (!dslashParam.commDim[i]) continue;
+        for (int i = 3; i >= 0; i--) {
+          if (!dslashParam.commDim[i]) continue;
 
-            for (int dir = 1; dir >= 0; dir--) {
+          for (int dir = 1; dir >= 0; dir--) {
 
-              // Query if comms have finished
-              if (!pattern.commsCompleted[2 * i + dir]) {
-                if (commsComplete(halo, dslash, i, dir, false, false, false)) {
-                  pattern.commsCompleted[2 * i + dir] = 1;
-                  pattern.completeSum++;
-                }
+            // Query if comms have finished
+            if (!pattern.commsCompleted[2 * i + dir]) {
+              if (commsComplete(halo, dslash, i, dir, false, false, false)) {
+                pattern.commsCompleted[2 * i + dir] = 1;
+                pattern.completeSum++;
               }
             }
+          }
 
-            if (!pattern.dslashCompleted[2 * i] && pattern.dslashCompleted[pattern.previousDir[2 * i + 1]]
-                && pattern.commsCompleted[2 * i] && pattern.commsCompleted[2 * i + 1]) {
-              for (int dir = 1; dir >= 0; dir--) {
-                if (!comm_peer2peer_enabled(
+          if (!pattern.dslashCompleted[2 * i] && pattern.dslashCompleted[pattern.previousDir[2 * i + 1]]
+              && pattern.commsCompleted[2 * i] && pattern.commsCompleted[2 * i + 1]) {
+            for (int dir = 1; dir >= 0; dir--) {
+              if (!comm_peer2peer_enabled(
                       1 - dir, i)) { // if not peer-to-peer we post an event in the scatter stream and wait on that
-                  // Record the end of the scattering
-                  PROFILE(qudaEventRecord(scatterEnd[2 * i + dir], device::get_stream(2 * i + dir)), profile,
-                          QUDA_PROFILE_EVENT_RECORD);
-                  // wait for scattering to finish and then launch dslash
-                  PROFILE(qudaStreamWaitEvent(device::get_default_stream(), scatterEnd[2 * i + dir], 0), profile,
-                          QUDA_PROFILE_STREAM_WAIT_EVENT);
-                }
+                // Record the end of the scattering
+                PROFILE(
+                        qudaEventRecord(scatterEnd[2 * i + dir], device::get_stream(2 * i + dir)), profile, QUDA_PROFILE_EVENT_RECORD);
+                // wait for scattering to finish and then launch dslash
+                PROFILE(qudaStreamWaitEvent(device::get_default_stream(), scatterEnd[2 * i + dir], 0), profile,
+                    QUDA_PROFILE_STREAM_WAIT_EVENT);
               }
+            }
 
-              dslashParam.kernel_type = static_cast<KernelType>(i);
-              dslashParam.threads = dslash.Nface() * halo.getDslashConstant().ghostFaceCB[i]; // updating 2 or 6 faces
+            dslashParam.kernel_type = static_cast<KernelType>(i);
+            dslashParam.threads = dslash.Nface() * halo.getDslashConstant().ghostFaceCB[i]; // updating 2 or 6 faces
 
-              // all faces use this stream
-              PROFILE(if (dslash_exterior_compute) dslash.apply(device::get_default_stream()), profile,
-                      QUDA_PROFILE_DSLASH_KERNEL);
+            // all faces use this stream
+            PROFILE(if (dslash_exterior_compute) dslash.apply(device::get_default_stream()), profile, QUDA_PROFILE_DSLASH_KERNEL);
 
-              pattern.dslashCompleted[2 * i] = 1;
-            }
+            pattern.dslashCompleted[2 * i] = 1;
           }
         }
-
-        completeDslash(halo, dslashParam);
-        halo.bufferIndex = (1 - halo.bufferIndex);
-        PROFILE_STOP(profile, QUDA_PROFILE_TOTAL);
       }
-    };
-
-    /**
-       Variation of multi-gpu dslash where the packing kernel writes
-       buffers directly to host memory with fused halo update kernel
-    */
-    template <typename Dslash> struct DslashFusedZeroCopyPack : DslashPolicyImp<Dslash> {
 
-      void operator()(Dslash &dslash, cvector_ref<const ColorSpinorField> &in, const ColorSpinorField &halo,
-                      TimeProfile &profile)
-      {
-        PROFILE_START(profile, QUDA_PROFILE_TOTAL);
+      completeDslash(halo, dslashParam);
+      halo.bufferIndex = (1 - halo.bufferIndex);
+      PROFILE_STOP(profile, QUDA_PROFILE_TOTAL);
+    }
+  };
 
-        auto &dslashParam = dslash.dslashParam;
-        dslashParam.kernel_type = INTERIOR_KERNEL;
-        dslashParam.threads = halo.getDslashConstant().volume_4d_cb;
-        dslash.setShmem(0);
+/**
+   Variation of multi-gpu dslash where the packing kernel writes
+   buffers directly to host memory with fused halo update kernel
+*/
+  template <typename Dslash> struct DslashFusedZeroCopyPack : DslashPolicyImp<Dslash> {
 
-        // record start of the dslash
-        PROFILE(qudaEventRecord(dslashStart[halo.bufferIndex], device::get_default_stream()), profile,
-                QUDA_PROFILE_EVENT_RECORD);
+    void operator()(Dslash &dslash, cvector_ref<const ColorSpinorField> &in, const ColorSpinorField &halo,
+                    TimeProfile &profile)
+    {
+      PROFILE_START(profile, QUDA_PROFILE_TOTAL);
 
-        const int packScatterIndex = getStreamIndex(dslashParam);
-        PROFILE(qudaStreamWaitEvent(device::get_stream(packScatterIndex), dslashStart[halo.bufferIndex], 0), profile,
-                QUDA_PROFILE_STREAM_WAIT_EVENT);
-        const int parity_src = (in.SiteSubset() == QUDA_PARITY_SITE_SUBSET ? 1 - dslashParam.parity : 0);
-        issuePack(halo, in, dslash, parity_src, static_cast<MemoryLocation>(Host | (Remote * dslashParam.remote_write)),
-                  packScatterIndex);
+      auto &dslashParam = dslash.dslashParam;
+      dslashParam.kernel_type = INTERIOR_KERNEL;
+      dslashParam.threads = halo.getDslashConstant().volume_4d_cb;
+      dslash.setShmem(0);
 
-        issueRecv(halo, dslash, false); // Prepost receives
+      // record start of the dslash
+      PROFILE(qudaEventRecord(dslashStart[halo.bufferIndex], device::get_default_stream()), profile,
+              QUDA_PROFILE_EVENT_RECORD);
 
-        PROFILE(if (dslash_interior_compute) dslash.apply(device::get_default_stream()), profile,
-                QUDA_PROFILE_DSLASH_KERNEL);
-        if (aux_worker) aux_worker->apply(device::get_default_stream());
+      const int packScatterIndex = getStreamIndex(dslashParam);
+      PROFILE(qudaStreamWaitEvent(device::get_stream(packScatterIndex), dslashStart[halo.bufferIndex], 0), profile,
+              QUDA_PROFILE_STREAM_WAIT_EVENT);
+      const int parity_src = (in.SiteSubset() == QUDA_PARITY_SITE_SUBSET ? 1 - dslashParam.parity : 0);
+      issuePack(halo, in, dslash, parity_src, static_cast<MemoryLocation>(Host | (Remote * dslashParam.remote_write)),
+                packScatterIndex);
 
-        for (int i = 3; i >= 0; i--) { // only synchronize if we need to
-          if (!dslashParam.remote_write
-              || (dslashParam.commDim[i] && (!comm_peer2peer_enabled(0, i) || !comm_peer2peer_enabled(1, i)))) {
-            qudaStreamSynchronize(device::get_stream(packScatterIndex));
-            break;
-          }
-        }
+      issueRecv(halo, dslash, false); // Prepost receives
 
-        for (int p2p = 0; p2p < 2; p2p++) { // schedule non-p2p traffic first, then do p2p
-          for (int i = 3; i >= 0; i--) {
-            if (!dslashParam.commDim[i]) continue;
+      PROFILE(if (dslash_interior_compute) dslash.apply(device::get_default_stream()), profile, QUDA_PROFILE_DSLASH_KERNEL);
+      if (aux_worker) aux_worker->apply(device::get_default_stream());
 
-            for (int dir = 1; dir >= 0; dir--) {
-              if ((comm_peer2peer_enabled(dir, i) + p2p) % 2 == 0) {
-                PROFILE(if (dslash_comms) halo.sendStart(
-                          2 * i + dir, device::get_stream(dslashParam.remote_write ? packScatterIndex : 2 * i + dir),
-                          false, dslashParam.remote_write),
-                        profile, QUDA_PROFILE_COMMS_START);
-              } // is p2p?
-            } // dir
-          } // i
-        } // p2p
+      for (int i = 3; i >= 0; i--) { // only synchronize if we need to
+        if (!dslashParam.remote_write
+            || (dslashParam.commDim[i] && (!comm_peer2peer_enabled(0, i) || !comm_peer2peer_enabled(1, i)))) {
+          qudaStreamSynchronize(device::get_stream(packScatterIndex));
+          break;
+        }
+      }
 
-        DslashCommsPattern pattern(dslashParam.commDim, true);
-        while (pattern.completeSum < pattern.commDimTotal) {
+      for (int p2p = 0; p2p < 2; p2p++) { // schedule non-p2p traffic first, then do p2p
+        for (int i = 3; i >= 0; i--) {
+          if (!dslashParam.commDim[i]) continue;
+
+          for (int dir = 1; dir >= 0; dir--) {
+            if ((comm_peer2peer_enabled(dir, i) + p2p) % 2 == 0) {
+              PROFILE(if (dslash_comms) halo.sendStart(
+                        2 * i + dir, device::get_stream(dslashParam.remote_write ? packScatterIndex : 2 * i + dir),
+                        false, dslashParam.remote_write),
+                      profile, QUDA_PROFILE_COMMS_START);
+            } // is p2p?
+          }   // dir
+        }     // i
+      }       // p2p
+
+      DslashCommsPattern pattern(dslashParam.commDim, true);
+      while (pattern.completeSum < pattern.commDimTotal) {
 
-          for (int i = 3; i >= 0; i--) {
-            if (!dslashParam.commDim[i]) continue;
+        for (int i = 3; i >= 0; i--) {
+          if (!dslashParam.commDim[i]) continue;
 
-            for (int dir = 1; dir >= 0; dir--) {
+          for (int dir = 1; dir >= 0; dir--) {
 
-              // Query if comms has finished
-              if (!pattern.commsCompleted[2 * i + dir]) {
-                if (commsComplete(halo, dslash, i, dir, false, false, false, packScatterIndex)) {
-                  pattern.commsCompleted[2 * i + dir] = 1;
-                  pattern.completeSum++;
-                }
+            // Query if comms has finished
+            if (!pattern.commsCompleted[2 * i + dir]) {
+              if (commsComplete(halo, dslash, i, dir, false, false, false, packScatterIndex)) {
+                pattern.commsCompleted[2 * i + dir] = 1;
+                pattern.completeSum++;
               }
+            }
 
-            } // dir=0,1
-          } // i
-        } // pattern.completeSum
-
-        for (int i = 3; i >= 0; i--) {
-          if (dslashParam.commDim[i] && (!comm_peer2peer_enabled(0, i) || !comm_peer2peer_enabled(1, i))) {
-            // if not peer-to-peer we post an event in the scatter stream and wait on that
-            PROFILE(qudaEventRecord(scatterEnd[0], device::get_stream(packScatterIndex)), profile,
-                    QUDA_PROFILE_EVENT_RECORD);
-            PROFILE(qudaStreamWaitEvent(device::get_default_stream(), scatterEnd[0], 0), profile,
-                    QUDA_PROFILE_STREAM_WAIT_EVENT);
-            break;
-          }
-        }
+          } // dir=0,1
+        }   // i
+      }     // pattern.completeSum
 
-        // Launch exterior kernel
-        if (pattern.commDimTotal) {
-          setFusedParam(dslashParam, dslash, halo); // setup for fused exterior kernel
-          PROFILE(if (dslash_exterior_compute) dslash.apply(device::get_default_stream()), profile,
-                  QUDA_PROFILE_DSLASH_KERNEL);
+      for (int i = 3; i >= 0; i--) {
+        if (dslashParam.commDim[i] && (!comm_peer2peer_enabled(0, i) || !comm_peer2peer_enabled(1, i))) {
+          // if not peer-to-peer we post an event in the scatter stream and wait on that
+          PROFILE(qudaEventRecord(scatterEnd[0], device::get_stream(packScatterIndex)), profile, QUDA_PROFILE_EVENT_RECORD);
+          PROFILE(qudaStreamWaitEvent(device::get_default_stream(), scatterEnd[0], 0), profile, QUDA_PROFILE_STREAM_WAIT_EVENT);
+          break;
         }
+      }
 
-        completeDslash(halo, dslashParam);
-        halo.bufferIndex = (1 - halo.bufferIndex);
-        PROFILE_STOP(profile, QUDA_PROFILE_TOTAL);
+      // Launch exterior kernel
+      if (pattern.commDimTotal) {
+        setFusedParam(dslashParam, dslash, halo); // setup for fused exterior kernel
+        PROFILE(if (dslash_exterior_compute) dslash.apply(device::get_default_stream()), profile, QUDA_PROFILE_DSLASH_KERNEL);
       }
-    };
 
-    /**
-       Multi-GPU Dslash zero-copy for the send and GDR for the receive
-     */
-    template <typename Dslash> struct DslashZeroCopyPackGDRRecv : DslashPolicyImp<Dslash> {
+      completeDslash(halo, dslashParam);
+      halo.bufferIndex = (1 - halo.bufferIndex);
+      PROFILE_STOP(profile, QUDA_PROFILE_TOTAL);
+    }
+  };
 
-      void operator()(Dslash &dslash, cvector_ref<const ColorSpinorField> &in, const ColorSpinorField &halo,
-                      TimeProfile &profile)
-      {
-        PROFILE_START(profile, QUDA_PROFILE_TOTAL);
+/**
+   Multi-GPU Dslash zero-copy for the send and GDR for the receive
+ */
+  template <typename Dslash> struct DslashZeroCopyPackGDRRecv : DslashPolicyImp<Dslash> {
 
-        auto &dslashParam = dslash.dslashParam;
-        dslashParam.kernel_type = INTERIOR_KERNEL;
-        dslashParam.threads = halo.getDslashConstant().volume_4d_cb;
-        dslash.setShmem(0);
+    void operator()(Dslash &dslash, cvector_ref<const ColorSpinorField> &in, const ColorSpinorField &halo,
+                    TimeProfile &profile)
+    {
+      PROFILE_START(profile, QUDA_PROFILE_TOTAL);
 
-        // record start of the dslash
-        PROFILE(qudaEventRecord(dslashStart[halo.bufferIndex], device::get_default_stream()), profile,
-                QUDA_PROFILE_EVENT_RECORD);
+      auto &dslashParam = dslash.dslashParam;
+      dslashParam.kernel_type = INTERIOR_KERNEL;
+      dslashParam.threads = halo.getDslashConstant().volume_4d_cb;
+      dslash.setShmem(0);
 
-        issueRecv(halo, dslash, true); // Prepost receives
+      // record start of the dslash
+      PROFILE(qudaEventRecord(dslashStart[halo.bufferIndex], device::get_default_stream()), profile,
+              QUDA_PROFILE_EVENT_RECORD);
 
-        const int packIndex = getStreamIndex(dslashParam);
-        PROFILE(qudaStreamWaitEvent(device::get_stream(packIndex), dslashStart[halo.bufferIndex], 0), profile,
-                QUDA_PROFILE_STREAM_WAIT_EVENT);
-        const int parity_src = (in.SiteSubset() == QUDA_PARITY_SITE_SUBSET ? 1 - dslashParam.parity : 0);
-        issuePack(halo, in, dslash, parity_src, static_cast<MemoryLocation>(Host | (Remote * dslashParam.remote_write)),
-                  packIndex);
-
-        PROFILE(if (dslash_interior_compute) dslash.apply(device::get_default_stream()), profile,
-                QUDA_PROFILE_DSLASH_KERNEL);
-        if (aux_worker) aux_worker->apply(device::get_default_stream());
-
-        for (int i = 3; i >= 0; i--) { // only synchronize if we need to
-          if (!dslashParam.remote_write
-              || (dslashParam.commDim[i] && (!comm_peer2peer_enabled(0, i) || !comm_peer2peer_enabled(1, i)))) {
-            qudaStreamSynchronize(device::get_stream(packIndex));
-            break;
-          }
-        }
+      issueRecv(halo, dslash, true); // Prepost receives
 
-        for (int p2p = 0; p2p < 2; p2p++) { // schedule non-p2p traffic first, then do p2p
-          for (int i = 3; i >= 0; i--) {
-            if (!dslashParam.commDim[i]) continue;
+      const int packIndex = getStreamIndex(dslashParam);
+      PROFILE(qudaStreamWaitEvent(device::get_stream(packIndex), dslashStart[halo.bufferIndex], 0), profile,
+              QUDA_PROFILE_STREAM_WAIT_EVENT);
+      const int parity_src = (in.SiteSubset() == QUDA_PARITY_SITE_SUBSET ? 1 - dslashParam.parity : 0);
+      issuePack(halo, in, dslash, parity_src, static_cast<MemoryLocation>(Host | (Remote * dslashParam.remote_write)),
+                packIndex);
 
-            for (int dir = 1; dir >= 0; dir--) {
-              if ((comm_peer2peer_enabled(dir, i) + p2p) % 2 == 0) {
-                PROFILE(if (dslash_comms) halo.sendStart(
-                          2 * i + dir, device::get_stream(dslashParam.remote_write ? packIndex : 2 * i + dir), false,
-                          dslashParam.remote_write),
-                        profile, QUDA_PROFILE_COMMS_START);
-              } // is p2p?
-            } // dir
-          } // i
-        } // p2p
+      PROFILE(if (dslash_interior_compute) dslash.apply(device::get_default_stream()), profile, QUDA_PROFILE_DSLASH_KERNEL);
+      if (aux_worker) aux_worker->apply(device::get_default_stream());
 
-        DslashCommsPattern pattern(dslashParam.commDim, true);
-        while (pattern.completeSum < pattern.commDimTotal) {
+      for (int i = 3; i >= 0; i--) { // only synchronize if we need to
+        if (!dslashParam.remote_write
+            || (dslashParam.commDim[i] && (!comm_peer2peer_enabled(0, i) || !comm_peer2peer_enabled(1, i)))) {
+          qudaStreamSynchronize(device::get_stream(packIndex));
+          break;
+        }
+      }
 
-          for (int i = 3; i >= 0; i--) {
-            if (!dslashParam.commDim[i]) continue;
+      for (int p2p = 0; p2p < 2; p2p++) { // schedule non-p2p traffic first, then do p2p
+        for (int i = 3; i >= 0; i--) {
+          if (!dslashParam.commDim[i]) continue;
+
+          for (int dir = 1; dir >= 0; dir--) {
+            if ((comm_peer2peer_enabled(dir, i) + p2p) % 2 == 0) {
+              PROFILE(if (dslash_comms) halo.sendStart(
+                        2 * i + dir, device::get_stream(dslashParam.remote_write ? packIndex : 2 * i + dir), false,
+                        dslashParam.remote_write),
+                      profile, QUDA_PROFILE_COMMS_START);
+            } // is p2p?
+          }   // dir
+        }     // i
+      }       // p2p
+
+      DslashCommsPattern pattern(dslashParam.commDim, true);
+      while (pattern.completeSum < pattern.commDimTotal) {
 
-            for (int dir = 1; dir >= 0; dir--) {
+        for (int i = 3; i >= 0; i--) {
+          if (!dslashParam.commDim[i]) continue;
 
-              // Query if comms has finished
-              if (!pattern.commsCompleted[2 * i + dir] && pattern.gatherCompleted[2 * i + dir]) {
-                if (commsComplete(halo, dslash, i, dir, false, true, false)) {
-                  pattern.commsCompleted[2 * i + dir] = 1;
-                  pattern.completeSum++;
-                }
+          for (int dir = 1; dir >= 0; dir--) {
+
+            // Query if comms has finished
+            if (!pattern.commsCompleted[2 * i + dir] && pattern.gatherCompleted[2 * i + dir]) {
+              if (commsComplete(halo, dslash, i, dir, false, true, false)) {
+                pattern.commsCompleted[2 * i + dir] = 1;
+                pattern.completeSum++;
               }
+            }
 
-            } // dir=0,1
+          } // dir=0,1
 
-            if (!pattern.dslashCompleted[2 * i] && pattern.dslashCompleted[pattern.previousDir[2 * i + 1]]
-                && pattern.commsCompleted[2 * i] && pattern.commsCompleted[2 * i + 1]) {
-              dslashParam.kernel_type = static_cast<KernelType>(i);
-              dslashParam.threads = dslash.Nface() * halo.getDslashConstant().ghostFaceCB[i]; // updating 2 or 6 faces
+          if (!pattern.dslashCompleted[2 * i] && pattern.dslashCompleted[pattern.previousDir[2 * i + 1]]
+              && pattern.commsCompleted[2 * i] && pattern.commsCompleted[2 * i + 1]) {
+            dslashParam.kernel_type = static_cast<KernelType>(i);
+            dslashParam.threads = dslash.Nface() * halo.getDslashConstant().ghostFaceCB[i]; // updating 2 or 6 faces
 
-              // all faces use this stream
-              PROFILE(if (dslash_exterior_compute) dslash.apply(device::get_default_stream()), profile,
-                      QUDA_PROFILE_DSLASH_KERNEL);
+            // all faces use this stream
+            PROFILE(if (dslash_exterior_compute) dslash.apply(device::get_default_stream()), profile, QUDA_PROFILE_DSLASH_KERNEL);
 
-              pattern.dslashCompleted[2 * i] = 1;
-            }
+            pattern.dslashCompleted[2 * i] = 1;
           }
         }
-
-        completeDslash(halo, dslashParam);
-        halo.bufferIndex = (1 - halo.bufferIndex);
-        PROFILE_STOP(profile, QUDA_PROFILE_TOTAL);
       }
-    };
 
-    /**
-       Multi-GPU Dslash zero-copy for the send and GDR for the receive,
-       with fused halo update kernel
-     */
-    template <typename Dslash> struct DslashFusedZeroCopyPackGDRRecv : DslashPolicyImp<Dslash> {
-
-      void operator()(Dslash &dslash, cvector_ref<const ColorSpinorField> &in, const ColorSpinorField &halo,
-                      TimeProfile &profile)
-      {
-        PROFILE_START(profile, QUDA_PROFILE_TOTAL);
+      completeDslash(halo, dslashParam);
+      halo.bufferIndex = (1 - halo.bufferIndex);
+      PROFILE_STOP(profile, QUDA_PROFILE_TOTAL);
+    }
+  };
 
-        auto &dslashParam = dslash.dslashParam;
-        dslashParam.kernel_type = INTERIOR_KERNEL;
-        dslashParam.threads = halo.getDslashConstant().volume_4d_cb;
-        dslash.setShmem(0);
+/**
+   Multi-GPU Dslash zero-copy for the send and GDR for the receive,
+   with fused halo update kernel
+ */
+  template <typename Dslash> struct DslashFusedZeroCopyPackGDRRecv : DslashPolicyImp<Dslash> {
 
-        // record start of the dslash
-        PROFILE(qudaEventRecord(dslashStart[halo.bufferIndex], device::get_default_stream()), profile,
-                QUDA_PROFILE_EVENT_RECORD);
+    void operator()(Dslash &dslash, cvector_ref<const ColorSpinorField> &in, const ColorSpinorField &halo,
+                    TimeProfile &profile)
+    {
+      PROFILE_START(profile, QUDA_PROFILE_TOTAL);
 
-        const int packIndex = getStreamIndex(dslashParam);
-        PROFILE(qudaStreamWaitEvent(device::get_stream(packIndex), dslashStart[halo.bufferIndex], 0), profile,
-                QUDA_PROFILE_STREAM_WAIT_EVENT);
-        const int parity_src = (in.SiteSubset() == QUDA_PARITY_SITE_SUBSET ? 1 - dslashParam.parity : 0);
-        issuePack(halo, in, dslash, parity_src, static_cast<MemoryLocation>(Host | (Remote * dslashParam.remote_write)),
-                  packIndex);
+      auto &dslashParam = dslash.dslashParam;
+      dslashParam.kernel_type = INTERIOR_KERNEL;
+      dslashParam.threads = halo.getDslashConstant().volume_4d_cb;
+      dslash.setShmem(0);
 
-        issueRecv(halo, dslash, true); // Prepost receives
+      // record start of the dslash
+      PROFILE(qudaEventRecord(dslashStart[halo.bufferIndex], device::get_default_stream()), profile,
+              QUDA_PROFILE_EVENT_RECORD);
 
-        PROFILE(if (dslash_interior_compute) dslash.apply(device::get_default_stream()), profile,
-                QUDA_PROFILE_DSLASH_KERNEL);
-        if (aux_worker) aux_worker->apply(device::get_default_stream());
+      const int packIndex = getStreamIndex(dslashParam);
+      PROFILE(qudaStreamWaitEvent(device::get_stream(packIndex), dslashStart[halo.bufferIndex], 0), profile,
+              QUDA_PROFILE_STREAM_WAIT_EVENT);
+      const int parity_src = (in.SiteSubset() == QUDA_PARITY_SITE_SUBSET ? 1 - dslashParam.parity : 0);
+      issuePack(halo, in, dslash, parity_src, static_cast<MemoryLocation>(Host | (Remote * dslashParam.remote_write)),
+                packIndex);
 
-        for (int i = 3; i >= 0; i--) { // only synchronize if we need to
-          if (!dslashParam.remote_write
-              || (dslashParam.commDim[i] && (!comm_peer2peer_enabled(0, i) || !comm_peer2peer_enabled(1, i)))) {
-            qudaStreamSynchronize(device::get_stream(packIndex));
-            break;
-          }
-        }
+      issueRecv(halo, dslash, true); // Prepost receives
 
-        for (int p2p = 0; p2p < 2; p2p++) { // schedule non-p2p traffic first, then do p2p
-          for (int i = 3; i >= 0; i--) {
-            if (!dslashParam.commDim[i]) continue;
+      PROFILE(if (dslash_interior_compute) dslash.apply(device::get_default_stream()), profile, QUDA_PROFILE_DSLASH_KERNEL);
+      if (aux_worker) aux_worker->apply(device::get_default_stream());
 
-            for (int dir = 1; dir >= 0; dir--) {
-              if ((comm_peer2peer_enabled(dir, i) + p2p) % 2 == 0) {
-                PROFILE(if (dslash_comms) halo.sendStart(
-                          2 * i + dir, device::get_stream(dslashParam.remote_write ? packIndex : 2 * i + dir), false,
-                          dslashParam.remote_write),
-                        profile, QUDA_PROFILE_COMMS_START);
-              } // is p2p?
-            } // dir
-          } // i
-        } // p2p
+      for (int i = 3; i >= 0; i--) { // only synchronize if we need to
+        if (!dslashParam.remote_write
+            || (dslashParam.commDim[i] && (!comm_peer2peer_enabled(0, i) || !comm_peer2peer_enabled(1, i)))) {
+          qudaStreamSynchronize(device::get_stream(packIndex));
+          break;
+        }
+      }
 
-        DslashCommsPattern pattern(dslashParam.commDim, true);
-        while (pattern.completeSum < pattern.commDimTotal) {
+      for (int p2p = 0; p2p < 2; p2p++) { // schedule non-p2p traffic first, then do p2p
+        for (int i = 3; i >= 0; i--) {
+          if (!dslashParam.commDim[i]) continue;
+
+          for (int dir = 1; dir >= 0; dir--) {
+            if ((comm_peer2peer_enabled(dir, i) + p2p) % 2 == 0) {
+              PROFILE(if (dslash_comms) halo.sendStart(
+                        2 * i + dir, device::get_stream(dslashParam.remote_write ? packIndex : 2 * i + dir), false,
+                        dslashParam.remote_write),
+                      profile, QUDA_PROFILE_COMMS_START);
+            } // is p2p?
+          }   // dir
+        }     // i
+      }       // p2p
+
+      DslashCommsPattern pattern(dslashParam.commDim, true);
+      while (pattern.completeSum < pattern.commDimTotal) {
 
-          for (int i = 3; i >= 0; i--) {
-            if (!dslashParam.commDim[i]) continue;
+        for (int i = 3; i >= 0; i--) {
+          if (!dslashParam.commDim[i]) continue;
 
-            for (int dir = 1; dir >= 0; dir--) {
+          for (int dir = 1; dir >= 0; dir--) {
 
-              // Query if comms has finished
-              if (!pattern.commsCompleted[2 * i + dir] && pattern.gatherCompleted[2 * i + dir]) {
-                if (commsComplete(halo, dslash, i, dir, false, true, false)) {
-                  pattern.commsCompleted[2 * i + dir] = 1;
-                  pattern.completeSum++;
-                }
+            // Query if comms has finished
+            if (!pattern.commsCompleted[2 * i + dir] && pattern.gatherCompleted[2 * i + dir]) {
+              if (commsComplete(halo, dslash, i, dir, false, true, false)) {
+                pattern.commsCompleted[2 * i + dir] = 1;
+                pattern.completeSum++;
               }
-            } // dir=0,1
-          } // i
-        } // while(pattern.completeSum < commDimTotal)
-
-        // Launch exterior kernel
-        if (pattern.commDimTotal) {
-          setFusedParam(dslashParam, dslash, halo); // setup for fused exterior kernel
-          PROFILE(if (dslash_exterior_compute) dslash.apply(device::get_default_stream()), profile,
-                  QUDA_PROFILE_DSLASH_KERNEL);
-        }
-
-        completeDslash(halo, dslashParam);
-        halo.bufferIndex = (1 - halo.bufferIndex);
-        PROFILE_STOP(profile, QUDA_PROFILE_TOTAL);
+            }
+          } // dir=0,1
+        }   // i
+      }     // while(pattern.completeSum < commDimTotal)
+
+      // Launch exterior kernel
+      if (pattern.commDimTotal) {
+        setFusedParam(dslashParam, dslash, halo); // setup for fused exterior kernel
+        PROFILE(if (dslash_exterior_compute) dslash.apply(device::get_default_stream()), profile, QUDA_PROFILE_DSLASH_KERNEL);
       }
-    };
 
-    /**
-       Variation of multi-gpu dslash where the packing kernel writes
-       buffers directly to host memory
-    */
-    template <typename Dslash> struct DslashZeroCopy : DslashPolicyImp<Dslash> {
+      completeDslash(halo, dslashParam);
+      halo.bufferIndex = (1 - halo.bufferIndex);
+      PROFILE_STOP(profile, QUDA_PROFILE_TOTAL);
+    }
+  };
 
-      void operator()(Dslash &dslash, cvector_ref<const ColorSpinorField> &in, const ColorSpinorField &halo,
-                      TimeProfile &profile)
-      {
-        PROFILE_START(profile, QUDA_PROFILE_TOTAL);
+/**
+   Variation of multi-gpu dslash where the packing kernel writes
+   buffers directly to host memory
+*/
+  template <typename Dslash> struct DslashZeroCopy : DslashPolicyImp<Dslash> {
 
-        auto &dslashParam = dslash.dslashParam;
-        dslashParam.kernel_type = INTERIOR_KERNEL;
-        dslashParam.threads = halo.getDslashConstant().volume_4d_cb;
-        dslash.setShmem(0);
+    void operator()(Dslash &dslash, cvector_ref<const ColorSpinorField> &in, const ColorSpinorField &halo,
+                    TimeProfile &profile)
+    {
+      PROFILE_START(profile, QUDA_PROFILE_TOTAL);
 
-        // record start of the dslash
-        PROFILE(qudaEventRecord(dslashStart[halo.bufferIndex], device::get_default_stream()), profile,
-                QUDA_PROFILE_EVENT_RECORD);
+      auto &dslashParam = dslash.dslashParam;
+      dslashParam.kernel_type = INTERIOR_KERNEL;
+      dslashParam.threads = halo.getDslashConstant().volume_4d_cb;
+      dslash.setShmem(0);
 
-        issueRecv(halo, dslash, false); // Prepost receives
+      // record start of the dslash
+      PROFILE(qudaEventRecord(dslashStart[halo.bufferIndex], device::get_default_stream()), profile,
+              QUDA_PROFILE_EVENT_RECORD);
 
-        const int packIndex = getStreamIndex(dslashParam);
-        PROFILE(qudaStreamWaitEvent(device::get_stream(packIndex), dslashStart[halo.bufferIndex], 0), profile,
-                QUDA_PROFILE_STREAM_WAIT_EVENT);
-        const int parity_src = (in.SiteSubset() == QUDA_PARITY_SITE_SUBSET ? 1 - dslashParam.parity : 0);
-        issuePack(halo, in, dslash, parity_src, static_cast<MemoryLocation>(Host | (Remote * dslashParam.remote_write)),
-                  packIndex);
-
-        PROFILE(if (dslash_interior_compute) dslash.apply(device::get_default_stream()), profile,
-                QUDA_PROFILE_DSLASH_KERNEL);
-        if (aux_worker) aux_worker->apply(device::get_default_stream());
-
-        for (int i = 3; i >= 0; i--) { // only synchronize if we need to
-          if (!dslashParam.remote_write
-              || (dslashParam.commDim[i] && (!comm_peer2peer_enabled(0, i) || !comm_peer2peer_enabled(1, i)))) {
-            qudaStreamSynchronize(device::get_stream(packIndex));
-            break;
-          }
-        }
+      issueRecv(halo, dslash, false); // Prepost receives
 
-        for (int p2p = 0; p2p < 2; p2p++) { // schedule non-p2p traffic first, then do p2p
-          for (int i = 3; i >= 0; i--) {
-            if (!dslashParam.commDim[i]) continue;
+      const int packIndex = getStreamIndex(dslashParam);
+      PROFILE(qudaStreamWaitEvent(device::get_stream(packIndex), dslashStart[halo.bufferIndex], 0), profile,
+              QUDA_PROFILE_STREAM_WAIT_EVENT);
+      const int parity_src = (in.SiteSubset() == QUDA_PARITY_SITE_SUBSET ? 1 - dslashParam.parity : 0);
+      issuePack(halo, in, dslash, parity_src, static_cast<MemoryLocation>(Host | (Remote * dslashParam.remote_write)),
+                packIndex);
 
-            for (int dir = 1; dir >= 0; dir--) {
-              if ((comm_peer2peer_enabled(dir, i) + p2p) % 2 == 0) {
-                PROFILE(if (dslash_comms) halo.sendStart(
-                          2 * i + dir, device::get_stream(dslashParam.remote_write ? packIndex : 2 * i + dir), false,
-                          dslashParam.remote_write),
-                        profile, QUDA_PROFILE_COMMS_START);
-              } // is p2p?
-            } // dir
-          } // i
-        } // p2p
+      PROFILE(if (dslash_interior_compute) dslash.apply(device::get_default_stream()), profile, QUDA_PROFILE_DSLASH_KERNEL);
+      if (aux_worker) aux_worker->apply(device::get_default_stream());
 
-        DslashCommsPattern pattern(dslashParam.commDim, true);
-        while (pattern.completeSum < pattern.commDimTotal) {
+      for (int i = 3; i >= 0; i--) { // only synchronize if we need to
+        if (!dslashParam.remote_write
+            || (dslashParam.commDim[i] && (!comm_peer2peer_enabled(0, i) || !comm_peer2peer_enabled(1, i)))) {
+          qudaStreamSynchronize(device::get_stream(packIndex));
+          break;
+        }
+      }
 
-          for (int i = 3; i >= 0; i--) {
-            if (!dslashParam.commDim[i]) continue;
+      for (int p2p = 0; p2p < 2; p2p++) { // schedule non-p2p traffic first, then do p2p
+        for (int i = 3; i >= 0; i--) {
+          if (!dslashParam.commDim[i]) continue;
+
+          for (int dir = 1; dir >= 0; dir--) {
+            if ((comm_peer2peer_enabled(dir, i) + p2p) % 2 == 0) {
+              PROFILE(if (dslash_comms) halo.sendStart(
+                        2 * i + dir, device::get_stream(dslashParam.remote_write ? packIndex : 2 * i + dir), false,
+                        dslashParam.remote_write),
+                      profile, QUDA_PROFILE_COMMS_START);
+            } // is p2p?
+          }   // dir
+        }     // i
+      }       // p2p
+
+      DslashCommsPattern pattern(dslashParam.commDim, true);
+      while (pattern.completeSum < pattern.commDimTotal) {
 
-            for (int dir = 1; dir >= 0; dir--) {
+        for (int i = 3; i >= 0; i--) {
+          if (!dslashParam.commDim[i]) continue;
+
+          for (int dir = 1; dir >= 0; dir--) {
 
-              // Query if comms have finished
-              if (!pattern.commsCompleted[2 * i + dir]) {
-                if (commsComplete(halo, dslash, i, dir, false, false, true)) {
-                  pattern.commsCompleted[2 * i + dir] = 1;
-                  pattern.completeSum++;
-                }
+            // Query if comms have finished
+            if (!pattern.commsCompleted[2 * i + dir]) {
+              if (commsComplete(halo, dslash, i, dir, false, false, true)) {
+                pattern.commsCompleted[2 * i + dir] = 1;
+                pattern.completeSum++;
               }
             }
+          }
 
-            // enqueue the boundary dslash kernel as soon as the scatters have been enqueued
-            if (!pattern.dslashCompleted[2 * i] && pattern.dslashCompleted[pattern.previousDir[2 * i + 1]]
-                && pattern.commsCompleted[2 * i] && pattern.commsCompleted[2 * i + 1]) {
-              dslashParam.kernel_type = static_cast<KernelType>(i);
-              dslashParam.threads = dslash.Nface() * halo.getDslashConstant().ghostFaceCB[i]; // updating 2 or 6 faces
+          // enqueue the boundary dslash kernel as soon as the scatters have been enqueued
+          if (!pattern.dslashCompleted[2 * i] && pattern.dslashCompleted[pattern.previousDir[2 * i + 1]]
+              && pattern.commsCompleted[2 * i] && pattern.commsCompleted[2 * i + 1]) {
+            dslashParam.kernel_type = static_cast<KernelType>(i);
+            dslashParam.threads = dslash.Nface() * halo.getDslashConstant().ghostFaceCB[i]; // updating 2 or 6 faces
 
-              setMappedGhost(dslash, halo, true);
-              PROFILE(if (dslash_exterior_compute) dslash.apply(device::get_default_stream()), profile,
-                      QUDA_PROFILE_DSLASH_KERNEL);
-              setMappedGhost(dslash, halo, false);
+            setMappedGhost(dslash, halo, true);
+            PROFILE(if (dslash_exterior_compute) dslash.apply(device::get_default_stream()), profile, QUDA_PROFILE_DSLASH_KERNEL);
+            setMappedGhost(dslash, halo, false);
 
-              pattern.dslashCompleted[2 * i] = 1;
-            }
+            pattern.dslashCompleted[2 * i] = 1;
           }
         }
-
-        halo.bufferIndex = (1 - halo.bufferIndex);
-        PROFILE_STOP(profile, QUDA_PROFILE_TOTAL);
       }
-    };
 
-    /**
-       Variation of multi-gpu dslash where the packing kernel writes
-       buffers directly to host memory with fused halo update kernel
-    */
-    template <typename Dslash> struct DslashFusedZeroCopy : DslashPolicyImp<Dslash> {
+      halo.bufferIndex = (1 - halo.bufferIndex);
+      PROFILE_STOP(profile, QUDA_PROFILE_TOTAL);
+    }
+  };
 
-      void operator()(Dslash &dslash, cvector_ref<const ColorSpinorField> &in, const ColorSpinorField &halo,
-                      TimeProfile &profile)
-      {
-        PROFILE_START(profile, QUDA_PROFILE_TOTAL);
+/**
+   Variation of multi-gpu dslash where the packing kernel writes
+   buffers directly to host memory with fused halo update kernel
+*/
+  template <typename Dslash> struct DslashFusedZeroCopy : DslashPolicyImp<Dslash> {
 
-        auto &dslashParam = dslash.dslashParam;
-        dslashParam.kernel_type = INTERIOR_KERNEL;
-        dslashParam.threads = halo.getDslashConstant().volume_4d_cb;
-        dslash.setShmem(0);
+    void operator()(Dslash &dslash, cvector_ref<const ColorSpinorField> &in, const ColorSpinorField &halo,
+                    TimeProfile &profile)
+    {
+      PROFILE_START(profile, QUDA_PROFILE_TOTAL);
 
-        // record start of the dslash
-        PROFILE(qudaEventRecord(dslashStart[halo.bufferIndex], device::get_default_stream()), profile,
-                QUDA_PROFILE_EVENT_RECORD);
+      auto &dslashParam = dslash.dslashParam;
+      dslashParam.kernel_type = INTERIOR_KERNEL;
+      dslashParam.threads = halo.getDslashConstant().volume_4d_cb;
+      dslash.setShmem(0);
 
-        issueRecv(halo, dslash, false); // Prepost receives
+      // record start of the dslash
+      PROFILE(qudaEventRecord(dslashStart[halo.bufferIndex], device::get_default_stream()), profile,
+              QUDA_PROFILE_EVENT_RECORD);
 
-        const int packIndex = getStreamIndex(dslashParam);
-        PROFILE(qudaStreamWaitEvent(device::get_stream(packIndex), dslashStart[halo.bufferIndex], 0), profile,
-                QUDA_PROFILE_STREAM_WAIT_EVENT);
-        const int parity_src = (in.SiteSubset() == QUDA_PARITY_SITE_SUBSET ? 1 - dslashParam.parity : 0);
-        issuePack(halo, in, dslash, parity_src, static_cast<MemoryLocation>(Host | (Remote * dslashParam.remote_write)),
-                  packIndex);
-
-        PROFILE(if (dslash_interior_compute) dslash.apply(device::get_default_stream()), profile,
-                QUDA_PROFILE_DSLASH_KERNEL);
-        if (aux_worker) aux_worker->apply(device::get_default_stream());
-
-        for (int i = 3; i >= 0; i--) { // only synchronize if we need to
-          if (!dslashParam.remote_write
-              || (dslashParam.commDim[i] && (!comm_peer2peer_enabled(0, i) || !comm_peer2peer_enabled(1, i)))) {
-            qudaStreamSynchronize(device::get_stream(packIndex));
-            break;
-          }
-        }
+      issueRecv(halo, dslash, false); // Prepost receives
 
-        for (int p2p = 0; p2p < 2; p2p++) { // schedule non-p2p traffic first, then do p2p
-          for (int i = 3; i >= 0; i--) {
-            if (!dslashParam.commDim[i]) continue;
+      const int packIndex = getStreamIndex(dslashParam);
+      PROFILE(qudaStreamWaitEvent(device::get_stream(packIndex), dslashStart[halo.bufferIndex], 0), profile,
+              QUDA_PROFILE_STREAM_WAIT_EVENT);
+      const int parity_src = (in.SiteSubset() == QUDA_PARITY_SITE_SUBSET ? 1 - dslashParam.parity : 0);
+      issuePack(halo, in, dslash, parity_src, static_cast<MemoryLocation>(Host | (Remote * dslashParam.remote_write)),
+                packIndex);
 
-            for (int dir = 1; dir >= 0; dir--) {
-              if ((comm_peer2peer_enabled(dir, i) + p2p) % 2 == 0) {
-                PROFILE(if (dslash_comms) halo.sendStart(
-                          2 * i + dir, device::get_stream(dslashParam.remote_write ? packIndex : 2 * i + dir), false,
-                          dslashParam.remote_write),
-                        profile, QUDA_PROFILE_COMMS_START);
-              } // is p2p?
-            } // dir
-          } // i
-        } // p2p
+      PROFILE(if (dslash_interior_compute) dslash.apply(device::get_default_stream()), profile, QUDA_PROFILE_DSLASH_KERNEL);
+      if (aux_worker) aux_worker->apply(device::get_default_stream());
 
-        DslashCommsPattern pattern(dslashParam.commDim, true);
-        while (pattern.completeSum < pattern.commDimTotal) {
+      for (int i = 3; i >= 0; i--) { // only synchronize if we need to
+        if (!dslashParam.remote_write
+            || (dslashParam.commDim[i] && (!comm_peer2peer_enabled(0, i) || !comm_peer2peer_enabled(1, i)))) {
+          qudaStreamSynchronize(device::get_stream(packIndex));
+          break;
+        }
+      }
 
-          for (int i = 3; i >= 0; i--) {
-            if (!dslashParam.commDim[i]) continue;
+      for (int p2p = 0; p2p < 2; p2p++) { // schedule non-p2p traffic first, then do p2p
+        for (int i = 3; i >= 0; i--) {
+          if (!dslashParam.commDim[i]) continue;
+
+          for (int dir = 1; dir >= 0; dir--) {
+            if ((comm_peer2peer_enabled(dir, i) + p2p) % 2 == 0) {
+              PROFILE(if (dslash_comms) halo.sendStart(
+                        2 * i + dir, device::get_stream(dslashParam.remote_write ? packIndex : 2 * i + dir), false,
+                        dslashParam.remote_write),
+                      profile, QUDA_PROFILE_COMMS_START);
+            } // is p2p?
+          }   // dir
+        }     // i
+      }       // p2p
+
+      DslashCommsPattern pattern(dslashParam.commDim, true);
+      while (pattern.completeSum < pattern.commDimTotal) {
 
-            for (int dir = 1; dir >= 0; dir--) {
+        for (int i = 3; i >= 0; i--) {
+          if (!dslashParam.commDim[i]) continue;
 
-              // Query if comms have finished
-              if (!pattern.commsCompleted[2 * i + dir]) {
-                if (commsComplete(halo, dslash, i, dir, false, false, true)) {
-                  pattern.commsCompleted[2 * i + dir] = 1;
-                  pattern.completeSum++;
-                }
+          for (int dir = 1; dir >= 0; dir--) {
+
+            // Query if comms have finished
+            if (!pattern.commsCompleted[2 * i + dir]) {
+              if (commsComplete(halo, dslash, i, dir, false, false, true)) {
+                pattern.commsCompleted[2 * i + dir] = 1;
+                pattern.completeSum++;
               }
             }
           }
         }
+      }
 
-        if (pattern.commDimTotal) {
-          setFusedParam(dslashParam, dslash, halo); // setup for fused exterior kernel
-          setMappedGhost(dslash, halo, true);
-          PROFILE(if (dslash_exterior_compute) dslash.apply(device::get_default_stream()), profile,
-                  QUDA_PROFILE_DSLASH_KERNEL);
-          setMappedGhost(dslash, halo, false);
-        }
-
-        completeDslash(halo, dslashParam);
-        halo.bufferIndex = (1 - halo.bufferIndex);
-        PROFILE_STOP(profile, QUDA_PROFILE_TOTAL);
+      if (pattern.commDimTotal) {
+        setFusedParam(dslashParam, dslash, halo); // setup for fused exterior kernel
+        setMappedGhost(dslash, halo, true);
+        PROFILE(if (dslash_exterior_compute) dslash.apply(device::get_default_stream()), profile, QUDA_PROFILE_DSLASH_KERNEL);
+        setMappedGhost(dslash, halo, false);
       }
-    };
 
-    /**
-       Variation of multi-gpu dslash where the packing kernel is fused
-       into the interior dslash kernel.  Only really makes sense on
-       systems that are fully peer connected.
-    */
-    template <typename Dslash> struct DslashFusedPack : DslashPolicyImp<Dslash> {
-
-      void operator()(Dslash &dslash, cvector_ref<const ColorSpinorField> &, const ColorSpinorField &halo,
-                      TimeProfile &profile)
-      {
-        PROFILE_START(profile, QUDA_PROFILE_TOTAL);
+      completeDslash(halo, dslashParam);
+      halo.bufferIndex = (1 - halo.bufferIndex);
+      PROFILE_STOP(profile, QUDA_PROFILE_TOTAL);
+    }
+  };
 
-        auto &dslashParam = dslash.dslashParam;
-        dslashParam.kernel_type = INTERIOR_KERNEL;
-        dslashParam.threads = halo.getDslashConstant().volume_4d_cb;
-        dslash.setShmem(0);
+  /**
+     Variation of multi-gpu dslash where the packing kernel is fused
+     into the interior dslash kernel.  Only really makes sense on
+     systems that are fully peer connected.
+  */
+  template <typename Dslash> struct DslashFusedPack : DslashPolicyImp<Dslash> {
 
-        // record start of the dslash
-        PROFILE(qudaEventRecord(dslashStart[halo.bufferIndex], device::get_default_stream()), profile,
-                QUDA_PROFILE_EVENT_RECORD);
+    void operator()(Dslash &dslash, cvector_ref<const ColorSpinorField> &, const ColorSpinorField &halo,
+                    TimeProfile &profile)
+    {
+      PROFILE_START(profile, QUDA_PROFILE_TOTAL);
 
-        issueRecv(halo, dslash, false); // Prepost receives
+      auto &dslashParam = dslash.dslashParam;
+      dslashParam.kernel_type = INTERIOR_KERNEL;
+      dslashParam.threads = halo.getDslashConstant().volume_4d_cb;
+      dslash.setShmem(0);
 
-        MemoryLocation location = static_cast<MemoryLocation>(Host | (Remote * dslashParam.remote_write));
-        dslash.setPack(true, location); // enable fused kernel packing
+      // record start of the dslash
+      PROFILE(qudaEventRecord(dslashStart[halo.bufferIndex], device::get_default_stream()), profile,
+              QUDA_PROFILE_EVENT_RECORD);
 
-        PROFILE(if (dslash_interior_compute) dslash.apply(device::get_default_stream()), profile,
-                QUDA_PROFILE_DSLASH_KERNEL);
+      issueRecv(halo, dslash, false); // Prepost receives
 
-        dslash.setPack(false, location); // disable fused kernel packing
-        if (aux_worker) aux_worker->apply(device::get_default_stream());
+      MemoryLocation location = static_cast<MemoryLocation>(Host | (Remote * dslashParam.remote_write));
+      dslash.setPack(true, location); // enable fused kernel packing
 
-        for (int i = 3; i >= 0; i--) { // only synchronize if we need to
-          if (!dslashParam.remote_write
-              || (dslashParam.commDim[i] && (!comm_peer2peer_enabled(0, i) || !comm_peer2peer_enabled(1, i)))) {
-            qudaStreamSynchronize(device::get_default_stream());
-            break;
-          }
-        }
+      PROFILE(if (dslash_interior_compute) dslash.apply(device::get_default_stream()), profile, QUDA_PROFILE_DSLASH_KERNEL);
 
-        for (int p2p = 0; p2p < 2; p2p++) { // schedule non-p2p traffic first, then do p2p
-          for (int i = 3; i >= 0; i--) {
-            if (!dslashParam.commDim[i]) continue;
+      dslash.setPack(false, location); // disable fused kernel packing
+      if (aux_worker) aux_worker->apply(device::get_default_stream());
 
-            for (int dir = 1; dir >= 0; dir--) {
-              if ((comm_peer2peer_enabled(dir, i) + p2p) % 2 == 0) {
-                PROFILE(if (dslash_comms) halo.sendStart(2 * i + dir,
-                                                         dslashParam.remote_write ? device::get_default_stream() :
-                                                                                    device::get_stream(2 * i + dir),
-                                                         false, dslashParam.remote_write),
-                        profile, QUDA_PROFILE_COMMS_START);
-              } // is p2p?
-            } // dir
-          } // i
-        } // p2p
+      for (int i = 3; i >= 0; i--) { // only synchronize if we need to
+        if (!dslashParam.remote_write
+            || (dslashParam.commDim[i] && (!comm_peer2peer_enabled(0, i) || !comm_peer2peer_enabled(1, i)))) {
+          qudaStreamSynchronize(device::get_default_stream());
+          break;
+        }
+      }
 
-        DslashCommsPattern pattern(dslashParam.commDim, true);
-        while (pattern.completeSum < pattern.commDimTotal) {
+      for (int p2p = 0; p2p < 2; p2p++) { // schedule non-p2p traffic first, then do p2p
+        for (int i = 3; i >= 0; i--) {
+          if (!dslashParam.commDim[i]) continue;
+
+          for (int dir = 1; dir >= 0; dir--) {
+            if ((comm_peer2peer_enabled(dir, i) + p2p) % 2 == 0) {
+              PROFILE(if (dslash_comms) halo.sendStart(2 * i + dir,
+                                                       dslashParam.remote_write ? device::get_default_stream() :
+                                                                                  device::get_stream(2 * i + dir),
+                                                       false, dslashParam.remote_write),
+                      profile, QUDA_PROFILE_COMMS_START);
+            } // is p2p?
+          }   // dir
+        }     // i
+      }       // p2p
+
+      DslashCommsPattern pattern(dslashParam.commDim, true);
+      while (pattern.completeSum < pattern.commDimTotal) {
 
-          for (int i = 3; i >= 0; i--) {
-            if (!dslashParam.commDim[i]) continue;
+        for (int i = 3; i >= 0; i--) {
+          if (!dslashParam.commDim[i]) continue;
 
-            for (int dir = 1; dir >= 0; dir--) {
+          for (int dir = 1; dir >= 0; dir--) {
 
-              // Query if comms have finished
-              if (!pattern.commsCompleted[2 * i + dir]) {
-                if (commsComplete(halo, dslash, i, dir, false, false, true)) {
-                  pattern.commsCompleted[2 * i + dir] = 1;
-                  pattern.completeSum++;
-                }
+            // Query if comms have finished
+            if (!pattern.commsCompleted[2 * i + dir]) {
+              if (commsComplete(halo, dslash, i, dir, false, false, true)) {
+                pattern.commsCompleted[2 * i + dir] = 1;
+                pattern.completeSum++;
               }
             }
+          }
 
-            // enqueue the boundary dslash kernel as soon as the scatters have been
-            // enqueued
-            if (!pattern.dslashCompleted[2 * i] && pattern.dslashCompleted[pattern.previousDir[2 * i + 1]]
-                && pattern.commsCompleted[2 * i] && pattern.commsCompleted[2 * i + 1]) {
-              dslashParam.kernel_type = static_cast<KernelType>(i);
-              dslashParam.threads = dslash.Nface() * halo.getDslashConstant().ghostFaceCB[i]; // updating 2 or 6 faces
+          // enqueue the boundary dslash kernel as soon as the scatters have been
+          // enqueued
+          if (!pattern.dslashCompleted[2 * i] && pattern.dslashCompleted[pattern.previousDir[2 * i + 1]]
+              && pattern.commsCompleted[2 * i] && pattern.commsCompleted[2 * i + 1]) {
+            dslashParam.kernel_type = static_cast<KernelType>(i);
+            dslashParam.threads = dslash.Nface() * halo.getDslashConstant().ghostFaceCB[i]; // updating 2 or 6 faces
 
-              setMappedGhost(dslash, halo, true);
-              PROFILE(if (dslash_exterior_compute) dslash.apply(device::get_default_stream()), profile,
-                      QUDA_PROFILE_DSLASH_KERNEL);
-              setMappedGhost(dslash, halo, false);
+            setMappedGhost(dslash, halo, true);
+            PROFILE(if (dslash_exterior_compute) dslash.apply(device::get_default_stream()), profile, QUDA_PROFILE_DSLASH_KERNEL);
+            setMappedGhost(dslash, halo, false);
 
-              pattern.dslashCompleted[2 * i] = 1;
-            }
+            pattern.dslashCompleted[2 * i] = 1;
           }
         }
-
-        halo.bufferIndex = (1 - halo.bufferIndex);
-        PROFILE_STOP(profile, QUDA_PROFILE_TOTAL);
       }
-    };
 
-    /**
-       Variation of multi-gpu dslash where the packing kernel is fused
-       into the interior dslash kernel, and the halo update kernels are
-       all fused.  Only really makes sense on systems that are fully peer
-       connected.
-    */
-    template <typename Dslash> struct DslashFusedPackFusedHalo : DslashPolicyImp<Dslash> {
-
-      void operator()(Dslash &dslash, cvector_ref<const ColorSpinorField> &, const ColorSpinorField &halo,
-                      TimeProfile &profile)
-      {
-        PROFILE_START(profile, QUDA_PROFILE_TOTAL);
+      halo.bufferIndex = (1 - halo.bufferIndex);
+      PROFILE_STOP(profile, QUDA_PROFILE_TOTAL);
+    }
+  };
+
+  /**
+     Variation of multi-gpu dslash where the packing kernel is fused
+     into the interior dslash kernel, and the halo update kernels are
+     all fused.  Only really makes sense on systems that are fully peer
+     connected.
+  */
+  template <typename Dslash> struct DslashFusedPackFusedHalo : DslashPolicyImp<Dslash> {
+
+    void operator()(Dslash &dslash, cvector_ref<const ColorSpinorField> &, const ColorSpinorField &halo,
+                    TimeProfile &profile)
+    {
+      PROFILE_START(profile, QUDA_PROFILE_TOTAL);
 
-        auto &dslashParam = dslash.dslashParam;
-        dslashParam.kernel_type = INTERIOR_KERNEL;
-        dslashParam.threads = halo.getDslashConstant().volume_4d_cb;
-        dslash.setShmem(0);
+      auto &dslashParam = dslash.dslashParam;
+      dslashParam.kernel_type = INTERIOR_KERNEL;
+      dslashParam.threads = halo.getDslashConstant().volume_4d_cb;
+      dslash.setShmem(0);
 
-        // record start of the dslash
-        PROFILE(qudaEventRecord(dslashStart[halo.bufferIndex], device::get_default_stream()), profile,
-                QUDA_PROFILE_EVENT_RECORD);
+      // record start of the dslash
+      PROFILE(qudaEventRecord(dslashStart[halo.bufferIndex], device::get_default_stream()), profile,
+              QUDA_PROFILE_EVENT_RECORD);
 
-        issueRecv(halo, dslash, false); // Prepost receives
+      issueRecv(halo, dslash, false); // Prepost receives
 
-        MemoryLocation location = static_cast<MemoryLocation>(Host | (Remote * dslashParam.remote_write));
-        dslash.setPack(true, location); // enable fused kernel packing
+      MemoryLocation location = static_cast<MemoryLocation>(Host | (Remote * dslashParam.remote_write));
+      dslash.setPack(true, location); // enable fused kernel packing
 
-        PROFILE(if (dslash_interior_compute) dslash.apply(device::get_default_stream()), profile,
-                QUDA_PROFILE_DSLASH_KERNEL);
+      PROFILE(if (dslash_interior_compute) dslash.apply(device::get_default_stream()), profile, QUDA_PROFILE_DSLASH_KERNEL);
 
-        dslash.setPack(false, location); // disable fused kernel packing
-        if (aux_worker) aux_worker->apply(device::get_default_stream());
+      dslash.setPack(false, location); // disable fused kernel packing
+      if (aux_worker) aux_worker->apply(device::get_default_stream());
 
-        for (int i = 3; i >= 0; i--) { // only synchronize if we need to
-          if (!dslashParam.remote_write
-              || (dslashParam.commDim[i] && (!comm_peer2peer_enabled(0, i) || !comm_peer2peer_enabled(1, i)))) {
-            qudaStreamSynchronize(device::get_default_stream());
-            break;
-          }
+      for (int i = 3; i >= 0; i--) { // only synchronize if we need to
+        if (!dslashParam.remote_write
+            || (dslashParam.commDim[i] && (!comm_peer2peer_enabled(0, i) || !comm_peer2peer_enabled(1, i)))) {
+          qudaStreamSynchronize(device::get_default_stream());
+          break;
         }
+      }
 
-        for (int p2p = 0; p2p < 2; p2p++) { // schedule non-p2p traffic first, then do p2p
-          for (int i = 3; i >= 0; i--) {
-            if (!dslashParam.commDim[i]) continue;
-
-            for (int dir = 1; dir >= 0; dir--) {
-              if ((comm_peer2peer_enabled(dir, i) + p2p) % 2 == 0) {
-                PROFILE(if (dslash_comms) halo.sendStart(2 * i + dir,
-                                                         dslashParam.remote_write ? device::get_default_stream() :
-                                                                                    device::get_stream(2 * i + dir),
-                                                         false, dslashParam.remote_write),
-                        profile, QUDA_PROFILE_COMMS_START);
-              } // is p2p?
-            } // dir
-          } // i
-        } // p2p
-
-        DslashCommsPattern pattern(dslashParam.commDim, true);
-        while (pattern.completeSum < pattern.commDimTotal) {
+      for (int p2p = 0; p2p < 2; p2p++) { // schedule non-p2p traffic first, then do p2p
+        for (int i = 3; i >= 0; i--) {
+          if (!dslashParam.commDim[i]) continue;
+
+          for (int dir = 1; dir >= 0; dir--) {
+            if ((comm_peer2peer_enabled(dir, i) + p2p) % 2 == 0) {
+              PROFILE(if (dslash_comms) halo.sendStart(2 * i + dir,
+                                                       dslashParam.remote_write ? device::get_default_stream() :
+                                                                                  device::get_stream(2 * i + dir),
+                                                       false, dslashParam.remote_write),
+                      profile, QUDA_PROFILE_COMMS_START);
+            } // is p2p?
+          }   // dir
+        }     // i
+      }       // p2p
+
+      DslashCommsPattern pattern(dslashParam.commDim, true);
+      while (pattern.completeSum < pattern.commDimTotal) {
 
-          for (int i = 3; i >= 0; i--) {
-            if (!dslashParam.commDim[i]) continue;
+        for (int i = 3; i >= 0; i--) {
+          if (!dslashParam.commDim[i]) continue;
 
-            for (int dir = 1; dir >= 0; dir--) {
+          for (int dir = 1; dir >= 0; dir--) {
 
-              // Query if comms have finished
-              if (!pattern.commsCompleted[2 * i + dir]) {
-                if (commsComplete(halo, dslash, i, dir, false, false, true)) {
-                  pattern.commsCompleted[2 * i + dir] = 1;
-                  pattern.completeSum++;
-                }
+            // Query if comms have finished
+            if (!pattern.commsCompleted[2 * i + dir]) {
+              if (commsComplete(halo, dslash, i, dir, false, false, true)) {
+                pattern.commsCompleted[2 * i + dir] = 1;
+                pattern.completeSum++;
               }
             }
           }
         }
-
-        if (pattern.commDimTotal) {
-          setFusedParam(dslashParam, dslash, halo); // setup for fused exterior kernel
-          setMappedGhost(dslash, halo, true);
-          PROFILE(if (dslash_exterior_compute) dslash.apply(device::get_default_stream()), profile,
-                  QUDA_PROFILE_DSLASH_KERNEL);
-          setMappedGhost(dslash, halo, false);
-        }
-
-        completeDslash(halo, dslashParam);
-        halo.bufferIndex = (1 - halo.bufferIndex);
-        PROFILE_STOP(profile, QUDA_PROFILE_TOTAL);
       }
-    };
-
-    // whether we have initialized the dslash policy tuner
-    extern bool dslash_policy_init;
-
-    // used to keep track of which policy to start the autotuning
-    extern int first_active_policy;
-    extern int first_active_p2p_policy;
-
-    enum class QudaDslashPolicy {
-      QUDA_DSLASH,
-      QUDA_FUSED_DSLASH,
-      QUDA_GDR_DSLASH,
-      QUDA_FUSED_GDR_DSLASH,
-      QUDA_GDR_RECV_DSLASH,
-      QUDA_FUSED_GDR_RECV_DSLASH,
-      QUDA_ZERO_COPY_PACK_DSLASH,
-      QUDA_FUSED_ZERO_COPY_PACK_DSLASH,
-      QUDA_ZERO_COPY_DSLASH,
-      QUDA_FUSED_ZERO_COPY_DSLASH,
-      QUDA_ZERO_COPY_PACK_GDR_RECV_DSLASH,
-      QUDA_FUSED_ZERO_COPY_PACK_GDR_RECV_DSLASH,
-      QUDA_DSLASH_FUSED_PACK,
-      QUDA_DSLASH_FUSED_PACK_FUSED_HALO,
-      QUDA_SHMEM_UBER_PACKINTRA_DSLASH,
-      QUDA_SHMEM_UBER_PACKFULL_DSLASH,
-      QUDA_SHMEM_PACKINTRA_DSLASH,
-      QUDA_SHMEM_PACKFULL_DSLASH,
-      QUDA_DSLASH_POLICY_DISABLED // this MUST be the last element
-    };
-
-    // list of dslash policies that are enabled
-    extern std::vector<QudaDslashPolicy> policies;
-
-    // string used as a tunekey to ensure we retune if the dslash policy env changes
-    extern char policy_string[TuneKey::aux_n];
-
-    enum class QudaP2PPolicy {
-      QUDA_P2P_DEFAULT,         // no special hanlding for p2p
-      QUDA_P2P_COPY_ENGINE,     // use copy engine for p2p traffic
-      QUDA_P2P_REMOTE_WRITE,    // write packed halos directly to peers
-      QUDA_P2P_POLICY_DISABLED, // this must be the last element
-    };
 
-    // list of p2p policies that are enabled
-    extern std::vector<QudaP2PPolicy> p2p_policies;
-
-    template <typename Dslash> struct DslashFactory {
-
-      // map of GDR policies to their non-GDR equivalents
-      static auto blacklist_map(const QudaDslashPolicy &policy)
-      {
-        switch (policy) {
-        case QudaDslashPolicy::QUDA_GDR_DSLASH:
-        case QudaDslashPolicy::QUDA_GDR_RECV_DSLASH: return QudaDslashPolicy::QUDA_DSLASH;
-        case QudaDslashPolicy::QUDA_FUSED_GDR_DSLASH:
-        case QudaDslashPolicy::QUDA_FUSED_GDR_RECV_DSLASH: return QudaDslashPolicy::QUDA_FUSED_DSLASH;
-        case QudaDslashPolicy::QUDA_ZERO_COPY_PACK_GDR_RECV_DSLASH: return QudaDslashPolicy::QUDA_ZERO_COPY_PACK_DSLASH;
-        case QudaDslashPolicy::QUDA_FUSED_ZERO_COPY_PACK_GDR_RECV_DSLASH:
-          return QudaDslashPolicy::QUDA_FUSED_ZERO_COPY_PACK_DSLASH;
-        default: return policy;
-        }
+      if (pattern.commDimTotal) {
+        setFusedParam(dslashParam, dslash, halo); // setup for fused exterior kernel
+        setMappedGhost(dslash, halo, true);
+        PROFILE(if (dslash_exterior_compute) dslash.apply(device::get_default_stream()), profile, QUDA_PROFILE_DSLASH_KERNEL);
+        setMappedGhost(dslash, halo, false);
       }
 
-      static std::unique_ptr<DslashPolicyImp<Dslash>> create(const QudaDslashPolicy &policy_)
-      {
-        // if GDR policy and blacklist enabled, create the non-GDR equivalent
-        QudaDslashPolicy policy = comm_gdr_blacklist() ? blacklist_map(policy_) : policy_;
-
-        switch (policy) {
-        case QudaDslashPolicy::QUDA_DSLASH: return std::make_unique<DslashBasic<Dslash>>();
-        case QudaDslashPolicy::QUDA_FUSED_DSLASH: return std::make_unique<DslashFusedExterior<Dslash>>();
-        case QudaDslashPolicy::QUDA_GDR_DSLASH: return std::make_unique<DslashGDR<Dslash>>();
-        case QudaDslashPolicy::QUDA_FUSED_GDR_DSLASH: return std::make_unique<DslashFusedGDR<Dslash>>();
-        case QudaDslashPolicy::QUDA_GDR_RECV_DSLASH: return std::make_unique<DslashGDRRecv<Dslash>>();
-        case QudaDslashPolicy::QUDA_FUSED_GDR_RECV_DSLASH: return std::make_unique<DslashFusedGDRRecv<Dslash>>();
-        case QudaDslashPolicy::QUDA_ZERO_COPY_PACK_DSLASH: return std::make_unique<DslashZeroCopyPack<Dslash>>();
-        case QudaDslashPolicy::QUDA_FUSED_ZERO_COPY_PACK_DSLASH:
-          return std::make_unique<DslashFusedZeroCopyPack<Dslash>>();
-        case QudaDslashPolicy::QUDA_ZERO_COPY_PACK_GDR_RECV_DSLASH:
-          return std::make_unique<DslashZeroCopyPackGDRRecv<Dslash>>();
-        case QudaDslashPolicy::QUDA_FUSED_ZERO_COPY_PACK_GDR_RECV_DSLASH:
-          return std::make_unique<DslashFusedZeroCopyPackGDRRecv<Dslash>>();
-        case QudaDslashPolicy::QUDA_ZERO_COPY_DSLASH: return std::make_unique<DslashZeroCopy<Dslash>>();
-        case QudaDslashPolicy::QUDA_FUSED_ZERO_COPY_DSLASH: return std::make_unique<DslashFusedZeroCopy<Dslash>>();
-        case QudaDslashPolicy::QUDA_DSLASH_FUSED_PACK: return std::make_unique<DslashFusedPack<Dslash>>();
-        case QudaDslashPolicy::QUDA_DSLASH_FUSED_PACK_FUSED_HALO:
-          return std::make_unique<DslashFusedPackFusedHalo<Dslash>>();
-        case QudaDslashPolicy::QUDA_SHMEM_UBER_PACKINTRA_DSLASH:
-          return std::make_unique<DslashShmemUberPackIntra<Dslash>>();
-        case QudaDslashPolicy::QUDA_SHMEM_UBER_PACKFULL_DSLASH:
-          return std::make_unique<DslashShmemUberPackFull<Dslash>>();
-        case QudaDslashPolicy::QUDA_SHMEM_PACKINTRA_DSLASH: return std::make_unique<DslashShmemPackIntra<Dslash>>();
-        case QudaDslashPolicy::QUDA_SHMEM_PACKFULL_DSLASH: return std::make_unique<DslashShmemPackFull<Dslash>>();
-        default: errorQuda("Dslash policy %d not recognized", static_cast<int>(policy));
-        }
-
-        return std::make_unique<DslashPolicyImp<Dslash>>();
-      }
-    };
-
-    inline void enable_policy(QudaDslashPolicy p)
+      completeDslash(halo, dslashParam);
+      halo.bufferIndex = (1 - halo.bufferIndex);
+      PROFILE_STOP(profile, QUDA_PROFILE_TOTAL);
+    }
+  };
+
+  // whether we have initialized the dslash policy tuner
+  extern bool dslash_policy_init;
+
+  // used to keep track of which policy to start the autotuning
+  extern int first_active_policy;
+  extern int first_active_p2p_policy;
+
+  enum class QudaDslashPolicy {
+    QUDA_DSLASH,
+    QUDA_FUSED_DSLASH,
+    QUDA_GDR_DSLASH,
+    QUDA_FUSED_GDR_DSLASH,
+    QUDA_GDR_RECV_DSLASH,
+    QUDA_FUSED_GDR_RECV_DSLASH,
+    QUDA_ZERO_COPY_PACK_DSLASH,
+    QUDA_FUSED_ZERO_COPY_PACK_DSLASH,
+    QUDA_ZERO_COPY_DSLASH,
+    QUDA_FUSED_ZERO_COPY_DSLASH,
+    QUDA_ZERO_COPY_PACK_GDR_RECV_DSLASH,
+    QUDA_FUSED_ZERO_COPY_PACK_GDR_RECV_DSLASH,
+    QUDA_DSLASH_FUSED_PACK,
+    QUDA_DSLASH_FUSED_PACK_FUSED_HALO,
+    QUDA_SHMEM_UBER_PACKINTRA_DSLASH,
+    QUDA_SHMEM_UBER_PACKFULL_DSLASH,
+    QUDA_SHMEM_PACKINTRA_DSLASH,
+    QUDA_SHMEM_PACKFULL_DSLASH,
+    QUDA_DSLASH_POLICY_DISABLED // this MUST be the last element
+  };
+
+  // list of dslash policies that are enabled
+  extern std::vector<QudaDslashPolicy> policies;
+
+  // string used as a tunekey to ensure we retune if the dslash policy env changes
+  extern char policy_string[TuneKey::aux_n];
+
+  enum class QudaP2PPolicy {
+    QUDA_P2P_DEFAULT,         // no special hanlding for p2p
+    QUDA_P2P_COPY_ENGINE,     // use copy engine for p2p traffic
+    QUDA_P2P_REMOTE_WRITE,    // write packed halos directly to peers
+    QUDA_P2P_POLICY_DISABLED, // this must be the last element
+  };
+
+  // list of p2p policies that are enabled
+  extern std::vector<QudaP2PPolicy> p2p_policies;
+
+  template <typename Dslash> struct DslashFactory {
+
+    // map of GDR policies to their non-GDR equivalents
+    static auto blacklist_map(const QudaDslashPolicy &policy)
     {
-      size_t p_idx = static_cast<std::size_t>(p);
-      if (p >= QudaDslashPolicy::QUDA_DSLASH_POLICY_DISABLED) errorQuda("Invalid policy %lu", p_idx);
-      policies[p_idx] = p;
+      switch (policy) {
+      case QudaDslashPolicy::QUDA_GDR_DSLASH:
+      case QudaDslashPolicy::QUDA_GDR_RECV_DSLASH:
+        return QudaDslashPolicy::QUDA_DSLASH;
+      case QudaDslashPolicy::QUDA_FUSED_GDR_DSLASH:
+      case QudaDslashPolicy::QUDA_FUSED_GDR_RECV_DSLASH:
+        return QudaDslashPolicy::QUDA_FUSED_DSLASH;
+      case QudaDslashPolicy::QUDA_ZERO_COPY_PACK_GDR_RECV_DSLASH:
+        return QudaDslashPolicy::QUDA_ZERO_COPY_PACK_DSLASH;
+      case QudaDslashPolicy::QUDA_FUSED_ZERO_COPY_PACK_GDR_RECV_DSLASH:
+        return QudaDslashPolicy::QUDA_FUSED_ZERO_COPY_PACK_DSLASH;
+      default: return policy;
+      }
     }
 
-    inline void disable_policy(QudaDslashPolicy p)
+    static std::unique_ptr<DslashPolicyImp<Dslash>> create(const QudaDslashPolicy &policy_)
     {
-      policies[static_cast<std::size_t>(p)] = QudaDslashPolicy::QUDA_DSLASH_POLICY_DISABLED;
+      // if GDR policy and blacklist enabled, create the non-GDR equivalent
+      QudaDslashPolicy policy = comm_gdr_blacklist() ? blacklist_map(policy_) : policy_;
+
+      switch (policy) {
+      case QudaDslashPolicy::QUDA_DSLASH: return std::make_unique<DslashBasic<Dslash>>();
+      case QudaDslashPolicy::QUDA_FUSED_DSLASH: return std::make_unique<DslashFusedExterior<Dslash>>();
+      case QudaDslashPolicy::QUDA_GDR_DSLASH: return std::make_unique<DslashGDR<Dslash>>();
+      case QudaDslashPolicy::QUDA_FUSED_GDR_DSLASH: return std::make_unique<DslashFusedGDR<Dslash>>();
+      case QudaDslashPolicy::QUDA_GDR_RECV_DSLASH: return std::make_unique<DslashGDRRecv<Dslash>>();
+      case QudaDslashPolicy::QUDA_FUSED_GDR_RECV_DSLASH: return std::make_unique<DslashFusedGDRRecv<Dslash>>();
+      case QudaDslashPolicy::QUDA_ZERO_COPY_PACK_DSLASH: return std::make_unique<DslashZeroCopyPack<Dslash>>();
+      case QudaDslashPolicy::QUDA_FUSED_ZERO_COPY_PACK_DSLASH: return std::make_unique<DslashFusedZeroCopyPack<Dslash>>();
+      case QudaDslashPolicy::QUDA_ZERO_COPY_PACK_GDR_RECV_DSLASH: return std::make_unique<DslashZeroCopyPackGDRRecv<Dslash>>();
+      case QudaDslashPolicy::QUDA_FUSED_ZERO_COPY_PACK_GDR_RECV_DSLASH: return std::make_unique<DslashFusedZeroCopyPackGDRRecv<Dslash>>();
+      case QudaDslashPolicy::QUDA_ZERO_COPY_DSLASH: return std::make_unique<DslashZeroCopy<Dslash>>();
+      case QudaDslashPolicy::QUDA_FUSED_ZERO_COPY_DSLASH: return std::make_unique<DslashFusedZeroCopy<Dslash>>();
+      case QudaDslashPolicy::QUDA_DSLASH_FUSED_PACK: return std::make_unique<DslashFusedPack<Dslash>>();
+      case QudaDslashPolicy::QUDA_DSLASH_FUSED_PACK_FUSED_HALO: return std::make_unique<DslashFusedPackFusedHalo<Dslash>>();
+      case QudaDslashPolicy::QUDA_SHMEM_UBER_PACKINTRA_DSLASH: return std::make_unique<DslashShmemUberPackIntra<Dslash>>();
+      case QudaDslashPolicy::QUDA_SHMEM_UBER_PACKFULL_DSLASH: return std::make_unique<DslashShmemUberPackFull<Dslash>>();
+      case QudaDslashPolicy::QUDA_SHMEM_PACKINTRA_DSLASH: return std::make_unique<DslashShmemPackIntra<Dslash>>();
+      case QudaDslashPolicy::QUDA_SHMEM_PACKFULL_DSLASH: return std::make_unique<DslashShmemPackFull<Dslash>>();
+      default: errorQuda("Dslash policy %d not recognized", static_cast<int>(policy));
+      }
+
+      return std::make_unique<DslashPolicyImp<Dslash>>();
     }
+  };
 
-    template <typename Dslash> class DslashPolicyTune : public Tunable
+  inline void enable_policy(QudaDslashPolicy p)
+  {
+    size_t p_idx = static_cast<std::size_t>(p);
+    if (p >= QudaDslashPolicy::QUDA_DSLASH_POLICY_DISABLED) errorQuda("Invalid policy %lu", p_idx);
+    policies[p_idx] = p;
+  }
+
+  inline void disable_policy(QudaDslashPolicy p)
+  {
+    policies[static_cast<std::size_t>(p)] = QudaDslashPolicy::QUDA_DSLASH_POLICY_DISABLED;
+  }
+
+  template <typename Dslash> class DslashPolicyTune : public Tunable
+  {
+    Dslash &dslash;
+    using Arg = std::remove_reference_t<decltype(dslash.dslashParam)>;
+    Arg &dslashParam;
+    cvector_ref<const ColorSpinorField> &in;
+    const ColorSpinorField &halo;
+    TimeProfile &profile;
+
+    bool tuneGridDim() const override { return false; } // Don't tune the grid dimensions.
+    bool tuneAuxDim() const override { return true; }   // Do tune the aux dimensions.
+    unsigned int sharedBytesPerThread() const override { return 0; }
+    unsigned int sharedBytesPerBlock(const TuneParam &) const override { return 0; }
+
+  public:
+    DslashPolicyTune(Dslash &dslash, cvector_ref<const ColorSpinorField> &in, const ColorSpinorField &halo,
+                     TimeProfile &profile) :
+      dslash(dslash), dslashParam(dslash.dslashParam), in(in), halo(halo), profile(profile)
     {
-      Dslash &dslash;
-      using Arg = std::remove_reference_t<decltype(dslash.dslashParam)>;
-      Arg &dslashParam;
-      cvector_ref<const ColorSpinorField> &in;
-      const ColorSpinorField &halo;
-      TimeProfile &profile;
-
-      bool tuneGridDim() const override { return false; } // Don't tune the grid dimensions.
-      bool tuneAuxDim() const override { return true; }   // Do tune the aux dimensions.
-      unsigned int sharedBytesPerThread() const override { return 0; }
-      unsigned int sharedBytesPerBlock(const TuneParam &) const override { return 0; }
-
-    public:
-      DslashPolicyTune(Dslash &dslash, cvector_ref<const ColorSpinorField> &in, const ColorSpinorField &halo,
-                       TimeProfile &profile) :
-        dslash(dslash), dslashParam(dslash.dslashParam), in(in), halo(halo), profile(profile)
-      {
-        if (!dslash_policy_init) {
+      if (!dslash_policy_init) {
 
-          first_active_policy = static_cast<int>(QudaDslashPolicy::QUDA_DSLASH_POLICY_DISABLED);
-          first_active_p2p_policy = static_cast<int>(QudaP2PPolicy::QUDA_P2P_POLICY_DISABLED);
+        first_active_policy = static_cast<int>(QudaDslashPolicy::QUDA_DSLASH_POLICY_DISABLED);
+        first_active_p2p_policy = static_cast<int>(QudaP2PPolicy::QUDA_P2P_POLICY_DISABLED);
 
-          if (comm_peer2peer_enabled_global() & 2) { // enable/disable p2p copy engine policy tuning
-            p2p_policies[static_cast<std::size_t>(QudaP2PPolicy::QUDA_P2P_REMOTE_WRITE)]
+        if (comm_peer2peer_enabled_global() & 2) { // enable/disable p2p copy engine policy tuning
+          p2p_policies[static_cast<std::size_t>(QudaP2PPolicy::QUDA_P2P_REMOTE_WRITE)]
               = QudaP2PPolicy::QUDA_P2P_REMOTE_WRITE;
-            first_active_p2p_policy = static_cast<int>(QudaP2PPolicy::QUDA_P2P_REMOTE_WRITE);
-          }
+          first_active_p2p_policy = static_cast<int>(QudaP2PPolicy::QUDA_P2P_REMOTE_WRITE);
+        }
 
-          if (comm_peer2peer_enabled_global() & 1) { // enable/disable p2p direct store policy tuning
-            p2p_policies[static_cast<std::size_t>(QudaP2PPolicy::QUDA_P2P_COPY_ENGINE)]
+        if (comm_peer2peer_enabled_global() & 1) { // enable/disable p2p direct store policy tuning
+          p2p_policies[static_cast<std::size_t>(QudaP2PPolicy::QUDA_P2P_COPY_ENGINE)]
               = QudaP2PPolicy::QUDA_P2P_COPY_ENGINE;
-            first_active_p2p_policy = static_cast<int>(QudaP2PPolicy::QUDA_P2P_COPY_ENGINE);
-          }
+          first_active_p2p_policy = static_cast<int>(QudaP2PPolicy::QUDA_P2P_COPY_ENGINE);
+        }
 
-          if (!(comm_peer2peer_enabled_global() & 4)) { // enable/disable non-p2p policy tuning
-            p2p_policies[static_cast<std::size_t>(QudaP2PPolicy::QUDA_P2P_DEFAULT)] = QudaP2PPolicy::QUDA_P2P_DEFAULT;
-            first_active_p2p_policy = static_cast<int>(QudaP2PPolicy::QUDA_P2P_DEFAULT);
-          }
+        if (!(comm_peer2peer_enabled_global() & 4)) { // enable/disable non-p2p policy tuning
+          p2p_policies[static_cast<std::size_t>(QudaP2PPolicy::QUDA_P2P_DEFAULT)] = QudaP2PPolicy::QUDA_P2P_DEFAULT;
+          first_active_p2p_policy = static_cast<int>(QudaP2PPolicy::QUDA_P2P_DEFAULT);
+        }
 
-          static char *dslash_policy_env = getenv("QUDA_ENABLE_DSLASH_POLICY");
-          if (dslash_policy_env) { // set the policies to tune for explicitly
-            std::stringstream policy_list(dslash_policy_env);
-
-            int policy_;
-            while (policy_list >> policy_) {
-              QudaDslashPolicy dslash_policy = static_cast<QudaDslashPolicy>(policy_);
-
-              // check this is a valid policy choice
-              if ((dslash_policy == QudaDslashPolicy::QUDA_GDR_DSLASH
-                   || dslash_policy == QudaDslashPolicy::QUDA_FUSED_GDR_DSLASH
-                   || dslash_policy == QudaDslashPolicy::QUDA_GDR_RECV_DSLASH
-                   || dslash_policy == QudaDslashPolicy::QUDA_FUSED_GDR_RECV_DSLASH)
-                  && !comm_gdr_enabled()) {
-                errorQuda("Cannot select a GDR policy %d unless QUDA_ENABLE_GDR is set", static_cast<int>(dslash_policy));
-              }
+        static char *dslash_policy_env = getenv("QUDA_ENABLE_DSLASH_POLICY");
+        if (dslash_policy_env) { // set the policies to tune for explicitly
+          std::stringstream policy_list(dslash_policy_env);
+
+          int policy_;
+          while (policy_list >> policy_) {
+            QudaDslashPolicy dslash_policy = static_cast<QudaDslashPolicy>(policy_);
+
+            // check this is a valid policy choice
+            if ((dslash_policy == QudaDslashPolicy::QUDA_GDR_DSLASH
+                    || dslash_policy == QudaDslashPolicy::QUDA_FUSED_GDR_DSLASH
+                    || dslash_policy == QudaDslashPolicy::QUDA_GDR_RECV_DSLASH
+                    || dslash_policy == QudaDslashPolicy::QUDA_FUSED_GDR_RECV_DSLASH)
+                && !comm_gdr_enabled()) {
+              errorQuda("Cannot select a GDR policy %d unless QUDA_ENABLE_GDR is set", static_cast<int>(dslash_policy));
+            }
 
-              // check valid policy for nvshmem
-              if (dslash_policy == QudaDslashPolicy::QUDA_SHMEM_UBER_PACKINTRA_DSLASH
-                  || dslash_policy == QudaDslashPolicy::QUDA_SHMEM_UBER_PACKFULL_DSLASH
-                  || dslash_policy == QudaDslashPolicy::QUDA_SHMEM_PACKINTRA_DSLASH
-                  || dslash_policy == QudaDslashPolicy::QUDA_SHMEM_PACKFULL_DSLASH) {
+            // check valid policy for nvshmem
+            if (dslash_policy == QudaDslashPolicy::QUDA_SHMEM_UBER_PACKINTRA_DSLASH
+                || dslash_policy == QudaDslashPolicy::QUDA_SHMEM_UBER_PACKFULL_DSLASH
+                || dslash_policy == QudaDslashPolicy::QUDA_SHMEM_PACKINTRA_DSLASH
+                || dslash_policy == QudaDslashPolicy::QUDA_SHMEM_PACKFULL_DSLASH) {
 #ifndef NVSHMEM_COMMS
-                errorQuda("Cannot select a NVSHMEM policy %d when QUDA is not build with QUDA_NVSHMEM enabled.",
-                          static_cast<int>(dslash_policy));
+              errorQuda("Cannot select a NVSHMEM policy %d when QUDA is not build with QUDA_NVSHMEM enabled.",
+                        static_cast<int>(dslash_policy));
 #endif
-              }
-
-              enable_policy(static_cast<QudaDslashPolicy>(policy_));
-              first_active_policy = policy_ < first_active_policy ? policy_ : first_active_policy;
-              if (policy_list.peek() == ',') policy_list.ignore();
-            }
-            if (first_active_policy == static_cast<int>(QudaDslashPolicy::QUDA_DSLASH_POLICY_DISABLED))
-              errorQuda("No valid policy found in QUDA_ENABLE_DSLASH_POLICY");
-          } else {
-            enable_policy(QudaDslashPolicy::QUDA_DSLASH);
-            first_active_policy = 0;
-            enable_policy(QudaDslashPolicy::QUDA_FUSED_DSLASH);
-
-            // if we have gdr then enable tuning these policies
-            if (comm_gdr_enabled()) {
-              enable_policy(QudaDslashPolicy::QUDA_GDR_DSLASH);
-              enable_policy(QudaDslashPolicy::QUDA_FUSED_GDR_DSLASH);
-              enable_policy(QudaDslashPolicy::QUDA_GDR_RECV_DSLASH);
-              enable_policy(QudaDslashPolicy::QUDA_FUSED_GDR_RECV_DSLASH);
-            }
-
-            if (comm_zero_copy_enabled()) {
-              enable_policy(QudaDslashPolicy::QUDA_ZERO_COPY_PACK_DSLASH);
-              enable_policy(QudaDslashPolicy::QUDA_FUSED_ZERO_COPY_PACK_DSLASH);
-            }
-
-            if (comm_gdr_enabled()) {
-              enable_policy(QudaDslashPolicy::QUDA_ZERO_COPY_PACK_GDR_RECV_DSLASH);
-              enable_policy(QudaDslashPolicy::QUDA_FUSED_ZERO_COPY_PACK_GDR_RECV_DSLASH);
-            }
-
-            if (comm_zero_copy_enabled()) {
-              enable_policy(QudaDslashPolicy::QUDA_ZERO_COPY_DSLASH);
-              enable_policy(QudaDslashPolicy::QUDA_FUSED_ZERO_COPY_DSLASH);
-              enable_policy(QudaDslashPolicy::QUDA_DSLASH_FUSED_PACK);
-              enable_policy(QudaDslashPolicy::QUDA_DSLASH_FUSED_PACK_FUSED_HALO);
             }
 
-            if (comm_nvshmem_enabled()) {
-              enable_policy(QudaDslashPolicy::QUDA_SHMEM_UBER_PACKINTRA_DSLASH);
-              enable_policy(QudaDslashPolicy::QUDA_SHMEM_UBER_PACKFULL_DSLASH);
-              enable_policy(QudaDslashPolicy::QUDA_SHMEM_PACKINTRA_DSLASH);
-              enable_policy(QudaDslashPolicy::QUDA_SHMEM_PACKFULL_DSLASH);
-            }
-          }
-          // construct string specifying which policies have been enabled
-          for (int i = 0; i < (int)QudaDslashPolicy::QUDA_DSLASH_POLICY_DISABLED; i++) {
-            strcat(policy_string, (int)policies[i] == i ? "1" : "0");
+            enable_policy(static_cast<QudaDslashPolicy>(policy_));
+            first_active_policy = policy_ < first_active_policy ? policy_ : first_active_policy;
+            if (policy_list.peek() == ',') policy_list.ignore();
           }
-
-          static char *dslash_pack_env = getenv("QUDA_ENABLE_DSLASH_PACK");
-          if (dslash_pack_env && strcmp(dslash_pack_env, "0") == 0) {
-            if (getVerbosity() > QUDA_SILENT) warningQuda("Disabling Dslash halo packing");
-            dslash_pack_compute = false;
+          if (first_active_policy == static_cast<int>(QudaDslashPolicy::QUDA_DSLASH_POLICY_DISABLED))
+            errorQuda("No valid policy found in QUDA_ENABLE_DSLASH_POLICY");
+        } else {
+          enable_policy(QudaDslashPolicy::QUDA_DSLASH);
+          first_active_policy = 0;
+          enable_policy(QudaDslashPolicy::QUDA_FUSED_DSLASH);
+
+          // if we have gdr then enable tuning these policies
+          if (comm_gdr_enabled()) {
+            enable_policy(QudaDslashPolicy::QUDA_GDR_DSLASH);
+            enable_policy(QudaDslashPolicy::QUDA_FUSED_GDR_DSLASH);
+            enable_policy(QudaDslashPolicy::QUDA_GDR_RECV_DSLASH);
+            enable_policy(QudaDslashPolicy::QUDA_FUSED_GDR_RECV_DSLASH);
           }
 
-          static char *dslash_interior_env = getenv("QUDA_ENABLE_DSLASH_INTERIOR");
-          if (dslash_interior_env && strcmp(dslash_interior_env, "0") == 0) {
-            if (getVerbosity() > QUDA_SILENT) warningQuda("Disabling Dslash interior computation");
-            dslash_interior_compute = false;
+          if (comm_zero_copy_enabled()) {
+            enable_policy(QudaDslashPolicy::QUDA_ZERO_COPY_PACK_DSLASH);
+            enable_policy(QudaDslashPolicy::QUDA_FUSED_ZERO_COPY_PACK_DSLASH);
           }
 
-          static char *dslash_exterior_env = getenv("QUDA_ENABLE_DSLASH_EXTERIOR");
-          if (dslash_exterior_env && strcmp(dslash_exterior_env, "0") == 0) {
-            if (getVerbosity() > QUDA_SILENT) warningQuda("Disabling Dslash exterior computation");
-            dslash_exterior_compute = false;
+          if (comm_gdr_enabled()) {
+            enable_policy(QudaDslashPolicy::QUDA_ZERO_COPY_PACK_GDR_RECV_DSLASH);
+            enable_policy(QudaDslashPolicy::QUDA_FUSED_ZERO_COPY_PACK_GDR_RECV_DSLASH);
           }
 
-          static char *dslash_copy_env = getenv("QUDA_ENABLE_DSLASH_COPY");
-          if (dslash_copy_env && strcmp(dslash_copy_env, "0") == 0) {
-            if (getVerbosity() > QUDA_SILENT) warningQuda("Disabling Dslash host-device copying");
-            dslash_copy = false;
+          if (comm_zero_copy_enabled()) {
+            enable_policy(QudaDslashPolicy::QUDA_ZERO_COPY_DSLASH);
+            enable_policy(QudaDslashPolicy::QUDA_FUSED_ZERO_COPY_DSLASH);
+            enable_policy(QudaDslashPolicy::QUDA_DSLASH_FUSED_PACK);
+            enable_policy(QudaDslashPolicy::QUDA_DSLASH_FUSED_PACK_FUSED_HALO);
           }
 
-          static char *dslash_comms_env = getenv("QUDA_ENABLE_DSLASH_COMMS");
-          if (dslash_comms_env && strcmp(dslash_comms_env, "0") == 0) {
-            if (getVerbosity() > QUDA_SILENT) warningQuda("Disabling Dslash communication");
-            dslash_comms = false;
+          if (comm_nvshmem_enabled()) {
+            enable_policy(QudaDslashPolicy::QUDA_SHMEM_UBER_PACKINTRA_DSLASH);
+            enable_policy(QudaDslashPolicy::QUDA_SHMEM_UBER_PACKFULL_DSLASH);
+            enable_policy(QudaDslashPolicy::QUDA_SHMEM_PACKINTRA_DSLASH);
+            enable_policy(QudaDslashPolicy::QUDA_SHMEM_PACKFULL_DSLASH);
           }
         }
-
-        // before we do policy tuning we must ensure the kernel
-        // constituents have been tuned since we can't do nested tuning
-        if (!tuned()) {
-          disableProfileCount();
-
-          for (auto &p2p : p2p_policies) {
-
-            if (p2p == QudaP2PPolicy::QUDA_P2P_POLICY_DISABLED) continue;
-
-            bool p2p_enabled = comm_peer2peer_enabled_global();
-            if (p2p == QudaP2PPolicy::QUDA_P2P_DEFAULT)
-              comm_enable_peer2peer(false); // disable p2p if using default policy
-            dslashParam.remote_write = (p2p == QudaP2PPolicy::QUDA_P2P_REMOTE_WRITE ? 1 : 0);
-
-            for (auto &i : policies) {
-
-              if (i == QudaDslashPolicy::QUDA_DSLASH || i == QudaDslashPolicy::QUDA_FUSED_DSLASH
-                  || i == QudaDslashPolicy::QUDA_ZERO_COPY_PACK_DSLASH
-                  || i == QudaDslashPolicy::QUDA_FUSED_ZERO_COPY_PACK_DSLASH || i == QudaDslashPolicy::QUDA_ZERO_COPY_DSLASH
-                  || i == QudaDslashPolicy::QUDA_FUSED_ZERO_COPY_DSLASH || i == QudaDslashPolicy::QUDA_DSLASH_FUSED_PACK
-                  || i == QudaDslashPolicy::QUDA_DSLASH_FUSED_PACK_FUSED_HALO
-                  || i == QudaDslashPolicy::QUDA_SHMEM_UBER_PACKINTRA_DSLASH
-                  || i == QudaDslashPolicy::QUDA_SHMEM_UBER_PACKFULL_DSLASH
-                  || i == QudaDslashPolicy::QUDA_SHMEM_PACKINTRA_DSLASH
-                  || i == QudaDslashPolicy::QUDA_SHMEM_PACKFULL_DSLASH) {
-
-                auto dslashImp = DslashFactory<Dslash>::create(i);
-                (*dslashImp)(dslash, in, halo, profile);
-
-              } else if (i == QudaDslashPolicy::QUDA_GDR_DSLASH || i == QudaDslashPolicy::QUDA_FUSED_GDR_DSLASH
-                         || i == QudaDslashPolicy::QUDA_GDR_RECV_DSLASH || i == QudaDslashPolicy::QUDA_FUSED_GDR_RECV_DSLASH
-                         || i == QudaDslashPolicy::QUDA_ZERO_COPY_PACK_GDR_RECV_DSLASH
-                         || i == QudaDslashPolicy::QUDA_FUSED_ZERO_COPY_PACK_GDR_RECV_DSLASH) {
-                // if we are using GDR policies then we must tune the
-                // non-GDR equivalent as well - this ensures that all GPUs
-                // will have the required tunecache entries prior to
-                // potential process divergence regardless of which GPUs
-                // are blacklisted.
-                {
-                  QudaDslashPolicy policy = DslashFactory<Dslash>::blacklist_map(i);
-                  auto dslashImp = DslashFactory<Dslash>::create(policy);
-                  (*dslashImp)(dslash, in, halo, profile);
-                }
-
-                auto dslashImp = DslashFactory<Dslash>::create(i);
-                (*dslashImp)(dslash, in, halo, profile);
-
-              } else if (i != QudaDslashPolicy::QUDA_DSLASH_POLICY_DISABLED) {
-                errorQuda("Unsupported dslash policy %d\n", static_cast<int>(i));
-              }
-            }
-
-            comm_enable_peer2peer(p2p_enabled); // restore p2p state
-          } // p2p policies
-
-          enableProfileCount();
-          setPolicyTuning(true);
-        }
-        dslash_policy_init = true;
-
-        apply(device::get_default_stream());
-      }
-
-      virtual ~DslashPolicyTune() { setPolicyTuning(false); }
-
-    private:
-      void apply(const qudaStream_t &) override
-      {
-        TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
-
-        if (tp.aux.x >= static_cast<int>(policies.size())) errorQuda("Requested policy that is outside of range");
-        if (static_cast<QudaDslashPolicy>(tp.aux.x) == QudaDslashPolicy::QUDA_DSLASH_POLICY_DISABLED)
-          errorQuda("Requested policy is disabled");
-
-        bool p2p_enabled = comm_peer2peer_enabled_global();
-        if (p2p_policies[tp.aux.y] == QudaP2PPolicy::QUDA_P2P_DEFAULT)
-          comm_enable_peer2peer(false); // disable p2p if using default policy
-        dslashParam.remote_write = (p2p_policies[tp.aux.y] == QudaP2PPolicy::QUDA_P2P_REMOTE_WRITE ?
-                                      1 :
-                                      0); // set whether we are using remote packing writes or copy engines
-
-        auto dslashImp = DslashFactory<Dslash>::create(static_cast<QudaDslashPolicy>(tp.aux.x));
-        (*dslashImp)(dslash, in, halo, profile);
-
-        // restore p2p state
-        comm_enable_peer2peer(p2p_enabled);
-      }
-
-      // Find the best dslash policy
-      bool advanceAux(TuneParam &param) const override
-      {
-        while ((unsigned)param.aux.x < policies.size() - 1) {
-          param.aux.x++;
-          if (policies[param.aux.x] != QudaDslashPolicy::QUDA_DSLASH_POLICY_DISABLED) return true;
-        }
-        param.aux.x = first_active_policy;
-
-        while ((unsigned)param.aux.y < p2p_policies.size() - 1) {
-          param.aux.y++;
-          if (p2p_policies[param.aux.y] != QudaP2PPolicy::QUDA_P2P_POLICY_DISABLED) return true;
+        // construct string specifying which policies have been enabled
+        for (int i = 0; i < (int)QudaDslashPolicy::QUDA_DSLASH_POLICY_DISABLED; i++) {
+          strcat(policy_string, (int)policies[i] == i ? "1" : "0");
         }
-        param.aux.y = first_active_p2p_policy;
 
-        return false;
+       static char *dslash_pack_env = getenv("QUDA_ENABLE_DSLASH_PACK");
+       if (dslash_pack_env && strcmp(dslash_pack_env, "0") == 0) {
+	 if (getVerbosity() > QUDA_SILENT) warningQuda("Disabling Dslash halo packing");
+	 dslash_pack_compute = false;
+       }
+
+       static char *dslash_interior_env = getenv("QUDA_ENABLE_DSLASH_INTERIOR");
+       if (dslash_interior_env && strcmp(dslash_interior_env, "0") == 0) {
+	 if (getVerbosity() > QUDA_SILENT) warningQuda("Disabling Dslash interior computation");
+	 dslash_interior_compute = false;
+       }
+
+       static char *dslash_exterior_env = getenv("QUDA_ENABLE_DSLASH_EXTERIOR");
+       if (dslash_exterior_env && strcmp(dslash_exterior_env, "0") == 0) {
+	 if (getVerbosity() > QUDA_SILENT) warningQuda("Disabling Dslash exterior computation");
+	 dslash_exterior_compute = false;
+       }
+
+       static char *dslash_copy_env = getenv("QUDA_ENABLE_DSLASH_COPY");
+       if (dslash_copy_env && strcmp(dslash_copy_env, "0") == 0) {
+	 if (getVerbosity() > QUDA_SILENT) warningQuda("Disabling Dslash host-device copying");
+	 dslash_copy = false;
+       }
+
+       static char *dslash_comms_env = getenv("QUDA_ENABLE_DSLASH_COMMS");
+       if (dslash_comms_env && strcmp(dslash_comms_env, "0") == 0) {
+	 if (getVerbosity() > QUDA_SILENT) warningQuda("Disabling Dslash communication");
+	 dslash_comms = false;
+       }
       }
 
-      bool advanceTuneParam(TuneParam &param) const override { return advanceAux(param); }
+      // before we do policy tuning we must ensure the kernel
+      // constituents have been tuned since we can't do nested tuning
+      if (!tuned()) {
+        disableProfileCount();
+
+        for (auto &p2p : p2p_policies) {
+
+          if (p2p == QudaP2PPolicy::QUDA_P2P_POLICY_DISABLED) continue;
+
+          bool p2p_enabled = comm_peer2peer_enabled_global();
+          if (p2p == QudaP2PPolicy::QUDA_P2P_DEFAULT)
+            comm_enable_peer2peer(false); // disable p2p if using default policy
+          dslashParam.remote_write = (p2p == QudaP2PPolicy::QUDA_P2P_REMOTE_WRITE ? 1 : 0);
+
+          for (auto &i : policies) {
+
+            if (i == QudaDslashPolicy::QUDA_DSLASH ||
+                i == QudaDslashPolicy::QUDA_FUSED_DSLASH ||
+                i == QudaDslashPolicy::QUDA_ZERO_COPY_PACK_DSLASH ||
+                i == QudaDslashPolicy::QUDA_FUSED_ZERO_COPY_PACK_DSLASH ||
+                i == QudaDslashPolicy::QUDA_ZERO_COPY_DSLASH ||
+                i == QudaDslashPolicy::QUDA_FUSED_ZERO_COPY_DSLASH ||
+                i == QudaDslashPolicy::QUDA_DSLASH_FUSED_PACK ||
+                i == QudaDslashPolicy::QUDA_DSLASH_FUSED_PACK_FUSED_HALO ||
+                i == QudaDslashPolicy::QUDA_SHMEM_UBER_PACKINTRA_DSLASH ||
+                i == QudaDslashPolicy::QUDA_SHMEM_UBER_PACKFULL_DSLASH ||
+                i == QudaDslashPolicy::QUDA_SHMEM_PACKINTRA_DSLASH ||
+                i == QudaDslashPolicy::QUDA_SHMEM_PACKFULL_DSLASH) {
+
+              auto dslashImp = DslashFactory<Dslash>::create(i);
+              (*dslashImp)(dslash, in, halo, profile);
+
+          } else if (i == QudaDslashPolicy::QUDA_GDR_DSLASH ||
+                     i == QudaDslashPolicy::QUDA_FUSED_GDR_DSLASH ||
+                     i == QudaDslashPolicy::QUDA_GDR_RECV_DSLASH ||
+                     i == QudaDslashPolicy::QUDA_FUSED_GDR_RECV_DSLASH ||
+                     i == QudaDslashPolicy::QUDA_ZERO_COPY_PACK_GDR_RECV_DSLASH ||
+                     i == QudaDslashPolicy::QUDA_FUSED_ZERO_COPY_PACK_GDR_RECV_DSLASH) {
+              // if we are using GDR policies then we must tune the
+              // non-GDR equivalent as well - this ensures that all GPUs
+              // will have the required tunecache entries prior to
+              // potential process divergence regardless of which GPUs
+              // are blacklisted.
+              {
+                QudaDslashPolicy policy = DslashFactory<Dslash>::blacklist_map(i);
+                auto dslashImp = DslashFactory<Dslash>::create(policy);
+                (*dslashImp)(dslash, in, halo, profile);
+              }
 
-      void initTuneParam(TuneParam &param) const override
-      {
-        Tunable::initTuneParam(param);
-        param.aux.x = first_active_policy;
-        param.aux.y = first_active_p2p_policy;
-        param.aux.z = 0;
-      }
+              auto dslashImp = DslashFactory<Dslash>::create(i);
+              (*dslashImp)(dslash, in, halo, profile);
 
-      void defaultTuneParam(TuneParam &param) const override
-      {
-        Tunable::defaultTuneParam(param);
-        param.aux.x = first_active_policy;
-        param.aux.y = first_active_p2p_policy;
-        param.aux.z = 0;
-      }
+            } else if (i != QudaDslashPolicy::QUDA_DSLASH_POLICY_DISABLED) {
+              errorQuda("Unsupported dslash policy %d\n", static_cast<int>(i));
+            }
 
-      TuneKey tuneKey() const override
-      {
-        KernelType kernel_type = dslashParam.kernel_type;
-        dslashParam.kernel_type = KERNEL_POLICY;
-        TuneKey key = dslash.tuneKey();
-        strcat(key.aux, comm_dim_topology_string());
-        strcat(key.aux, comm_config_string()); // any change in P2P/GDR will be stored as a separate tunecache entry
-        strcat(key.aux, policy_string);        // any change in policies enabled will be stored as a separate entry
-        dslashParam.kernel_type = kernel_type;
-        return key;
-      }
+          }
 
-      long long flops() const override
-      {
-        KernelType kernel_type = dslashParam.kernel_type;
-        dslashParam.kernel_type = KERNEL_POLICY;
-        long long flops_ = dslash.flops();
-        dslashParam.kernel_type = kernel_type;
-        return flops_;
-      }
+          comm_enable_peer2peer(p2p_enabled); // restore p2p state
+        }                                     // p2p policies
 
-      long long bytes() const override
-      {
-        KernelType kernel_type = dslashParam.kernel_type;
-        dslashParam.kernel_type = KERNEL_POLICY;
-        long long bytes_ = dslash.bytes();
-        dslashParam.kernel_type = kernel_type;
-        return bytes_;
+        enableProfileCount();
+        setPolicyTuning(true);
       }
+      dslash_policy_init = true;
 
-      void preTune() override { dslash.preTune(); }
-
-      void postTune() override { dslash.postTune(); }
+      apply(device::get_default_stream());
+    }
 
-      int32_t getTuneRank() const override { return dslash.getTuneRank(); }
-    };
+   virtual ~DslashPolicyTune() { setPolicyTuning(false); }
+
+  private:
+   void apply(const qudaStream_t &) override
+   {
+     TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
+
+     if (tp.aux.x >= static_cast<int>(policies.size())) errorQuda("Requested policy that is outside of range");
+     if (static_cast<QudaDslashPolicy>(tp.aux.x) == QudaDslashPolicy::QUDA_DSLASH_POLICY_DISABLED)  errorQuda("Requested policy is disabled");
+
+     bool p2p_enabled = comm_peer2peer_enabled_global();
+     if (p2p_policies[tp.aux.y] == QudaP2PPolicy::QUDA_P2P_DEFAULT) comm_enable_peer2peer(false); // disable p2p if using default policy
+     dslashParam.remote_write = (p2p_policies[tp.aux.y] == QudaP2PPolicy::QUDA_P2P_REMOTE_WRITE ? 1 : 0); // set whether we are using remote packing writes or copy engines
+
+     auto dslashImp = DslashFactory<Dslash>::create(static_cast<QudaDslashPolicy>(tp.aux.x));
+     (*dslashImp)(dslash, in, halo, profile);
+
+     // restore p2p state
+     comm_enable_peer2peer(p2p_enabled);
+   }
+
+   // Find the best dslash policy
+   bool advanceAux(TuneParam &param) const override
+   {
+     while ((unsigned)param.aux.x < policies.size()-1) {
+       param.aux.x++;
+       if (policies[param.aux.x] != QudaDslashPolicy::QUDA_DSLASH_POLICY_DISABLED) return true;
+     }
+     param.aux.x = first_active_policy;
+
+     while ((unsigned)param.aux.y < p2p_policies.size()-1) {
+       param.aux.y++;
+       if (p2p_policies[param.aux.y] != QudaP2PPolicy::QUDA_P2P_POLICY_DISABLED) return true;
+     }
+     param.aux.y = first_active_p2p_policy;
+
+     return false;
+   }
+
+   bool advanceTuneParam(TuneParam &param) const override { return advanceAux(param); }
+
+   void initTuneParam(TuneParam &param) const override {
+     Tunable::initTuneParam(param);
+     param.aux.x = first_active_policy;
+     param.aux.y = first_active_p2p_policy;
+     param.aux.z = 0;
+   }
+
+   void defaultTuneParam(TuneParam &param) const override {
+     Tunable::defaultTuneParam(param);
+     param.aux.x = first_active_policy;
+     param.aux.y = first_active_p2p_policy;
+     param.aux.z = 0;
+   }
+
+   TuneKey tuneKey() const override {
+     KernelType kernel_type = dslashParam.kernel_type;
+     dslashParam.kernel_type = KERNEL_POLICY;
+     TuneKey key = dslash.tuneKey();
+     strcat(key.aux, comm_dim_topology_string());
+     strcat(key.aux, comm_config_string()); // any change in P2P/GDR will be stored as a separate tunecache entry
+     strcat(key.aux, policy_string);        // any change in policies enabled will be stored as a separate entry
+     dslashParam.kernel_type = kernel_type;
+     return key;
+   }
+
+   long long flops() const override {
+     KernelType kernel_type = dslashParam.kernel_type;
+     dslashParam.kernel_type = KERNEL_POLICY;
+     long long flops_ = dslash.flops();
+     dslashParam.kernel_type = kernel_type;
+     return flops_;
+   }
+
+   long long bytes() const override {
+     KernelType kernel_type = dslashParam.kernel_type;
+     dslashParam.kernel_type = KERNEL_POLICY;
+     long long bytes_ = dslash.bytes();
+     dslashParam.kernel_type = kernel_type;
+     return bytes_;
+   }
+
+   void preTune() override { dslash.preTune(); }
+
+   void postTune() override { dslash.postTune(); }
+
+   int32_t getTuneRank() const override { return dslash.getTuneRank(); }
+  };
 
   } // namespace dslash
 

From 83bb965fb99f92a7c7b7b31a93c32698b24ddb98 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Mon, 17 Jun 2024 17:11:57 +0200
Subject: [PATCH 055/108] Restoring untouched files

---
 include/blas_helper.cuh                 |  2 +-
 include/clover_backup.h                 |  6 ++++--
 include/color_spinor_field_order.h      |  2 +-
 include/dirac_quda.h                    |  7 +++----
 include/gauge_backup.h                  | 12 ++++++------
 include/kernels/blas_core.cuh           |  4 +++-
 include/kernels/contraction.cuh         |  4 ++--
 include/kernels/dslash_gamma_helper.cuh |  8 +++-----
 include/kernels/reduce_core.cuh         | 13 +++++++++----
 include/tune_quda.h                     |  4 ++--
 lib/dslash_clover_helper.cu             |  1 +
 lib/interface_quda.cpp                  | 14 ++++++++------
 lib/llfat_quda.cu                       |  4 ++--
 lib/quda_ptr.cpp                        |  5 ++++-
 lib/reduce_quda.cu                      |  4 ++--
 tests/invert_test.cpp                   |  4 ++--
 16 files changed, 53 insertions(+), 41 deletions(-)

diff --git a/include/blas_helper.cuh b/include/blas_helper.cuh
index ea60702db7..647290ee98 100644
--- a/include/blas_helper.cuh
+++ b/include/blas_helper.cuh
@@ -379,7 +379,7 @@ namespace quda
     template <template <typename...> class Functor,
               template <template <typename...> class, typename store_t, typename y_store_t, int, typename> class Blas,
               typename T, typename store_t, typename y_store_t, typename V, typename... Args>
-    constexpr void instantiate(const T &a, const T &b, const T &c, V &x_, Args &&...args)
+    constexpr void instantiate(const T &a, const T &b, const T &c, V &x_, Args &&... args)
     {
       unwrap_t<V> &x(x_);
       if (x.Nspin() == 4 || x.Nspin() == 2) {
diff --git a/include/clover_backup.h b/include/clover_backup.h
index f318056b76..65907bb923 100644
--- a/include/clover_backup.h
+++ b/include/clover_backup.h
@@ -54,12 +54,13 @@ namespace quda
       else {
         eigensolver = new CloverField(*eigensolver_);
       }
+
     }
   }; // Class
 
   void setupCloverFields(CloverField *collected_clover, CloverField *&precise, CloverField *&sloppy,
-                         CloverField *&precondition, CloverField *&refinement, CloverField *&eigensolver,
-                         const CloverBundleBackup &bkup)
+                        CloverField *&precondition, CloverField *&refinement, CloverField *&eigensolver,
+                        const CloverBundleBackup &bkup)
   {
     // First things first. The new collected gauge is going to become the 'precise'
     // Things to check: what to do about precise first (We copied it so we can free it)
@@ -135,5 +136,6 @@ namespace quda
       eigensolver = new CloverField(eigensolver_param);
       eigensolver->copy(*precise);
     }
+
   }
 } // namespace quda
diff --git a/include/color_spinor_field_order.h b/include/color_spinor_field_order.h
index 91a2957496..25f5234390 100644
--- a/include/color_spinor_field_order.h
+++ b/include/color_spinor_field_order.h
@@ -1201,7 +1201,7 @@ namespace quda
       using AllocInt = typename AllocType<huge_alloc>::type;
       using norm_type = float;
       Float *field = nullptr;
-      // #define LEGACY_ACCESSOR_NORM // legacy code where norm pointer and offset are stored instead of computed
+      //#define LEGACY_ACCESSOR_NORM // legacy code where norm pointer and offset are stored instead of computed
 #ifdef LEGACY_ACCESSOR_NORM
       norm_type *norm = nullptr;
 #endif
diff --git a/include/dirac_quda.h b/include/dirac_quda.h
index 2a66931db7..f68b235fde 100644
--- a/include/dirac_quda.h
+++ b/include/dirac_quda.h
@@ -228,15 +228,14 @@ namespace quda {
     /**
        @brief apply 'dslash' operator for the DiracOp. This may be e.g. AD
     */
-    virtual void Dslash(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in, QudaParity parity) const
-      = 0;
+    virtual void Dslash(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+                        QudaParity parity) const = 0;
 
     /**
        @brief Xpay version of Dslash
     */
     virtual void DslashXpay(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
-                            QudaParity parity, cvector_ref<const ColorSpinorField> &x, double k) const
-      = 0;
+                            QudaParity parity, cvector_ref<const ColorSpinorField> &x, double k) const = 0;
 
     /**
        @brief Similar to the Xpay version of Dslash, but used only by the Laplace op for
diff --git a/include/gauge_backup.h b/include/gauge_backup.h
index 823b4e1ad1..9294dfcb4a 100644
--- a/include/gauge_backup.h
+++ b/include/gauge_backup.h
@@ -7,12 +7,12 @@ namespace quda
 {
 
   struct GaugeBundleBackup {
-    GaugeField *precise = nullptr;
-    GaugeField *sloppy = nullptr;
-    GaugeField *precondition = nullptr;
-    GaugeField *refinement = nullptr;
-    GaugeField *eigensolver = nullptr;
-    GaugeField *extended = nullptr;
+    GaugeField *precise=nullptr;
+    GaugeField *sloppy=nullptr;
+    GaugeField *precondition=nullptr;
+    GaugeField *refinement=nullptr;
+    GaugeField *eigensolver=nullptr;
+    GaugeField *extended=nullptr;
 
     GaugeBundleBackup() = default;
 
diff --git a/include/kernels/blas_core.cuh b/include/kernels/blas_core.cuh
index 3aac7832c0..969a31fcae 100644
--- a/include/kernels/blas_core.cuh
+++ b/include/kernels/blas_core.cuh
@@ -53,7 +53,9 @@ namespace quda
     */
     template <typename Arg> struct Blas_ {
       Arg &arg;
-      constexpr Blas_(const Arg &arg) : arg(const_cast<Arg &>(arg)) { }
+      constexpr Blas_(const Arg &arg) : arg(const_cast<Arg&>(arg))
+      {
+      }
       static constexpr const char *filename() { return KERNEL_FILE; }
 
       __device__ __host__ inline void operator()(int i, int src_idx, int parity) const
diff --git a/include/kernels/contraction.cuh b/include/kernels/contraction.cuh
index 97b88cf407..96079ed041 100644
--- a/include/kernels/contraction.cuh
+++ b/include/kernels/contraction.cuh
@@ -114,8 +114,8 @@ namespace quda
       using real = typename Arg::real;
       using Vector = ColorSpinor<real, nColor, nSpin>;
 
-      constexpr array<array<int, nSpin>, nSpin * nSpin> gm_i = get_dr_gm_i();
-      constexpr array<array<complex<real>, nSpin>, nSpin * nSpin> g5gm_z = get_dr_g5gm_z<real>();
+      constexpr array<array<int, nSpin>, nSpin *nSpin> gm_i = get_dr_gm_i();
+      constexpr array<array<complex<real>, nSpin>, nSpin *nSpin> g5gm_z = get_dr_g5gm_z<real>();
 
       int s1 = arg.s1;
       int b1 = arg.b1;
diff --git a/include/kernels/dslash_gamma_helper.cuh b/include/kernels/dslash_gamma_helper.cuh
index 49b798b7a9..0d18a416ad 100644
--- a/include/kernels/dslash_gamma_helper.cuh
+++ b/include/kernels/dslash_gamma_helper.cuh
@@ -30,8 +30,7 @@ namespace quda {
     GammaArg(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in, int d, int proj = 0,
              real kappa = 0.0, real mu = 0.0, real epsilon = 0.0, bool dagger = false,
              QudaTwistGamma5Type twist = QUDA_TWIST_GAMMA5_INVALID) :
-      kernel_param(dim3(in.TwistFlavor() == QUDA_TWIST_NONDEG_DOUBLET ? in.VolumeCB() / 2 : in.VolumeCB(), in.size(),
-                        in.SiteSubset())),
+      kernel_param(dim3(in.TwistFlavor() == QUDA_TWIST_NONDEG_DOUBLET ? in.VolumeCB() / 2 : in.VolumeCB(), in.size(), in.SiteSubset())),
       d(d),
       proj(proj),
       doublet(in.TwistFlavor() == QUDA_TWIST_NONDEG_DOUBLET),
@@ -168,13 +167,12 @@ namespace quda {
 
     F out[MAX_MULTI_RHS]; // output vector field
     F in[MAX_MULTI_RHS];  // input vector field
-    const int d;          // which gamma matrix are we applying
+    const int d;        // which gamma matrix are we applying
     bool doublet;       // whether we applying the operator to a doublet
     const int volumeCB; // checkerboarded volume
 
     TauArg(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in, int d) :
-      kernel_param(dim3(in.TwistFlavor() == QUDA_TWIST_NONDEG_DOUBLET ? in.VolumeCB() / 2 : in.VolumeCB(), in.size(),
-                        in.SiteSubset())),
+      kernel_param(dim3(in.TwistFlavor() == QUDA_TWIST_NONDEG_DOUBLET ? in.VolumeCB() / 2 : in.VolumeCB(), in.size(), in.SiteSubset())),
       d(d),
       doublet(in.TwistFlavor() == QUDA_TWIST_NONDEG_DOUBLET),
       volumeCB(doublet ? in.VolumeCB() / 2 : in.VolumeCB())
diff --git a/include/kernels/reduce_core.cuh b/include/kernels/reduce_core.cuh
index 05f6771240..566a5238cf 100644
--- a/include/kernels/reduce_core.cuh
+++ b/include/kernels/reduce_core.cuh
@@ -59,7 +59,9 @@ namespace quda
       using Arg::Reducer::reducer::operator();
       static constexpr int reduce_block_dim = 1; // x_cb and parity are mapped to x dim
       Arg &arg;
-      constexpr Reduce_(const Arg &arg) : arg(const_cast<Arg &>(arg)) { }
+      constexpr Reduce_(const Arg &arg) : arg(const_cast<Arg&>(arg))
+      {
+      }
       static constexpr const char *filename() { return KERNEL_FILE; }
 
       __device__ __host__ inline reduce_t operator()(reduce_t &sum, int tid, int, int src_idx) const
@@ -94,7 +96,8 @@ namespace quda
        @tparam reduce_t The fundamental reduction type
        @tparam site_unroll Whether each thread must update the entire site
     */
-    template <typename reduce_t_, bool site_unroll_ = false> struct ReduceFunctor {
+    template <typename reduce_t_, bool site_unroll_ = false>
+    struct ReduceFunctor {
       using reduce_t = reduce_t_;
       using reducer = plus<reduce_t>;
       static constexpr bool site_unroll = site_unroll_;
@@ -475,7 +478,8 @@ namespace quda
        lattice site hence the site_unroll template parameter must be
        set true.
     */
-    template <typename real_reduce_t, typename real> struct HeavyQuarkResidualNorm_ {
+    template <typename real_reduce_t, typename real>
+    struct HeavyQuarkResidualNorm_ {
       using reduce_t = array<real_reduce_t, 3>;
       using reducer = plus<reduce_t>;
       static constexpr bool site_unroll = true;
@@ -511,7 +515,8 @@ namespace quda
       an entire lattice site hence the site_unroll template parameter
       must be set true.
     */
-    template <typename real_reduce_t, typename real> struct xpyHeavyQuarkResidualNorm_ {
+    template <typename real_reduce_t, typename real>
+    struct xpyHeavyQuarkResidualNorm_ {
       using reduce_t = array<real_reduce_t, 3>;
       using reducer = plus<reduce_t>;
       static constexpr bool site_unroll = true;
diff --git a/include/tune_quda.h b/include/tune_quda.h
index 409de4cbf2..a16738bcea 100644
--- a/include/tune_quda.h
+++ b/include/tune_quda.h
@@ -277,8 +277,8 @@ namespace quda {
       const int min_block_size = blockMin();
 
       if (tuneGridDim()) {
-        param.block = dim3(min_block_size, 1, 1);
-        param.grid = dim3(min_grid_size, 1, 1);
+	param.block = dim3(min_block_size,1,1);
+	param.grid = dim3(min_grid_size,1,1);
       } else {
 	// find the minimum valid blockDim
 	param.block = dim3((minThreads()+max_blocks-1)/max_blocks, 1, 1);
diff --git a/lib/dslash_clover_helper.cu b/lib/dslash_clover_helper.cu
index 56154c816b..0754191850 100644
--- a/lib/dslash_clover_helper.cu
+++ b/lib/dslash_clover_helper.cu
@@ -141,4 +141,5 @@ namespace quda {
       errorQuda("Twisted-clover operator has not been built");
     }
   }
+
 }
diff --git a/lib/interface_quda.cpp b/lib/interface_quda.cpp
index 6a907ec7bf..0b1bfe5473 100644
--- a/lib/interface_quda.cpp
+++ b/lib/interface_quda.cpp
@@ -3433,6 +3433,7 @@ void loadFatLongGaugeQuda(QudaInvertParam *inv_param, QudaGaugeParam *gauge_para
   }
 }
 
+
 template <class Interface, class... Args>
 void callMultiSrcQuda(void **_hp_x, void **_hp_b, QudaInvertParam *param, // color spinor field pointers, and inv_param
                       Interface op, Args... args)
@@ -3671,12 +3672,12 @@ void callMultiSrcQuda(void **_hp_x, void **_hp_b, QudaInvertParam *param, // col
     // Since input fields are in Native order now
     param_copy.dirac_order = QUDA_INTERNAL_DIRAC_ORDER;
 
-    // We need to set the cpu_prec in the param_copy, because the op() passed in
-    // to us will try to create wrappers to the pointers we pass in. They expect
-    // the input spinors to be on the host, and will use param_copy.cpu_prec to set
-    // the precision. We want to avoid the situation, where the internal prec and the
-    // cpu_prec are somehow different.
-    param_copy.cpu_prec = _collect_b[0].Precision();
+		// We need to set the cpu_prec in the param_copy, because the op() passed in
+		// to us will try to create wrappers to the pointers we pass in. They expect
+		// the input spinors to be on the host, and will use param_copy.cpu_prec to set
+		// the precision. We want to avoid the situation, where the internal prec and the
+		// cpu_prec are somehow different. 
+		param_copy.cpu_prec = _collect_b[0].Precision();
 
     // Do the solves
     for (int n = 0; n < param->num_src_per_sub_partition; n++) {
@@ -3751,6 +3752,7 @@ void invertMultiSrcQuda(void **_hp_x, void **_hp_b, QudaInvertParam *param)
   callMultiSrcQuda(_hp_x, _hp_b, param, op);
 }
 
+
 void dslashMultiSrcQuda(void **_hp_x, void **_hp_b, QudaInvertParam *param, QudaParity parity)
 {
   auto op = [](void *_x, void *_b, QudaInvertParam *param, QudaParity parity) { dslashQuda(_x, _b, param, parity); };
diff --git a/lib/llfat_quda.cu b/lib/llfat_quda.cu
index 3dc48279f2..a5feb80bb6 100644
--- a/lib/llfat_quda.cu
+++ b/lib/llfat_quda.cu
@@ -215,10 +215,10 @@ namespace quda {
                 for (int sig = 0; sig < 4; sig++) {
                   if (sig != nu && sig != rho) { computeStaple(fat, staple, staple1, u, sig, nu, rho, coeff[4], 0); }
                 } // sig
-              } // MIN_COEFF
+              }   // MIN_COEFF
             }
           } // rho
-        } // nu
+        }   // nu
       }
 
       getProfile().TPSTOP(QUDA_PROFILE_COMPUTE);
diff --git a/lib/quda_ptr.cpp b/lib/quda_ptr.cpp
index 7353721aa0..ac1a9bdd8b 100644
--- a/lib/quda_ptr.cpp
+++ b/lib/quda_ptr.cpp
@@ -88,7 +88,10 @@ namespace quda
     host = nullptr;
   }
 
-  quda_ptr::~quda_ptr() { destroy(); }
+  quda_ptr::~quda_ptr()
+  {
+    destroy();
+  }
 
   void quda_ptr::exchange(quda_ptr &obj, quda_ptr &&new_value)
   {
diff --git a/lib/reduce_quda.cu b/lib/reduce_quda.cu
index 7f09597421..61790d2ec5 100644
--- a/lib/reduce_quda.cu
+++ b/lib/reduce_quda.cu
@@ -147,8 +147,8 @@ namespace quda {
     // split the fields and recurse if needed
     template <template <typename reduce_t, typename real> class Functor, bool mixed, typename coeff_t, typename X,
               typename Y, typename Z, typename W, typename V>
-    auto instantiateReduce(const coeff_t &a, const coeff_t &b, const coeff_t &c, X &x, Y &y, Z &z, W &w,
-                           V &v) -> vector<typename Functor<double, double>::reduce_t>
+    auto instantiateReduce(const coeff_t &a, const coeff_t &b, const coeff_t &c, X &x, Y &y, Z &z, W &w, V &v)
+      -> vector<typename Functor<double, double>::reduce_t>
     {
       if (x.size() > get_max_multi_rhs()) {
         auto value0 = instantiateReduce<Functor, mixed, coeff_t, X, Y, Z, W, V>(
diff --git a/tests/invert_test.cpp b/tests/invert_test.cpp
index 133aa2acae..bf56dc2162 100644
--- a/tests/invert_test.cpp
+++ b/tests/invert_test.cpp
@@ -333,8 +333,8 @@ std::vector<std::array<double, 2>> solve(test_t param)
       _hp_x[i] = out[i].data();
       _hp_b[i] = in[i].data();
     }
-
-    // Run split grid
+    
+		// Run split grid
     invertMultiSrcQuda(_hp_x.data(), _hp_b.data(), &inv_param);
 
     quda::comm_allreduce_int(inv_param.iter);

From e4a1052d07fa8f9626e38c435a17576577d19652 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Mon, 17 Jun 2024 18:55:43 +0200
Subject: [PATCH 056/108] Adding const

---
 include/color_spinor_field.h       | 2 +-
 include/domain_decomposition.h     | 2 +-
 include/reference_wrapper_helper.h | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/include/color_spinor_field.h b/include/color_spinor_field.h
index f5d3f449a6..980b2e83b0 100644
--- a/include/color_spinor_field.h
+++ b/include/color_spinor_field.h
@@ -450,7 +450,7 @@ namespace quda
     /**
        @brief Returns DDParam
      */
-    DDParam DD() const { return dd; }
+    const DDParam DD() const { return dd; }
 
     /**
        @brief Sets DDParam from a given DDParam
diff --git a/include/domain_decomposition.h b/include/domain_decomposition.h
index a27632c2f4..cf88ba5070 100644
--- a/include/domain_decomposition.h
+++ b/include/domain_decomposition.h
@@ -140,7 +140,7 @@ namespace quda
       return true;
     }
 
-    inline bool operator!=(const DDParam &dd) { return !(*this == dd); }
+    inline bool operator!=(const DDParam &dd) const { return !(*this == dd); }
   };
 
 } // namespace quda
diff --git a/include/reference_wrapper_helper.h b/include/reference_wrapper_helper.h
index 5b2fe35828..7a4d549990 100644
--- a/include/reference_wrapper_helper.h
+++ b/include/reference_wrapper_helper.h
@@ -465,7 +465,7 @@ namespace quda
     }
 
     template <class U = T>
-    std::enable_if_t<std::is_same_v<std::remove_const_t<U>, ColorSpinorField>, DDParam &> DD() const
+    std::enable_if_t<std::is_same_v<std::remove_const_t<U>, ColorSpinorField>, const DDParam> DD() const
     {
       for (auto i = 1u; i < vector::size(); i++)
         if (operator[](i - 1).DD() != operator[](i).DD()) errorQuda("DD do not match %d != %d", i - 1, i);

From 83d5b9dfc4e4f6ce1db5a67fefdbfa4f3595dac7 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Mon, 17 Jun 2024 18:55:52 +0200
Subject: [PATCH 057/108] Fixing compilation issues

---
 include/kernels/dslash_domain_wall_4d.cuh        | 1 -
 include/kernels/dslash_domain_wall_5d.cuh        | 4 ++--
 lib/dslash_ndeg_twisted_clover_preconditioned.cu | 4 ++--
 3 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/include/kernels/dslash_domain_wall_4d.cuh b/include/kernels/dslash_domain_wall_4d.cuh
index 502ff35430..b04d488e01 100644
--- a/include/kernels/dslash_domain_wall_4d.cuh
+++ b/include/kernels/dslash_domain_wall_4d.cuh
@@ -58,7 +58,6 @@ namespace quda
 
       applyWilson<nParity, dagger, mykernel_type>(out, arg, coord, parity, idx, thread_dim, active, src_idx);
 
-      int xs = coord.x_cb + s * arg.dc.volume_4d_cb;
       if (xpay && mykernel_type == INTERIOR_KERNEL && arg.dd_x.isZero(coord)) {
         out = arg.a_5[s] * out;
       } else if (xpay && mykernel_type == INTERIOR_KERNEL) {
diff --git a/include/kernels/dslash_domain_wall_5d.cuh b/include/kernels/dslash_domain_wall_5d.cuh
index 783535bc63..27a5ce4eff 100644
--- a/include/kernels/dslash_domain_wall_5d.cuh
+++ b/include/kernels/dslash_domain_wall_5d.cuh
@@ -24,8 +24,8 @@ namespace quda
       m_f(m_f)
     {
       // remove the batch dimension from these constants, since these are used for 5-d checkerboard indexing
-      DslashArg<Float, nDim>::dc.X[4] = in.X(4);
-      DslashArg<Float, nDim>::dc.X5X4X3X2X1mX4X3X2X1 = (in.X(4) - 1) * DslashArg<Float, nDim>::dc.X4X3X2X1;
+      DslashArg<Float, nDim, DDArg>::dc.X[4] = in.X(4);
+      DslashArg<Float, nDim, DDArg>::dc.X5X4X3X2X1mX4X3X2X1 = (in.X(4) - 1) * DslashArg<Float, nDim, DDArg>::dc.X4X3X2X1;
     }
   };
 
diff --git a/lib/dslash_ndeg_twisted_clover_preconditioned.cu b/lib/dslash_ndeg_twisted_clover_preconditioned.cu
index a7e61d06e2..1cfbc280b7 100644
--- a/lib/dslash_ndeg_twisted_clover_preconditioned.cu
+++ b/lib/dslash_ndeg_twisted_clover_preconditioned.cu
@@ -104,8 +104,8 @@ namespace quda
       {
         constexpr int nDim = 4;
         auto halo = ColorSpinorField::create_comms_batch(in);
-        NdegTwistedCloverPreconditionedArg<Float, nColor, nDim, recon> arg(out, in, halo, U, A, a, b, c, xpay, x,
-                                                                           parity, dagger, comm_override);
+        NdegTwistedCloverPreconditionedArg<Float, nColor, nDim, DDArg, recon> arg(out, in, halo, U, A, a, b, c, xpay, x,
+                                                                                  parity, dagger, comm_override);
         NdegTwistedCloverPreconditioned<decltype(arg)> twisted(arg, out, in, halo);
         dslash::DslashPolicyTune<decltype(twisted)> policy(twisted, in, halo, profile);
       }

From 262d22a6559cddad7ddbd16120e339fb0a24f193 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Mon, 17 Jun 2024 22:02:04 +0200
Subject: [PATCH 058/108] supporting any value of dir in on_border

---
 include/domain_decomposition_helper.cuh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/include/domain_decomposition_helper.cuh b/include/domain_decomposition_helper.cuh
index 7b3418e9a9..a088322b29 100644
--- a/include/domain_decomposition_helper.cuh
+++ b/include/domain_decomposition_helper.cuh
@@ -73,7 +73,7 @@ namespace quda
 
     template <typename Coord> constexpr bool on_border(const Coord &x, const int &mu, const int &dir) const
     {
-      return (dir > 0) ? ((x.gx[mu] + 1) % blockDim[mu] == 0) : (x.gx[mu] % blockDim[mu] == 0);
+      return x.gx[mu] / blockDim[mu] != (x.gx[mu] + dir) / blockDim[mu];
     }
 
     template <typename Coord> constexpr bool isZero(const Coord &x) const

From ace5bf342598b937d397096cc81fcd98a3a26ed3 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Mon, 17 Jun 2024 22:02:21 +0200
Subject: [PATCH 059/108] solving compilation issue

---
 lib/dslash_domain_wall_4d_fused_m5.hpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/dslash_domain_wall_4d_fused_m5.hpp b/lib/dslash_domain_wall_4d_fused_m5.hpp
index cf689b7d37..48e1aaae9e 100644
--- a/lib/dslash_domain_wall_4d_fused_m5.hpp
+++ b/lib/dslash_domain_wall_4d_fused_m5.hpp
@@ -143,7 +143,8 @@ namespace quda
   };
 
   // use custom instantiate to deal with field splitting if needed
-  template <template <typename, int, QudaReconstructType> class Apply, typename Recon = ReconstructWilson, typename... Args>
+  template <template <typename, int, typename, QudaReconstructType> class Apply, typename Recon = ReconstructWilson,
+            typename... Args>
   void instantiate(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
                    cvector_ref<const ColorSpinorField> &x, cvector_ref<ColorSpinorField> &y, const GaugeField &U,
                    Args... args)

From 6d0100e9144b37789c68479de9e214527c86c717 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Mon, 17 Jun 2024 22:02:49 +0200
Subject: [PATCH 060/108] Making all kernels DD-aware. Not tested!

---
 include/kernels/covariant_derivative.cuh      | 11 +++++++++--
 .../kernels/dslash_ndeg_twisted_clover.cuh    | 10 ++++++++--
 ...ash_ndeg_twisted_clover_preconditioned.cuh |  9 ++++++---
 include/kernels/dslash_ndeg_twisted_mass.cuh  | 11 ++++++++---
 ...slash_ndeg_twisted_mass_preconditioned.cuh | 11 +++++++----
 include/kernels/dslash_staggered.cuh          | 16 +++++++++++-----
 .../dslash_twisted_clover_preconditioned.cuh  |  6 +++++-
 .../dslash_twisted_mass_preconditioned.cuh    | 12 +++++++++---
 include/kernels/dslash_wilson.cuh             |  7 ++++++-
 .../dslash_wilson_clover_hasenbusch_twist.cuh |  9 ++++++++-
 ...clover_hasenbusch_twist_preconditioned.cuh | 19 +++++++++++++++----
 .../dslash_wilson_clover_preconditioned.cuh   |  8 +++++++-
 include/kernels/laplace.cuh                   | 12 +++++++++---
 include/kernels/staggered_quark_smearing.cuh  |  9 +++++++--
 14 files changed, 115 insertions(+), 35 deletions(-)

diff --git a/include/kernels/covariant_derivative.cuh b/include/kernels/covariant_derivative.cuh
index 365d652438..f58e1b1ff7 100644
--- a/include/kernels/covariant_derivative.cuh
+++ b/include/kernels/covariant_derivative.cuh
@@ -77,7 +77,8 @@ namespace quda
 
     const int d = mu % 4;
 
-    if (mu < 4) { // Forward gather - compute fwd offset for vector fetch
+    if (mu < 4 and arg.dd_in.doHopping(coord, d, +1)) {
+      // Forward gather - compute fwd offset for vector fetch
 
       const int fwd_idx = getNeighborIndexCB(coord, d, +1, arg.dc);
       const bool ghost = (coord[d] + 1 >= arg.dim[d]) && isActive<kernel_type>(active, thread_dim, d, coord, arg);
@@ -96,7 +97,8 @@ namespace quda
         out += U * in;
       }
 
-    } else { // Backward gather - compute back offset for spinor and gauge fetch
+    } else if (mu >= 4 and arg.dd_in.doHopping(coord, d, -1)) {
+      // Backward gather - compute back offset for spinor and gauge fetch
 
       const int back_idx = getNeighborIndexCB(coord, d, -1, arg.dc);
       const int gauge_idx = back_idx;
@@ -144,6 +146,11 @@ namespace quda
       const int my_spinor_parity = nParity == 2 ? parity : 0;
       Vector out;
 
+      if (arg.dd_x.isZero(coord)) {
+        if (mykernel_type != EXTERIOR_KERNEL_ALL || active) arg.out[src_idx](coord.x_cb, my_spinor_parity) = out;
+        return;
+      }
+
       switch (arg.mu) { // ensure that mu is known to compiler for indexing in applyCovDev (avoid register spillage)
       case 0:
         applyCovDev<nParity, dagger, mykernel_type, 0>(out, arg, coord, parity, idx, thread_dim, active, src_idx);
diff --git a/include/kernels/dslash_ndeg_twisted_clover.cuh b/include/kernels/dslash_ndeg_twisted_clover.cuh
index 6f5ca92ef0..b5961ade89 100644
--- a/include/kernels/dslash_ndeg_twisted_clover.cuh
+++ b/include/kernels/dslash_ndeg_twisted_clover.cuh
@@ -68,11 +68,17 @@ namespace quda
       const int my_spinor_parity = nParity == 2 ? parity : 0;
       const int my_flavor_idx = coord.x_cb + flavor * arg.dc.volume_4d_cb;
       Vector out;
-      
+      if (arg.dd_out.isZero(coord)) {
+        if (mykernel_type != EXTERIOR_KERNEL_ALL || active) arg.out[src_idx](my_flavor_idx, my_spinor_parity) = out;
+        return;
+      }
+
       // defined in dslash_wilson.cuh
       applyWilson<nParity, dagger, mykernel_type>(out, arg, coord, parity, idx, thread_dim, active, src_idx);
 
-      if (mykernel_type == INTERIOR_KERNEL) {
+      if (mykernel_type == INTERIOR_KERNEL && arg.dd_x.isZero(coord)) {
+        out = arg.a * out;
+      } else if (mykernel_type == INTERIOR_KERNEL) {
         // apply the chiral and flavor twists
         // use consistent load order across s to ensure better cache locality
         Vector x = arg.x[src_idx](my_flavor_idx, my_spinor_parity);
diff --git a/include/kernels/dslash_ndeg_twisted_clover_preconditioned.cuh b/include/kernels/dslash_ndeg_twisted_clover_preconditioned.cuh
index 172516704b..70d5d6635d 100644
--- a/include/kernels/dslash_ndeg_twisted_clover_preconditioned.cuh
+++ b/include/kernels/dslash_ndeg_twisted_clover_preconditioned.cuh
@@ -71,13 +71,16 @@ namespace quda
       auto coord = getCoords<QUDA_4D_PC, mykernel_type>(arg, idx, flavor, parity, thread_dim);
 
       const int my_spinor_parity = nParity == 2 ? parity : 0;
+      int my_flavor_idx = coord.x_cb + flavor * arg.dc.volume_4d_cb;
       Vector out;
+      if (arg.dd_out.isZero(coord)) {
+        if (mykernel_type != EXTERIOR_KERNEL_ALL || active) arg.out[src_idx](my_flavor_idx, my_spinor_parity) = out;
+        return;
+      }
 
       // defined in dslash_wilson.cuh
       applyWilson<nParity, dagger, mykernel_type>(out, arg, coord, parity, idx, thread_dim, active, src_idx);
 
-      int my_flavor_idx = coord.x_cb + flavor * arg.dc.volume_4d_cb;
-
       if (mykernel_type != INTERIOR_KERNEL && active) {
         // if we're not the interior kernel, then we must sum the partial
         Vector x = arg.out[src_idx](my_flavor_idx, my_spinor_parity);
@@ -143,7 +146,7 @@ namespace quda
         Vector tmp = out_chi[0].chiral_reconstruct(0) + out_chi[1].chiral_reconstruct(1);
         tmp.toNonRel(); // switch back to non-chiral basis
 
-        if (xpay) {
+        if (xpay and not arg.dd_x.isZero(coord)) {
           Vector x = arg.x[src_idx](my_flavor_idx, my_spinor_parity);
           out = x + arg.a * tmp;
         } else {
diff --git a/include/kernels/dslash_ndeg_twisted_mass.cuh b/include/kernels/dslash_ndeg_twisted_mass.cuh
index 10422cebcc..1a91e59643 100644
--- a/include/kernels/dslash_ndeg_twisted_mass.cuh
+++ b/include/kernels/dslash_ndeg_twisted_mass.cuh
@@ -50,14 +50,19 @@ namespace quda
       auto coord = getCoords<QUDA_4D_PC, mykernel_type>(arg, idx, flavor, parity, thread_dim);
 
       const int my_spinor_parity = nParity == 2 ? parity : 0;
+      int my_flavor_idx = coord.x_cb + flavor * arg.dc.volume_4d_cb;
       Vector out;
+      if (arg.dd_out.isZero(coord)) {
+        if (mykernel_type != EXTERIOR_KERNEL_ALL || active) arg.out[src_idx](my_flavor_idx, my_spinor_parity) = out;
+        return;
+      }
 
       // defined in dslash_wilson.cuh
       applyWilson<nParity, dagger, mykernel_type>(out, arg, coord, parity, idx, thread_dim, active, src_idx);
 
-      int my_flavor_idx = coord.x_cb + flavor * arg.dc.volume_4d_cb;
-
-      if (mykernel_type == INTERIOR_KERNEL) {
+      if (mykernel_type == INTERIOR_KERNEL && arg.dd_x.isZero(coord)) {
+        out = arg.a * out;
+      } else if (mykernel_type == INTERIOR_KERNEL) {
         // apply the chiral and flavor twists
         // use consistent load order across s to ensure better cache locality
         Vector x0 = arg.x[src_idx](coord.x_cb + 0 * arg.dc.volume_4d_cb, my_spinor_parity);
diff --git a/include/kernels/dslash_ndeg_twisted_mass_preconditioned.cuh b/include/kernels/dslash_ndeg_twisted_mass_preconditioned.cuh
index 5d90fcb45e..e830aeaaaa 100644
--- a/include/kernels/dslash_ndeg_twisted_mass_preconditioned.cuh
+++ b/include/kernels/dslash_ndeg_twisted_mass_preconditioned.cuh
@@ -69,16 +69,19 @@ namespace quda
       auto coord = getCoords<QUDA_4D_PC, mykernel_type>(arg, idx, flavor, parity, thread_dim);
 
       const int my_spinor_parity = nParity == 2 ? parity : 0;
+      int my_flavor_idx = coord.x_cb + flavor * arg.dc.volume_4d_cb;
       Vector out;
+      if (arg.dd_out.isZero(coord)) {
+        if (mykernel_type != EXTERIOR_KERNEL_ALL || active) arg.out[src_idx](my_flavor_idx, my_spinor_parity) = out;
+        return;
+      }
 
       if (!dagger || Arg::asymmetric) // defined in dslash_wilson.cuh
         applyWilson<nParity, dagger, mykernel_type>(out, arg, coord, parity, idx, thread_dim, active, src_idx);
       else // defined in dslash_twisted_mass_preconditioned
         applyWilsonTM<nParity, dagger, 2, mykernel_type>(out, arg, coord, parity, idx, thread_dim, active, src_idx);
 
-      int my_flavor_idx = coord.x_cb + flavor * arg.dc.volume_4d_cb;
-
-      if (xpay && mykernel_type == INTERIOR_KERNEL) {
+      if (xpay && mykernel_type == INTERIOR_KERNEL && not arg.dd_x.isZero(coord)) {
 
         if (!dagger || Arg::asymmetric) { // apply inverse twist which is undone below
           // use consistent load order across s to ensure better cache locality
@@ -98,7 +101,7 @@ namespace quda
         Vector x = arg.out[src_idx](my_flavor_idx, my_spinor_parity);
         out += x;
       }
-      
+
       if (!dagger || Arg::asymmetric) { // apply A^{-1} to D*in
         SharedMemoryCache<Vector> cache;
         if (isComplete<mykernel_type>(arg, coord) && active) {
diff --git a/include/kernels/dslash_staggered.cuh b/include/kernels/dslash_staggered.cuh
index 38b742ca46..b359392862 100644
--- a/include/kernels/dslash_staggered.cuh
+++ b/include/kernels/dslash_staggered.cuh
@@ -99,7 +99,7 @@ namespace quda
     for (int d = 0; d < 4; d++) { // loop over dimension
 
       // standard - forward direction
-      {
+      if (arg.dd_in.doHopping(coord, d, +1)) {
         const bool ghost = (coord[d] + 1 >= arg.dim[d]) && isActive<kernel_type>(active, thread_dim, d, coord, arg);
         if (doHalo<kernel_type>(d) && ghost) {
           const int ghost_idx = ghostFaceIndexStaggered<1>(coord, arg.dim, d, 1);
@@ -115,7 +115,7 @@ namespace quda
       }
 
       // improved - forward direction
-      if (arg.improved) {
+      if (arg.improved and arg.dd_in.doHopping(coord, d, +3)) {
         const bool ghost = (coord[d] + 3 >= arg.dim[d]) && isActive<kernel_type>(active, thread_dim, d, coord, arg);
         if (doHalo<kernel_type>(d) && ghost) {
           const int ghost_idx = ghostFaceIndexStaggered<1>(coord, arg.dim, d, arg.nFace);
@@ -131,7 +131,7 @@ namespace quda
         }
       }
 
-      {
+      if (arg.dd_in.doHopping(coord, d, -1)) {
         // Backward gather - compute back offset for spinor and gauge fetch
         const bool ghost = (coord[d] - 1 < 0) && isActive<kernel_type>(active, thread_dim, d, coord, arg);
 
@@ -153,7 +153,7 @@ namespace quda
       }
 
       // improved - backward direction
-      if (arg.improved) {
+      if (arg.improved and arg.dd_in.doHopping(coord, d, -3)) {
         const bool ghost = (coord[d] - 3 < 0) && isActive<kernel_type>(active, thread_dim, d, coord, arg);
         if (doHalo<kernel_type>(d) && ghost) {
           // when updating replace arg.nFace with 1 here
@@ -196,12 +196,18 @@ namespace quda
       const int my_spinor_parity = nParity == 2 ? parity : 0;
 
       Vector out;
+      if (arg.dd_out.isZero(coord)) {
+        if (mykernel_type != EXTERIOR_KERNEL_ALL || active) arg.out[src_idx](coord.x_cb, my_spinor_parity) = out;
+        return;
+      }
 
       applyStaggered<nParity, mykernel_type>(out, arg, coord, parity, idx, thread_dim, active, src_idx);
 
       out *= arg.dagger_scale;
 
-      if (xpay && mykernel_type == INTERIOR_KERNEL) {
+      if (xpay && mykernel_type == INTERIOR_KERNEL && arg.dd_x.isZero(coord)) {
+        out = -out;
+      } else if (xpay && mykernel_type == INTERIOR_KERNEL) {
         Vector x = arg.x[src_idx](coord.x_cb, my_spinor_parity);
         out = arg.a * x - out;
       } else if (mykernel_type != INTERIOR_KERNEL) {
diff --git a/include/kernels/dslash_twisted_clover_preconditioned.cuh b/include/kernels/dslash_twisted_clover_preconditioned.cuh
index 10ea612a41..e3e86ee84a 100644
--- a/include/kernels/dslash_twisted_clover_preconditioned.cuh
+++ b/include/kernels/dslash_twisted_clover_preconditioned.cuh
@@ -67,6 +67,10 @@ namespace quda
       const int my_spinor_parity = nParity == 2 ? parity : 0;
 
       Vector out;
+      if (arg.dd_out.isZero(coord)) {
+        if (mykernel_type != EXTERIOR_KERNEL_ALL || active) arg.out[src_idx](coord.x_cb, my_spinor_parity) = out;
+        return;
+      }
 
       // defined in dslash_wilson.cuh
       applyWilson<nParity, dagger, mykernel_type>(out, arg, coord, parity, idx, thread_dim, active, src_idx);
@@ -104,7 +108,7 @@ namespace quda
 
         tmp.toNonRel(); // switch back to non-chiral basis
 
-        if (xpay) {
+        if (xpay && not arg.dd_x.isZero(coord)) {
           Vector x = arg.x[src_idx](coord.x_cb, my_spinor_parity);
           out = x + arg.a * tmp;
         } else {
diff --git a/include/kernels/dslash_twisted_mass_preconditioned.cuh b/include/kernels/dslash_twisted_mass_preconditioned.cuh
index 9485bb5c0c..6577848d2d 100644
--- a/include/kernels/dslash_twisted_mass_preconditioned.cuh
+++ b/include/kernels/dslash_twisted_mass_preconditioned.cuh
@@ -59,7 +59,8 @@ namespace quda
 
 #pragma unroll
     for (int d = 0; d < Arg::nDim; d++) { // loop over dimension
-      {                              // Forward gather - compute fwd offset for vector fetch
+      // Forward gather - compute fwd offset for vector fetch
+      if (arg.dd_in.doHopping(coord, d, +1)) {
         const int fwd_idx = getNeighborIndexCB(coord, d, +1, arg.dc);
         constexpr int proj_dir = dagger ? +1 : -1;
         const bool ghost
@@ -96,7 +97,8 @@ namespace quda
         }
       }
 
-      { // Backward gather - compute back offset for spinor and gauge fetch
+      // Backward gather - compute back offset for spinor and gauge fetch
+      if (arg.dd_in.doHopping(coord, d, -1)) {
         const int back_idx = getNeighborIndexCB(coord, d, -1, arg.dc);
         const int gauge_idx = back_idx;
         constexpr int proj_dir = dagger ? -1 : +1;
@@ -162,13 +164,17 @@ namespace quda
       const int my_spinor_parity = nParity == 2 ? parity : 0;
 
       Vector out;
+      if (arg.dd_out.isZero(coord)) {
+        if (mykernel_type != EXTERIOR_KERNEL_ALL || active) arg.out[src_idx](coord.x_cb, my_spinor_parity) = out;
+        return;
+      }
 
       if (!dagger || Arg::asymmetric) // defined in dslash_wilson.cuh
         applyWilson<nParity, dagger, mykernel_type>(out, arg, coord, parity, idx, thread_dim, active, src_idx);
       else // special dslash for symmetric dagger
         applyWilsonTM<nParity, dagger, 1, mykernel_type>(out, arg, coord, parity, idx, thread_dim, active, src_idx);
 
-      if (xpay && mykernel_type == INTERIOR_KERNEL) {
+      if (xpay && mykernel_type == INTERIOR_KERNEL && not arg.dd_x.isZero(coord)) {
         Vector x = arg.x[src_idx](coord.x_cb, my_spinor_parity);
         if (!dagger || Arg::asymmetric) {
           out += arg.a_inv * (x + arg.b_inv * x.igamma(4)); // apply inverse twist which is undone below
diff --git a/include/kernels/dslash_wilson.cuh b/include/kernels/dslash_wilson.cuh
index 12bdcd521a..ae2415f27e 100644
--- a/include/kernels/dslash_wilson.cuh
+++ b/include/kernels/dslash_wilson.cuh
@@ -178,10 +178,15 @@ namespace quda
       auto coord = getCoords<QUDA_4D_PC, mykernel_type>(arg, idx, 0, parity, thread_dim);
 
       const int my_spinor_parity = nParity == 2 ? parity : 0;
+      int xs = coord.x_cb + coord.s * arg.dc.volume_4d_cb;
       Vector out;
+      if (arg.dd_out.isZero(coord)) {
+        if (mykernel_type != EXTERIOR_KERNEL_ALL || active) arg.out[src_idx](xs, my_spinor_parity) = out;
+        return;
+      }
+
       applyWilson<nParity, dagger, mykernel_type>(out, arg, coord, parity, idx, thread_dim, active, src_idx);
 
-      int xs = coord.x_cb + coord.s * arg.dc.volume_4d_cb;
       if (xpay && mykernel_type == INTERIOR_KERNEL && arg.dd_x.isZero(coord)) {
         out = arg.a * out;
       } else if (xpay && mykernel_type == INTERIOR_KERNEL) {
diff --git a/include/kernels/dslash_wilson_clover_hasenbusch_twist.cuh b/include/kernels/dslash_wilson_clover_hasenbusch_twist.cuh
index 583b9bf959..d354ab07c4 100644
--- a/include/kernels/dslash_wilson_clover_hasenbusch_twist.cuh
+++ b/include/kernels/dslash_wilson_clover_hasenbusch_twist.cuh
@@ -60,10 +60,17 @@ namespace quda
       const int my_spinor_parity = nParity == 2 ? parity : 0;
       Vector out;
 
+      if (arg.dd_out.isZero(coord)) {
+        if (mykernel_type != EXTERIOR_KERNEL_ALL || active) arg.out[src_idx](coord.x_cb, my_spinor_parity) = out;
+        return;
+      }
+
       // defined in dslash_wilson.cuh
       applyWilson<nParity, dagger, mykernel_type>(out, arg, coord, parity, idx, thread_dim, active, src_idx);
 
-      if (mykernel_type == INTERIOR_KERNEL) {
+      if (mykernel_type == INTERIOR_KERNEL && arg.dd_x.isZero(coord)) {
+        out = arg.a * out;
+      } else if (mykernel_type == INTERIOR_KERNEL) {
         Vector x = arg.x[src_idx](coord.x_cb, my_spinor_parity);
         x.toRel(); // switch to chiral basis
 
diff --git a/include/kernels/dslash_wilson_clover_hasenbusch_twist_preconditioned.cuh b/include/kernels/dslash_wilson_clover_hasenbusch_twist_preconditioned.cuh
index 93474170d6..8b724962c5 100644
--- a/include/kernels/dslash_wilson_clover_hasenbusch_twist_preconditioned.cuh
+++ b/include/kernels/dslash_wilson_clover_hasenbusch_twist_preconditioned.cuh
@@ -64,6 +64,11 @@ namespace quda
 
       Vector out;
 
+      if (arg.dd_out.isZero(coord)) {
+        if (mykernel_type != EXTERIOR_KERNEL_ALL || active) arg.out[src_idx](coord.x_cb, my_spinor_parity) = out;
+        return;
+      }
+
       // defined in dslash_wilson.cuh
       applyWilson<nParity, dagger, mykernel_type>(out, arg, coord, parity, idx, thread_dim, active, src_idx);
 
@@ -75,7 +80,9 @@ namespace quda
 
       if (isComplete<mykernel_type>(arg, coord) && active) {
 
-        if (!Arg::clov_inv) {
+        if (!Arg::clov_inv and arg.dd_x.isZero(coord)) {
+          out = arg.a * out;
+        } else if (!Arg::clov_inv) {
           Vector x = arg.x[src_idx](coord.x_cb, my_spinor_parity);
           out = x + arg.a * out;
         } else {
@@ -100,14 +107,18 @@ namespace quda
           }
 
           tmp.toNonRel(); // switch back to non-chiral basis
-          Vector x = arg.x[src_idx](coord.x_cb, my_spinor_parity);
-          out = x + arg.a * tmp;
+          if (arg.dd_x.isZero(coord)) {
+            out = arg.a * tmp;
+          } else {
+            Vector x = arg.x[src_idx](coord.x_cb, my_spinor_parity);
+            out = x + arg.a * tmp;
+          }
         }
 
         // At this point: out = x + k A^{-1} D in or out = x + k D in
         //
         // now we must add on i g_5 b A x
-        {
+        if (not arg.dd_x.isZero(coord)) {
           Vector x = arg.x[src_idx](coord.x_cb, my_spinor_parity);
           x.toRel();
           Vector tmp;
diff --git a/include/kernels/dslash_wilson_clover_preconditioned.cuh b/include/kernels/dslash_wilson_clover_preconditioned.cuh
index d794ade8a8..0b2fb015e8 100644
--- a/include/kernels/dslash_wilson_clover_preconditioned.cuh
+++ b/include/kernels/dslash_wilson_clover_preconditioned.cuh
@@ -61,6 +61,10 @@ namespace quda
       const int my_spinor_parity = nParity == 2 ? parity : 0;
 
       Vector out;
+      if (arg.dd_out.isZero(coord)) {
+        if (mykernel_type != EXTERIOR_KERNEL_ALL || active) arg.out[src_idx](coord.x_cb, my_spinor_parity) = out;
+        return;
+      }
 
       // defined in dslash_wilson.cuh
       applyWilson<nParity, dagger, mykernel_type>(out, arg, coord, parity, idx, thread_dim, active, src_idx);
@@ -94,7 +98,9 @@ namespace quda
 
         tmp.toNonRel(); // switch back to non-chiral basis
 
-        if (xpay) {
+        if (xpay and arg.dd_x.isZero(coord)) {
+          out = arg.a * tmp;
+        } else if (xpay) {
           Vector x = arg.x[src_idx](coord.x_cb, my_spinor_parity);
           out = x + arg.a * tmp;
         } else {
diff --git a/include/kernels/laplace.cuh b/include/kernels/laplace.cuh
index 576ca435ed..3b520db587 100644
--- a/include/kernels/laplace.cuh
+++ b/include/kernels/laplace.cuh
@@ -86,7 +86,7 @@ namespace quda
 #pragma unroll
     for (int d = 0; d < Arg::nDim; d++) { // loop over dimension
       if (d != dir) {
-        {
+        if (arg.dd_in.doHopping(coord, d, +1)) {
           // Forward gather - compute fwd offset for vector fetch
           const bool ghost = (coord[d] + 1 >= arg.dim[d]) && isActive<kernel_type>(active, thread_dim, d, coord, arg);
 	  
@@ -107,7 +107,7 @@ namespace quda
             out += U * in;
           }
         }
-        {
+        if (arg.dd_in.doHopping(coord, d, -1)) {
           // Backward gather - compute back offset for spinor and gauge fetch
 
           const int back_idx = linkIndexM1(coord, arg.dim, d);
@@ -159,6 +159,10 @@ namespace quda
 
       const int my_spinor_parity = nParity == 2 ? parity : 0;
       Vector out;
+      if (arg.dd_out.isZero(coord)) {
+        if (kernel_type != EXTERIOR_KERNEL_ALL || active) arg.out[src_idx](coord.x_cb, my_spinor_parity) = out;
+        return;
+      }
 
       // We instantiate two kernel types:
       // case 4 is an operator in all x,y,z,t dimensions
@@ -173,7 +177,9 @@ namespace quda
         break;
       }
 
-      if (xpay && mykernel_type == INTERIOR_KERNEL) {
+      if (xpay && mykernel_type == INTERIOR_KERNEL && arg.dd_x.isZero(coord)) {
+        out = arg.a * out;
+      } else if (xpay && mykernel_type == INTERIOR_KERNEL) {
         Vector x = arg.x[src_idx](coord.x_cb, my_spinor_parity);
         out = arg.a * out + arg.b * x;
       } else if (mykernel_type != INTERIOR_KERNEL) {
diff --git a/include/kernels/staggered_quark_smearing.cuh b/include/kernels/staggered_quark_smearing.cuh
index f3e55c5e6d..8e8088c9cc 100644
--- a/include/kernels/staggered_quark_smearing.cuh
+++ b/include/kernels/staggered_quark_smearing.cuh
@@ -106,7 +106,7 @@ namespace quda
 #pragma unroll
     for (int d = 0; d < Arg::nDim; d++) { // loop over dimension
       if (d != dir) {
-        {
+        if (arg.dd_in.doHopping(coord, d, +2)) {
           // Forward gather - compute fwd offset for vector fetch
           const bool ghost
             = (coord[d] + 2 >= arg.dim[d]) && isActive<kernel_type>(active, thread_dim, d, coord, arg); // 1=>2
@@ -128,7 +128,7 @@ namespace quda
             out = mv_add(U_2link, in_2hop, out);
           }
         }
-        {
+        if (arg.dd_in.doHopping(coord, d, -2)) {
           // Backward gather - compute back offset for spinor and gauge fetch
           const bool ghost = (coord[d] - 2 < 0) && isActive<kernel_type>(active, thread_dim, d, coord, arg); // 1=>2
 
@@ -203,6 +203,11 @@ namespace quda
 
       const int my_spinor_parity = nParity == 2 ? parity : 0;
       Vector out;
+      if (arg.dd_out.isZero(coord)) {
+        if (kernel_type != EXTERIOR_KERNEL_ALL || active) arg.out[src_idx](coord.x_cb, my_spinor_parity) = out;
+        return;
+      }
+
       // We instantiate two kernel types:
       // case 4 is an operator in all x,y,z,t dimensions
       // case 3 is a spatial operator only, the t dimension is omitted.

From 0d293cf64eabfe05cdd69ca4e686925635fe8276 Mon Sep 17 00:00:00 2001
From: maddyscientist <mclark@nvidia.com>
Date: Fri, 21 Jun 2024 10:36:35 -0700
Subject: [PATCH 061/108] Fix warning in spin taste and minor cleanup

---
 include/kernels/spin_taste.cuh | 16 +++++++---------
 lib/spin_taste.cu              | 16 +++++-----------
 2 files changed, 12 insertions(+), 20 deletions(-)

diff --git a/include/kernels/spin_taste.cuh b/include/kernels/spin_taste.cuh
index bcd2460e2f..1c8ef0dbf6 100644
--- a/include/kernels/spin_taste.cuh
+++ b/include/kernels/spin_taste.cuh
@@ -19,16 +19,14 @@ namespace quda
     F out;      /** output vector field */
     const F in; /** input vector field */
 
-    SpinTasteArg(ColorSpinorField &out_, const ColorSpinorField &in_) :
-      kernel_param(dim3(in_.VolumeCB(), in_.SiteSubset(), 1)), out(out_), in(in_)
+    SpinTasteArg(ColorSpinorField &out, const ColorSpinorField &in) :
+      kernel_param(dim3(in.VolumeCB(), in.SiteSubset(), 1)), out(out), in(in)
     {
-      checkOrder(out_, in_);     // check all orders match
-      checkPrecision(out_, in_); // check all precisions match
-      checkLocation(out_, in_);  // check all locations match
-      if (!in_.isNative()) errorQuda("Unsupported field order colorspinor= %d \n", in_.FieldOrder());
-      if (!out_.isNative()) errorQuda("Unsupported field order colorspinor= %d \n", out_.FieldOrder());
-#pragma unroll
-      for (int i = 0; i < 4; i++) { X[i] = in_.X()[i]; }
+      checkOrder(out, in);     // check all orders match
+      checkPrecision(out, in); // check all precisions match
+      checkLocation(out, in);  // check all locations match
+      checkNative(out, in);
+      for (int i = 0; i < 4; i++) { X[i] = in.X()[i]; }
     }
   };
 
diff --git a/lib/spin_taste.cu b/lib/spin_taste.cu
index 824665f51e..0aefa2042e 100644
--- a/lib/spin_taste.cu
+++ b/lib/spin_taste.cu
@@ -67,22 +67,16 @@ namespace quda
     void preTune() { out.backup(); }
     void postTune() { out.restore(); }
 
-    long long flops() const { return 0; }
     long long bytes() const { return 2 * in.Bytes(); }
   };
 
-#ifdef GPU_STAGGERED_DIRAC
   void applySpinTaste(ColorSpinorField &out, const ColorSpinorField &in, QudaSpinTasteGamma gamma)
   {
-    instantiate<SpinTastePhase_>(out, in, gamma);
-    //// ensure that ghosts are updated if needed
-    // if (u.GhostExchange() == QUDA_GHOST_EXCHANGE_PAD) u.exchangeGhost();
-  }
-#else
-  void applySpinTaste(ColorSpinorField &out, const ColorSpinorField &in, QudaSpinTasteGamma gamma)
-  {
-    errorQuda("Gauge tools are not build");
+    if constexpr(is_enabled<QUDA_STAGGERED_DSLASH>()) {
+      instantiate<SpinTastePhase_>(out, in, gamma);
+    } else {
+      errorQuda("Staggered operator has not been built");
+    }
   }
-#endif
 
 } // namespace quda

From 2dae537db1227660d23b6a22efdd5ee9d502c5c4 Mon Sep 17 00:00:00 2001
From: maddyscientist <mclark@nvidia.com>
Date: Mon, 1 Jul 2024 16:39:10 -0700
Subject: [PATCH 062/108] Fix compilation issues

---
 include/color_spinor_field.h |  9 +++--
 tests/dslash_test_utils.h    | 64 +++++++++++++++++++++---------------
 2 files changed, 44 insertions(+), 29 deletions(-)

diff --git a/include/color_spinor_field.h b/include/color_spinor_field.h
index 980b2e83b0..2ab36a2a15 100644
--- a/include/color_spinor_field.h
+++ b/include/color_spinor_field.h
@@ -448,9 +448,14 @@ namespace quda
     void projectDD();
 
     /**
-       @brief Returns DDParam
+       @brief Returns DDParam (const version)
      */
-    const DDParam DD() const { return dd; }
+    const DDParam& DD() const { return dd; }
+
+    /**
+       @brief Returns DDParam (non const version)
+     */
+    DDParam& DD() { return dd; }
 
     /**
        @brief Sets DDParam from a given DDParam
diff --git a/tests/dslash_test_utils.h b/tests/dslash_test_utils.h
index 05ee37e69c..6a9494b80a 100644
--- a/tests/dslash_test_utils.h
+++ b/tests/dslash_test_utils.h
@@ -62,9 +62,9 @@ struct DslashTestWrapper {
   // CUDA color spinor fields
   std::vector<ColorSpinorField> cudaSpinor;
   std::vector<ColorSpinorField> cudaSpinorOut;
-  ColorSpinorField cudaSpinorTmp;
-  ColorSpinorField cudaSpinorTmp2;
-  ColorSpinorField cudaSpinorTmp3;
+  std::vector<ColorSpinorField> cudaSpinorTmp;
+  std::vector<ColorSpinorField> cudaSpinorTmp2;
+  std::vector<ColorSpinorField> cudaSpinorTmp3;
 
   // Dirac pointers
   quda::Dirac *dirac = nullptr;
@@ -339,9 +339,9 @@ struct DslashTestWrapper {
       cudaSpinorOut.resize(Nsrc);
       for (int i = 0; i < Nsrc; i++) cudaSpinorOut[i] = ColorSpinorField(csParam);
       if (test_domain_decomposition) {
-        cudaSpinorTmp = ColorSpinorField(csParam);
-        cudaSpinorTmp2 = ColorSpinorField(csParam);
-        cudaSpinorTmp3 = ColorSpinorField(csParam);
+        resize(cudaSpinorTmp, Nsrc, csParam);
+        resize(cudaSpinorTmp2, Nsrc, csParam);
+        resize(cudaSpinorTmp3, Nsrc, csParam);
       }
 
       printfQuda("Sending spinor field to GPU\n");
@@ -851,11 +851,13 @@ struct DslashTestWrapper {
     } else if (test_domain_decomposition) {
 
       if (dd_red_black) {
-        for (auto i = 0u; i < 4; i++) {
-          cudaSpinor.DD().blockDim[i] = dd_block_size[i];
-          cudaSpinorTmp.DD().blockDim[i] = dd_block_size[i];
-          cudaSpinorTmp2.DD().blockDim[i] = dd_block_size[i];
-          cudaSpinorTmp3.DD().blockDim[i] = dd_block_size[i];
+        for (int n = 0; n < Nsrc; n++) {
+          for (auto i = 0u; i < 4; i++) {
+            cudaSpinor[n].DD().blockDim[i] = dd_block_size[i];
+            cudaSpinorTmp[n].DD().blockDim[i] = dd_block_size[i];
+            cudaSpinorTmp2[n].DD().blockDim[i] = dd_block_size[i];
+            cudaSpinorTmp3[n].DD().blockDim[i] = dd_block_size[i];
+          }
         }
 
         blas::zero(cudaSpinorOut);
@@ -863,8 +865,10 @@ struct DslashTestWrapper {
         // We test that applying all 4 possible operators Drr, Drb, Dbr, Dbb gived D
         for (int col = 0; col < 4; col++) {
 
-          cudaSpinor.DD(DD::reset, DD::red_black_type, col % 2 == 0 ? DD::red_active : DD::black_active);
-          cudaSpinorTmp.DD(DD::reset, DD::red_black_type, col / 2 == 0 ? DD::red_active : DD::black_active);
+          for (int n = 0; n < num_src; n++) {
+            cudaSpinor[n].DD(DD::reset, DD::red_black_type, col % 2 == 0 ? DD::red_active : DD::black_active);
+            cudaSpinorTmp[n].DD(DD::reset, DD::red_black_type, col / 2 == 0 ? DD::red_active : DD::black_active);
+          }
 
           switch (dtest_type) {
           case dslash_test_type::Dslash: dirac->Dslash(cudaSpinorTmp, cudaSpinor, parity); break;
@@ -876,14 +880,16 @@ struct DslashTestWrapper {
             errorQuda("Test type %s not support for current Dslash", get_string(dtest_type_map, dtest_type).c_str());
           }
 
-          cudaSpinor.DD(DD::reset);
-          cudaSpinorTmp.DD(DD::reset);
+          for (int n = 0; n < num_src; n++) {
+            cudaSpinor[n].DD(DD::reset);
+            cudaSpinorTmp[n].DD(DD::reset);
 
-          // We also test that Dyx is same as D applied to projected in and out spinors
-          cudaSpinorTmp2 = cudaSpinor;
-          cudaSpinorTmp2.DD(DD::reset, DD::red_black_type, col % 2 == 0 ? DD::red_active : DD::black_active);
-          cudaSpinorTmp2.projectDD();
-          cudaSpinorTmp2.DD(DD::reset);
+            // We also test that Dyx is same as D applied to projected in and out spinors
+            cudaSpinorTmp2[n] = cudaSpinor[n];
+            cudaSpinorTmp2[n].DD(DD::reset, DD::red_black_type, col % 2 == 0 ? DD::red_active : DD::black_active);
+            cudaSpinorTmp2[n].projectDD();
+            cudaSpinorTmp2[n].DD(DD::reset);
+          }
 
           switch (dtest_type) {
           case dslash_test_type::Dslash: dirac->Dslash(cudaSpinorTmp3, cudaSpinorTmp2, parity); break;
@@ -895,9 +901,11 @@ struct DslashTestWrapper {
             errorQuda("Test type %s not support for current Dslash", get_string(dtest_type_map, dtest_type).c_str());
           }
 
-          cudaSpinorTmp3.DD(DD::reset, DD::red_black_type, col / 2 == 0 ? DD::red_active : DD::black_active);
-          cudaSpinorTmp3.projectDD();
-          cudaSpinorTmp3.DD(DD::reset);
+          for (int n = 0; n < num_src; n++) {
+            cudaSpinorTmp3[n].DD(DD::reset, DD::red_black_type, col / 2 == 0 ? DD::red_active : DD::black_active);
+            cudaSpinorTmp3[n].projectDD();
+            cudaSpinorTmp3[n].DD(DD::reset);
+          }
 
           if (dd_test_projection)
             blas::xpy(cudaSpinorTmp3, cudaSpinorOut);
@@ -907,10 +915,12 @@ struct DslashTestWrapper {
           spinorTmp = cudaSpinorTmp;
           spinorOut = cudaSpinorTmp3;
 
-          double deviation = std::pow(10, -(double)(ColorSpinorField::Compare(spinorTmp, spinorOut)));
-          printfQuda("Deviaton for (D-PDP)_{%d,%d}*spinor is %e\n", col % 2, col / 2, deviation);
-          double tol = getTolerance(cuda_prec);
-          EXPECT_LE(deviation, tol) << "Projected Dirac and project spinors do not agree";
+          for (int n = 0; n < Nsrc; n++) {
+            double deviation = std::pow(10, -(double)(ColorSpinorField::Compare(spinorTmp[n], spinorOut[n])));
+            printfQuda("Deviation for (D-PDP)_{%d,%d}*spinor is %e\n", col % 2, col / 2, deviation);
+            double tol = getTolerance(cuda_prec);
+            EXPECT_LE(deviation, tol) << "Projected Dirac and project spinors do not agree";
+          }
         }
       } else {
         errorQuda("Test dd type not supported");

From 158c34d8d3258f3043c70bdb2f9288665d8fb8fb Mon Sep 17 00:00:00 2001
From: maddyscientist <mclark@nvidia.com>
Date: Mon, 1 Jul 2024 16:39:58 -0700
Subject: [PATCH 063/108] Fix MR solver (avoid division by zero)

---
 lib/inv_mr_quda.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/inv_mr_quda.cpp b/lib/inv_mr_quda.cpp
index 324f3befc1..1f4422adf9 100644
--- a/lib/inv_mr_quda.cpp
+++ b/lib/inv_mr_quda.cpp
@@ -32,7 +32,7 @@ namespace quda
       bool local = true;
       for (int i = 0; i < QUDA_MAX_DIM; i++) {
         csParam.dd.blockDim[i] = param.schwarz_block[i];
-        local &= (x.full_dim(i) % csParam.dd.blockDim[i] == 0);
+        local &= (param.do_block_schwarz() && x.full_dim(i) % csParam.dd.blockDim[i] == 0);
       }
       // Disabling global_reduction if blocks are local and we do block_schwarz
       if (param.do_block_schwarz() and param.global_reduction) param.global_reduction = not local;

From 441be3cc671ee93f1438a8e1bc395b8d1657c229 Mon Sep 17 00:00:00 2001
From: maddyscientist <mclark@nvidia.com>
Date: Tue, 2 Jul 2024 16:39:00 -0700
Subject: [PATCH 064/108] Fix DD logic: issue was that global periodic boundary
 conditions were not being considered.  With these accounted for in on_border
 the tests appear to be working

---
 include/domain_decomposition_helper.cuh | 52 ++++++++++++-------------
 include/dslash_helper.cuh               |  6 ++-
 include/index_helper.cuh                |  5 ++-
 3 files changed, 31 insertions(+), 32 deletions(-)

diff --git a/include/domain_decomposition_helper.cuh b/include/domain_decomposition_helper.cuh
index a088322b29..49ad045630 100644
--- a/include/domain_decomposition_helper.cuh
+++ b/include/domain_decomposition_helper.cuh
@@ -19,7 +19,7 @@ namespace quda
     constexpr bool operator!() const { return true; }
 
     // Whether comms are required along given direction
-    template <typename DDArg, typename Arg> constexpr bool commDim(const int &, const DDArg &, const Arg &) const
+    template <typename DDArg, typename Arg> constexpr bool commDim(int, const DDArg &, const Arg &) const
     {
       return true;
     }
@@ -28,7 +28,7 @@ namespace quda
     template <typename Coord> constexpr bool isZero(const Coord &) const { return false; }
 
     // Whether do hopping with field at neighboring coord
-    template <typename Coord> constexpr bool doHopping(const Coord &, const int &, const int &) const { return true; }
+    template <typename Coord> constexpr bool doHopping(const Coord &, int, int) const { return true; }
   };
 
   // Red-black Block DD
@@ -45,13 +45,13 @@ namespace quda
       black_active(dd.type == QUDA_DD_NO or dd.is(DD::black_active)),
       block_hopping(dd.type == QUDA_DD_NO or not dd.is(DD::no_block_hopping))
     {
-      if (dd.type != QUDA_DD_NO and dd.type != QUDA_DD_RED_BLACK) { errorQuda("Unsupported type %d\n", dd.type); }
+      if (dd.type != QUDA_DD_NO and dd.type != QUDA_DD_RED_BLACK) { errorQuda("Unsupported type %d", dd.type); }
     }
 
     constexpr bool operator!() const { return false; }
 
     // Whether comms are required along given direction
-    template <typename DDArg, typename Arg> constexpr bool commDim(const int &d, const DDArg &dd, const Arg &arg) const
+    template <typename DDArg, typename Arg> constexpr bool commDim(int d, const DDArg &dd, const Arg &arg) const
     {
       if (not red_active and not black_active) return false;
       if (not dd.red_active and not dd.black_active) return false;
@@ -71,41 +71,37 @@ namespace quda
       return block_parity % 2 == 1;
     }
 
-    template <typename Coord> constexpr bool on_border(const Coord &x, const int &mu, const int &dir) const
+    template <typename Coord> constexpr bool on_border(const Coord &x, int mu, int dir) const
     {
-      return x.gx[mu] / blockDim[mu] != (x.gx[mu] + dir) / blockDim[mu];
+      int x_mu = x.gx[mu] + dir;
+      if (x_mu < 0) x_mu += x.gDim[mu];
+      if (x_mu >= x.gDim[mu]) x_mu -= x.gDim[mu];
+      return x.gx[mu] / blockDim[mu] != x_mu / blockDim[mu];
     }
 
     template <typename Coord> constexpr bool isZero(const Coord &x) const
     {
-      if (red_active and black_active) return false;
-      if (not red_active and not black_active) return true;
-
       bool is_black = block_parity(x);
+      bool is_red = not is_black;
 
-      // Checking if my parity is active
-      if (red_active and not is_black) return false;
-      if (black_active and is_black) return false;
-
+      if (is_red and red_active) return false;
+      if (is_black and black_active) return false;
       return true;
     }
 
-    template <typename Coord> constexpr bool doHopping(const Coord &x, const int &mu, const int &dir) const
+    template <typename Coord> constexpr bool doHopping(const Coord &x, int mu, int dir) const
     {
-      if (red_active and black_active and block_hopping) return true;
-      if (not red_active and not black_active) return false;
-
-      bool swap = on_border(x, mu, dir);
-      if (swap and not block_hopping) return false;
-      if (not swap and red_active and black_active) return true;
-
-      // Neighbor color
-      bool is_black = block_parity(x) ^ swap;
-
-      // Checking if neighbor is active
-      if (red_active and not is_black) return true;
-      if (black_active and is_black) return true;
-
+      bool is_black = block_parity(x);
+      bool is_red = !is_black;
+      bool is_border = on_border(x, mu, dir);
+
+      if (!is_border) { // Within block
+        if (is_red and red_active) return true;
+        if (is_black and black_active) return true;
+      } else if (block_hopping) { // Between blocks
+        if (is_red and black_active) return true;
+        if (is_black and red_active) return true;
+      }
       return false;
     }
   };
diff --git a/include/dslash_helper.cuh b/include/dslash_helper.cuh
index e728620c1d..9a88584cc3 100644
--- a/include/dslash_helper.cuh
+++ b/include/dslash_helper.cuh
@@ -105,6 +105,7 @@ namespace quda
   {
     constexpr auto nDim = Arg::nDim;
     Coord<nDim> coord;
+    for (auto i = 0; i < nDim; i++) coord.gDim[i] = arg.gDim[i];
     dim = kernel_type; // keep compiler happy
 
     // only for 5-d checkerboarding where we need to include the fifth dimension
@@ -256,6 +257,7 @@ namespace quda
 
     const int_fastdiv X0h;
     const int_fastdiv dim[5]; // full lattice dimensions
+    const int gDim[5];        // global full lattice dimensions
     const int volumeCB;       // checkerboarded volume
     int commDim[4];           // whether a given dimension is partitioned or not (potentially overridden for Schwarz)
 
@@ -328,9 +330,9 @@ namespace quda
       reconstruct(U.Reconstruct()),
       X0h(nParity == 2 ? in.X(0) / 2 : in.X(0)),
       dim {(3 - nParity) * in.X(0), in.X(1), in.X(2), in.X(3), in.Ndim() == 5 ? in.X(4) : 1},
+      gDim {comm_dim(0) * dim[0], comm_dim(1) * dim[1], comm_dim(2) * dim[2], comm_dim(3) * dim[3], dim[4]},
       volumeCB(in.VolumeCB()),
-      commCoord {comm_coord(0) * dim[0], comm_coord(1) * dim[1], comm_coord(2) * dim[2], comm_coord(3) * dim[3],
-                 comm_coord(4) * dim[4]},
+      commCoord {comm_coord(0) * dim[0], comm_coord(1) * dim[1], comm_coord(2) * dim[2], comm_coord(3) * dim[3], dim[4]},
       globalDim3(comm_dim(3) * this->dim[3]),
       dagger(dagger),
       xpay(xpay),
diff --git a/include/index_helper.cuh b/include/index_helper.cuh
index 26ea377940..4508098d47 100644
--- a/include/index_helper.cuh
+++ b/include/index_helper.cuh
@@ -230,8 +230,9 @@ namespace quda {
 
   template <int nDim>
   struct Coord {
-    int x[nDim]; // nDim lattice coordinates
-    int gx[nDim]; // nDim global lattice coordinates
+    array<int, nDim> x = {};    // nDim lattice coordinates
+    array<int, nDim> gx = {};   // nDim global lattice coordinates
+    array<int, nDim> gDim = {}; // global lattice dimensions
     int x_cb;    // checkerboard lattice site index
     int s;       // fifth dimension coord
     int X;       // full lattice site index

From 2893912e7cd0c2ccbd59dc169492076928a2ca98 Mon Sep 17 00:00:00 2001
From: maddyscientist <mclark@nvidia.com>
Date: Tue, 2 Jul 2024 16:43:14 -0700
Subject: [PATCH 065/108] Renable DD tests in dslash_ctest, and apply various
 cleanup.  Different DD tests are now named.

---
 include/domain_decomposition.h     |  32 +++----
 include/reference_wrapper_helper.h |   8 +-
 lib/color_spinor_project_dd.cu     |   1 -
 tests/dslash_ctest.cpp             |  48 ++++++----
 tests/dslash_test_utils.h          | 138 ++++++++++++++++-------------
 5 files changed, 129 insertions(+), 98 deletions(-)

diff --git a/include/domain_decomposition.h b/include/domain_decomposition.h
index cf88ba5070..aaca447d91 100644
--- a/include/domain_decomposition.h
+++ b/include/domain_decomposition.h
@@ -20,12 +20,12 @@ namespace quda
   // Params for domain decompation
   struct DDParam {
 
-    QudaDDType type;
-    bool flags[(int)DD::size];  // the default value of all flags is 0
-    int blockDim[QUDA_MAX_DIM]; // the size of the block per direction
+    QudaDDType type = QUDA_DD_NO;
+    array<bool, static_cast<int>(DD::size)> flags = {}; // the default value of all flags is 0
+    array<int, QUDA_MAX_DIM> block_dim = {};            // the size of the block per direction
 
     // Default constructor
-    DDParam() : type(QUDA_DD_NO), flags {0}, blockDim {0} { }
+    DDParam() = default;
 
     // returns false if in use
     constexpr bool operator!() const { return type == QUDA_DD_NO; }
@@ -54,7 +54,7 @@ namespace quda
     }
 
     // Pretty print the args struct
-    void print()
+    void print() const
     {
       if (not *this) {
         printfQuda("DD not in use\n");
@@ -63,7 +63,7 @@ namespace quda
       printfQuda("Printing DDParam\n");
       for (int i = 0; i < (int)DD::size; i++)
         printfQuda("flags[DD::%s] = %s\n", to_string((DD)i).c_str(), flags[i] ? "true" : "false");
-      for (int i = 0; i < QUDA_MAX_DIM; i++) printfQuda("blockDim[%d] = %d\n", i, static_cast<int>(blockDim[i]));
+      for (int i = 0; i < QUDA_MAX_DIM; i++) printfQuda("block_dim[%d] = %d\n", i, static_cast<int>(block_dim[i]));
     }
 
     // Checks if this matches to given DDParam
@@ -73,17 +73,17 @@ namespace quda
 
       if (type == QUDA_DD_RED_BLACK) {
         for (int i = 0; i < field.Ndim(); i++) {
-          if (blockDim[i] <= 0) {
-            if (verbose) printfQuda("blockDim[%d] = %d is not positive \n", i, blockDim[i]);
+          if (block_dim[i] <= 0) {
+            if (verbose) printfQuda("block_dim[%d] = %d is not positive \n", i, block_dim[i]);
             return false;
           }
           int globalDim = comm_dim(i) * field.X(i);
-          if (globalDim % blockDim[i] != 0) {
-            if (verbose) printfQuda("blockDim[%d] = %d does not divide %d \n", i, blockDim[i], globalDim);
+          if (globalDim % block_dim[i] != 0) {
+            if (verbose) printfQuda("block_dim[%d] = %d does not divide %d \n", i, block_dim[i], globalDim);
             return false;
           }
-          if ((globalDim / blockDim[i]) % 2 != 0) {
-            if (verbose) printfQuda("blockDim[%d] = %d does not divide %d **evenly** \n", i, blockDim[i], globalDim);
+          if ((globalDim / block_dim[i]) % 2 != 0) {
+            if (verbose) printfQuda("block_dim[%d] = %d does not divide %d **evenly** \n", i, block_dim[i], globalDim);
             return false;
           }
         }
@@ -106,8 +106,8 @@ namespace quda
 
       if (type == QUDA_DD_RED_BLACK) {
         for (int i = 0; i < QUDA_MAX_DIM; i++)
-          if (blockDim[i] != dd.blockDim[i]) {
-            if (verbose) printfQuda("blockDim[%d] = %d != %d \n", i, blockDim[i], dd.blockDim[i]);
+          if (block_dim[i] != dd.block_dim[i]) {
+            if (verbose) printfQuda("block_dim[%d] = %d != %d \n", i, block_dim[i], dd.block_dim[i]);
             return false;
           }
         if (is(DD::no_block_hopping) != dd.is(DD::no_block_hopping)) {
@@ -132,10 +132,10 @@ namespace quda
       for (int i = 0; i < (int)DD::size; i++)
         if (flags[i] != dd.flags[i]) return false;
 
-      // checking blockDim matches when needed
+      // checking block_dim matches when needed
       if (type == QUDA_DD_RED_BLACK)
         for (int i = 0; i < QUDA_MAX_DIM; i++)
-          if (blockDim[i] != dd.blockDim[i]) return false;
+          if (block_dim[i] != dd.block_dim[i]) return false;
 
       return true;
     }
diff --git a/include/reference_wrapper_helper.h b/include/reference_wrapper_helper.h
index 7a4d549990..5db8696667 100644
--- a/include/reference_wrapper_helper.h
+++ b/include/reference_wrapper_helper.h
@@ -473,11 +473,15 @@ namespace quda
     }
 
     template <class U = T, typename... Args>
-    std::enable_if_t<std::is_same_v<std::remove_const_t<U>, ColorSpinorField>, void> DD(const quda::DD &flag,
-                                                                                        const Args &...args)
+    std::enable_if_t<std::is_same_v<U, ColorSpinorField>, void> DD(const quda::DD &flag, const Args &...args)
     {
       for (auto i = 0u; i < vector::size(); i++) operator[](i).DD(flag, args...);
     }
+
+    template <class U = T, typename... Args> std::enable_if_t<std::is_same_v<U, ColorSpinorField>, void> projectDD()
+    {
+      for (auto i = 0u; i < vector::size(); i++) operator[](i).projectDD();
+    }
   };
 
   template <class T> using cvector_ref = const vector_ref<T>;
diff --git a/lib/color_spinor_project_dd.cu b/lib/color_spinor_project_dd.cu
index 8843a1d2ea..b364e877b1 100644
--- a/lib/color_spinor_project_dd.cu
+++ b/lib/color_spinor_project_dd.cu
@@ -41,7 +41,6 @@ namespace quda
       launch<ProjectDD_, enable_host>(tp, stream, Arg(out));
     }
 
-    long long flops() const { return 0; }
     long long bytes() const { return out.Bytes(); }
   };
 
diff --git a/tests/dslash_ctest.cpp b/tests/dslash_ctest.cpp
index fd8a286e4b..fde827bff1 100644
--- a/tests/dslash_ctest.cpp
+++ b/tests/dslash_ctest.cpp
@@ -7,7 +7,7 @@ int argc_copy;
 char **argv_copy;
 dslash_test_type dtest_type = dslash_test_type::Dslash;
 bool ctest_all_partitions = false;
-bool ctest_domain_decomposition = false; // currently disabled by default due to unresolved issues
+bool ctest_domain_decomposition = true; // currently disabled by default due to unresolved issues
 
 // For googletest names must be non-empty, unique, and may only contain ASCII
 // alphanumeric characters or underscore
@@ -18,10 +18,10 @@ using ::testing::Range;
 using ::testing::TestWithParam;
 using ::testing::Values;
 
-class DslashTest : public ::testing::TestWithParam<::testing::tuple<int, int, int, int>>
+class DslashTest : public ::testing::TestWithParam<::testing::tuple<int, int, int, int, int>>
 {
 protected:
-  ::testing::tuple<int, int, int, int> param;
+  ::testing::tuple<int, int, int, int, int> param;
 
   bool skip()
   {
@@ -87,7 +87,8 @@ class DslashTest : public ::testing::TestWithParam<::testing::tuple<int, int, in
     updateR();
 
     int dd_value = ::testing::get<3>(GetParam());
-    dslash_test_wrapper.init_ctest(argc_copy, argv_copy, prec, recon, dd_value);
+    int dd_color = ::testing::get<4>(GetParam());
+    dslash_test_wrapper.init_ctest(argc_copy, argv_copy, prec, recon, dd_value, dd_color);
     display_test_info(prec, recon);
   }
 
@@ -100,7 +101,6 @@ class DslashTest : public ::testing::TestWithParam<::testing::tuple<int, int, in
 
   static void SetUpTestCase()
   {
-    initQuda(device_ordinal);
     DslashTestWrapper::dtest_type = dtest_type;
   }
 
@@ -110,7 +110,6 @@ class DslashTest : public ::testing::TestWithParam<::testing::tuple<int, int, in
   static void TearDownTestCase()
   {
     DslashTestWrapper::destroy();
-    endQuda();
   }
 };
 
@@ -150,6 +149,7 @@ int main(int argc, char **argv)
   }
 
   initComms(argc, argv, gridsize_from_cmdline);
+  initQuda(device_ordinal);
 
   // The 'SetUp()' method of the Google Test class from which DslashTest
   // in derived has no arguments, but QUDA's implementation requires the
@@ -164,23 +164,34 @@ int main(int argc, char **argv)
 
   int test_rc = RUN_ALL_TESTS();
 
+  endQuda();
   finalizeComms();
   return test_rc;
 }
 
-std::string getdslashtestname(testing::TestParamInfo<::testing::tuple<int, int, int, int>> param)
+std::string getdslashtestname(testing::TestParamInfo<::testing::tuple<int, int, int, int, int>> param)
 {
   const int prec = ::testing::get<0>(param.param);
   const int recon = ::testing::get<1>(param.param);
   const int part = ::testing::get<2>(param.param);
   const int dd = ::testing::get<3>(param.param);
+  const int col = ::testing::get<4>(param.param);
   std::stringstream ss;
-  // std::cout << "getdslashtestname" << get_dslash_str(dslash_type) << "_" << prec_str[prec] << "_r" << recon <<
-  // "_partition" << part << std::endl; ss << get_dslash_str(dslash_type) << "_";
   ss << get_prec_str(getPrecision(prec));
   ss << "_r" << recon;
   ss << "_partition" << part;
-  ss << "_domain_decomposition" << dd;
+  if (dd > 0) {
+    switch (dd) {
+    case 1: ss << "_dd_local"; break;
+    case 2: ss << "_dd_global"; break;
+    }
+    switch (col) {
+    case 0: ss << "_red_red"; break;
+    case 1: ss << "_black_red"; break;
+    case 2: ss << "_red_black"; break;
+    case 3: ss << "_black_black"; break;
+    }
+  }
   return ss.str();
 }
 
@@ -190,17 +201,24 @@ std::string getdslashtestname(testing::TestParamInfo<::testing::tuple<int, int,
 #define N_PARTITIONS 1
 #endif
 
+// regular tests
+INSTANTIATE_TEST_SUITE_P(Regular, DslashTest,
+                         Combine(Range(0, 4),
+                                 ::testing::Values(QUDA_RECONSTRUCT_NO, QUDA_RECONSTRUCT_12, QUDA_RECONSTRUCT_8),
+                                 Range(0, N_PARTITIONS), ::testing::Values(0), ::testing::Values(0)),
+                         getdslashtestname);
+
 #ifdef GPU_DD_DIRAC
 #define N_DD_TESTS 3
-#else
-#define N_DD_TESTS 1
-#endif
 
-INSTANTIATE_TEST_SUITE_P(QUDA, DslashTest,
+// DD tests
+INSTANTIATE_TEST_SUITE_P(DD, DslashTest,
                          Combine(Range(0, 4),
                                  ::testing::Values(QUDA_RECONSTRUCT_NO, QUDA_RECONSTRUCT_12, QUDA_RECONSTRUCT_8),
-                                 Range(0, N_PARTITIONS), Range(0, N_DD_TESTS)),
+                                 Range(0, N_PARTITIONS), Range(1, N_DD_TESTS), Range(0, 4)),
                          getdslashtestname);
 
+#endif
+
 #undef N_PARTITIONS
 #undef N_DD_TESTS
diff --git a/tests/dslash_test_utils.h b/tests/dslash_test_utils.h
index 6a9494b80a..31cef86e9e 100644
--- a/tests/dslash_test_utils.h
+++ b/tests/dslash_test_utils.h
@@ -88,9 +88,11 @@ struct DslashTestWrapper {
   static inline bool test_domain_decomposition = false;
   int num_src = 1;
 
+  static inline int dd_col = 0;
+
   const bool transfer = false;
 
-  void init_ctest(int argc, char **argv, int precision, QudaReconstructType link_recon, int dd_value)
+  void init_ctest(int argc, char **argv, int precision, QudaReconstructType link_recon, int dd_value, int dd_color)
   {
     if (first_time) {
       gauge_param = newQudaGaugeParam();
@@ -118,7 +120,7 @@ struct DslashTestWrapper {
     inv_param.clover_cuda_prec_precondition = cuda_prec;
     inv_param.clover_cuda_prec_refinement_sloppy = cuda_prec;
 
-    init_domain_decomposition(dd_value);
+    init_domain_decomposition(dd_value, dd_color);
 
     init();
   }
@@ -282,13 +284,14 @@ struct DslashTestWrapper {
     inv_param.verbosity = verbosity;
   }
 
-  void init_domain_decomposition(int value)
+  void init_domain_decomposition(int value, int color)
   {
     if (value == 0) {
       test_domain_decomposition = false;
       return;
     }
     test_domain_decomposition = true;
+    dd_col = color;
 
     if (value < 3) {
       dd_red_black = true;
@@ -379,12 +382,21 @@ struct DslashTestWrapper {
   static void destroy()
   {
     for (int dir = 0; dir < 4; dir++)
-      if (hostGauge[dir]) host_free(hostGauge[dir]);
+      if (hostGauge[dir]) {
+        host_free(hostGauge[dir]);
+        hostGauge[dir] = nullptr;
+      }
 
     if (dslash_type == QUDA_CLOVER_WILSON_DSLASH || dslash_type == QUDA_TWISTED_CLOVER_DSLASH
         || dslash_type == QUDA_CLOVER_HASENBUSCH_TWIST_DSLASH) {
-      if (hostClover) host_free(hostClover);
-      if (hostCloverInv) host_free(hostCloverInv);
+      if (hostClover) {
+        host_free(hostClover);
+        hostClover = nullptr;
+      }
+      if (hostCloverInv) {
+        host_free(hostCloverInv);
+        hostCloverInv = nullptr;
+      }
     }
 
     spinor = {};
@@ -397,6 +409,7 @@ struct DslashTestWrapper {
       vp_spinorOut.clear();
       vp_spinorRef.clear();
     }
+    first_time = true;
   }
 
   void dslashRef()
@@ -850,78 +863,68 @@ struct DslashTestWrapper {
 
     } else if (test_domain_decomposition) {
 
+      vector_ref<ColorSpinorField> spinor = cudaSpinor;
+      vector_ref<ColorSpinorField> tmp = cudaSpinorTmp;
+      vector_ref<ColorSpinorField> tmp2 = cudaSpinorTmp2;
+      vector_ref<ColorSpinorField> tmp3 = cudaSpinorTmp3;
+
       if (dd_red_black) {
         for (int n = 0; n < Nsrc; n++) {
           for (auto i = 0u; i < 4; i++) {
-            cudaSpinor[n].DD().blockDim[i] = dd_block_size[i];
-            cudaSpinorTmp[n].DD().blockDim[i] = dd_block_size[i];
-            cudaSpinorTmp2[n].DD().blockDim[i] = dd_block_size[i];
-            cudaSpinorTmp3[n].DD().blockDim[i] = dd_block_size[i];
+            cudaSpinor[n].DD().block_dim[i] = dd_block_size[i];
+            cudaSpinorTmp[n].DD().block_dim[i] = dd_block_size[i];
+            cudaSpinorTmp2[n].DD().block_dim[i] = dd_block_size[i];
+            cudaSpinorTmp3[n].DD().block_dim[i] = dd_block_size[i];
           }
         }
 
         blas::zero(cudaSpinorOut);
+        blas::zero(cudaSpinorTmp);
 
-        // We test that applying all 4 possible operators Drr, Drb, Dbr, Dbb gived D
-        for (int col = 0; col < 4; col++) {
+        spinor.DD(DD::reset, DD::red_black_type, dd_col % 2 == 0 ? DD::red_active : DD::black_active);
+        tmp.DD(DD::reset, DD::red_black_type, dd_col / 2 == 0 ? DD::red_active : DD::black_active);
 
-          for (int n = 0; n < num_src; n++) {
-            cudaSpinor[n].DD(DD::reset, DD::red_black_type, col % 2 == 0 ? DD::red_active : DD::black_active);
-            cudaSpinorTmp[n].DD(DD::reset, DD::red_black_type, col / 2 == 0 ? DD::red_active : DD::black_active);
-          }
-
-          switch (dtest_type) {
-          case dslash_test_type::Dslash: dirac->Dslash(cudaSpinorTmp, cudaSpinor, parity); break;
-          case dslash_test_type::MatPC:
-          case dslash_test_type::Mat: dirac->M(cudaSpinorTmp, cudaSpinor); break;
-          case dslash_test_type::MatPCDagMatPC:
-          case dslash_test_type::MatDagMat: dirac->MdagM(cudaSpinorTmp, cudaSpinor); break;
-          default:
-            errorQuda("Test type %s not support for current Dslash", get_string(dtest_type_map, dtest_type).c_str());
-          }
+        switch (dtest_type) {
+        case dslash_test_type::Dslash: dirac->Dslash(cudaSpinorTmp, cudaSpinor, parity); break;
+        case dslash_test_type::MatPC:
+        case dslash_test_type::Mat: dirac->M(cudaSpinorTmp, cudaSpinor); break;
+        case dslash_test_type::MatPCDagMatPC:
+        case dslash_test_type::MatDagMat: dirac->MdagM(cudaSpinorTmp, cudaSpinor); break;
+        default:
+          errorQuda("Test type %s not support for current Dslash", get_string(dtest_type_map, dtest_type).c_str());
+        }
 
-          for (int n = 0; n < num_src; n++) {
-            cudaSpinor[n].DD(DD::reset);
-            cudaSpinorTmp[n].DD(DD::reset);
+        spinor.DD(DD::reset);
+        tmp.DD(DD::reset);
 
-            // We also test that Dyx is same as D applied to projected in and out spinors
-            cudaSpinorTmp2[n] = cudaSpinor[n];
-            cudaSpinorTmp2[n].DD(DD::reset, DD::red_black_type, col % 2 == 0 ? DD::red_active : DD::black_active);
-            cudaSpinorTmp2[n].projectDD();
-            cudaSpinorTmp2[n].DD(DD::reset);
-          }
+        // We also test that Dyx is same as D applied to projected in and out spinors
+        blas::copy(tmp2, cudaSpinor);
+        tmp2.DD(DD::reset, DD::red_black_type, dd_col % 2 == 0 ? DD::red_active : DD::black_active);
+        tmp2.projectDD();
+        tmp2.DD(DD::reset);
 
-          switch (dtest_type) {
-          case dslash_test_type::Dslash: dirac->Dslash(cudaSpinorTmp3, cudaSpinorTmp2, parity); break;
-          case dslash_test_type::MatPC:
-          case dslash_test_type::Mat: dirac->M(cudaSpinorTmp3, cudaSpinorTmp2); break;
-          case dslash_test_type::MatPCDagMatPC:
-          case dslash_test_type::MatDagMat: dirac->MdagM(cudaSpinorTmp3, cudaSpinorTmp2); break;
-          default:
-            errorQuda("Test type %s not support for current Dslash", get_string(dtest_type_map, dtest_type).c_str());
-          }
+        switch (dtest_type) {
+        case dslash_test_type::Dslash: dirac->Dslash(cudaSpinorTmp3, cudaSpinorTmp2, parity); break;
+        case dslash_test_type::MatPC:
+        case dslash_test_type::Mat: dirac->M(cudaSpinorTmp3, cudaSpinorTmp2); break;
+        case dslash_test_type::MatPCDagMatPC:
+        case dslash_test_type::MatDagMat: dirac->MdagM(cudaSpinorTmp3, cudaSpinorTmp2); break;
+        default:
+          errorQuda("Test type %s not support for current Dslash", get_string(dtest_type_map, dtest_type).c_str());
+        }
 
-          for (int n = 0; n < num_src; n++) {
-            cudaSpinorTmp3[n].DD(DD::reset, DD::red_black_type, col / 2 == 0 ? DD::red_active : DD::black_active);
-            cudaSpinorTmp3[n].projectDD();
-            cudaSpinorTmp3[n].DD(DD::reset);
-          }
+        tmp3.DD(DD::reset, DD::red_black_type, dd_col / 2 == 0 ? DD::red_active : DD::black_active);
+        tmp3.projectDD();
+        tmp3.DD(DD::reset);
 
-          if (dd_test_projection)
-            blas::xpy(cudaSpinorTmp3, cudaSpinorOut);
-          else
-            blas::xpy(cudaSpinorTmp, cudaSpinorOut);
+        if (dd_test_projection)
+          blas::xpy(cudaSpinorTmp3, cudaSpinorOut);
+        else
+          blas::xpy(cudaSpinorTmp, cudaSpinorOut);
 
-          spinorTmp = cudaSpinorTmp;
-          spinorOut = cudaSpinorTmp3;
+        spinorTmp = cudaSpinorTmp;
+        spinorOut = cudaSpinorTmp3;
 
-          for (int n = 0; n < Nsrc; n++) {
-            double deviation = std::pow(10, -(double)(ColorSpinorField::Compare(spinorTmp[n], spinorOut[n])));
-            printfQuda("Deviation for (D-PDP)_{%d,%d}*spinor is %e\n", col % 2, col / 2, deviation);
-            double tol = getTolerance(cuda_prec);
-            EXPECT_LE(deviation, tol) << "Projected Dirac and project spinors do not agree";
-          }
-        }
       } else {
         errorQuda("Test dd type not supported");
       }
@@ -1141,7 +1144,7 @@ struct DslashTestWrapper {
 
   void run_test(int niter, bool = false)
   {
-    {
+    if (getTuning()) {
       printfQuda("Tuning...\n");
       dslashCUDA(1); // warm-up run
     }
@@ -1157,7 +1160,7 @@ struct DslashTestWrapper {
     unsigned long long bytes = (quda::Tunable::bytes_global() - bytes0);
 
     if (!test_split_grid) {
-      if (!transfer) spinorOut = cudaSpinorOut;
+      if (!transfer && !test_domain_decomposition) spinorOut = cudaSpinorOut;
 
       // print timing information
       printfQuda("%fus per kernel call\n", 1e6 * dslash_time.event_time / niter);
@@ -1206,6 +1209,13 @@ struct DslashTestWrapper {
         deviation
           = std::max(deviation, std::pow(10, -(double)(ColorSpinorField::Compare(spinorRef[0], vp_spinorOut[n]))));
       }
+    } else if (test_domain_decomposition) {
+      for (int n = 0; n < Nsrc; n++) {
+        auto deviation = std::pow(10, -(double)(ColorSpinorField::Compare(spinorTmp[n], spinorOut[n])));
+        printfQuda("Deviation for (D-PDP)_{%d,%d}*spinor is %e\n", dd_col % 2, dd_col / 2, deviation);
+        double tol = getTolerance(cuda_prec);
+        EXPECT_LE(deviation, tol) << "Projected Dirac and project spinors do not agree";
+      }
     } else {
       for (int n = 0; n < Nsrc; n++) {
         auto norm_cpu = blas::norm2(spinorRef[n]);

From bfb4a3f2ba5f5fe4f7fcb013d00d79cd8913e375 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Mon, 8 Jul 2024 07:55:07 +0200
Subject: [PATCH 066/108] Replacing missed blockDim

---
 include/domain_decomposition_helper.cuh | 10 +++++-----
 lib/inv_mr_quda.cpp                     |  6 +++---
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/include/domain_decomposition_helper.cuh b/include/domain_decomposition_helper.cuh
index 49ad045630..35a09a5ed7 100644
--- a/include/domain_decomposition_helper.cuh
+++ b/include/domain_decomposition_helper.cuh
@@ -34,13 +34,13 @@ namespace quda
   // Red-black Block DD
   struct DDRedBlack {
 
-    const int_fastdiv blockDim[QUDA_MAX_DIM]; // the size of the block per direction
+    const int_fastdiv block_dim[QUDA_MAX_DIM]; // the size of the block per direction
     const bool red_active;         // if red blocks are active
     const bool black_active;       // if black blocks are active
     const bool block_hopping;      // if hopping between red and black is allowed
 
     DDRedBlack(const DDParam &dd) :
-      blockDim {dd.blockDim[0], dd.blockDim[1], dd.blockDim[2], dd.blockDim[3]},
+      block_dim {dd.block_dim[0], dd.block_dim[1], dd.block_dim[2], dd.block_dim[3]},
       red_active(dd.type == QUDA_DD_NO or dd.is(DD::red_active)),
       black_active(dd.type == QUDA_DD_NO or dd.is(DD::black_active)),
       block_hopping(dd.type == QUDA_DD_NO or not dd.is(DD::no_block_hopping))
@@ -55,7 +55,7 @@ namespace quda
     {
       if (not red_active and not black_active) return false;
       if (not dd.red_active and not dd.black_active) return false;
-      if (arg.dim[d] % blockDim[d] == 0) {
+      if (arg.dim[d] % block_dim[d] == 0) {
         if (not red_active and not dd.red_active) return false;
         if (not black_active and not dd.black_active) return false;
         if (not block_hopping and not dd.block_hopping) return false;
@@ -67,7 +67,7 @@ namespace quda
     template <typename Coord> constexpr bool block_parity(const Coord &x) const
     {
       int block_parity = 0;
-      for (int i = 0; i < x.size(); i++) { block_parity += x.gx[i] / blockDim[i]; }
+      for (int i = 0; i < x.size(); i++) { block_parity += x.gx[i] / block_dim[i]; }
       return block_parity % 2 == 1;
     }
 
@@ -76,7 +76,7 @@ namespace quda
       int x_mu = x.gx[mu] + dir;
       if (x_mu < 0) x_mu += x.gDim[mu];
       if (x_mu >= x.gDim[mu]) x_mu -= x.gDim[mu];
-      return x.gx[mu] / blockDim[mu] != x_mu / blockDim[mu];
+      return x.gx[mu] / block_dim[mu] != x_mu / block_dim[mu];
     }
 
     template <typename Coord> constexpr bool isZero(const Coord &x) const
diff --git a/lib/inv_mr_quda.cpp b/lib/inv_mr_quda.cpp
index 1f4422adf9..7b185d9ae7 100644
--- a/lib/inv_mr_quda.cpp
+++ b/lib/inv_mr_quda.cpp
@@ -28,11 +28,11 @@ namespace quda
       ColorSpinorParam csParam(b);
       csParam.create = QUDA_NULL_FIELD_CREATE;
 
-      // Setting the value of blockDim and checking if blocks are local
+      // Setting the value of block_dim and checking if blocks are local
       bool local = true;
       for (int i = 0; i < QUDA_MAX_DIM; i++) {
-        csParam.dd.blockDim[i] = param.schwarz_block[i];
-        local &= (param.do_block_schwarz() && x.full_dim(i) % csParam.dd.blockDim[i] == 0);
+        csParam.dd.block_dim[i] = param.schwarz_block[i];
+        local &= (param.do_block_schwarz() && x.full_dim(i) % csParam.dd.block_dim[i] == 0);
       }
       // Disabling global_reduction if blocks are local and we do block_schwarz
       if (param.do_block_schwarz() and param.global_reduction) param.global_reduction = not local;

From 16c533b4422329e084f25b3dff30adfaa842a61c Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Mon, 8 Jul 2024 09:45:06 +0200
Subject: [PATCH 067/108] Adding full_dim

---
 include/reference_wrapper_helper.h | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/include/reference_wrapper_helper.h b/include/reference_wrapper_helper.h
index 5db8696667..56d9a9643b 100644
--- a/include/reference_wrapper_helper.h
+++ b/include/reference_wrapper_helper.h
@@ -379,6 +379,15 @@ namespace quda
       return operator[](0).X(d);
     }
 
+    template <class U = T>
+    std::enable_if_t<std::is_same_v<std::remove_const_t<U>, ColorSpinorField>, int> full_dim(int d) const
+    {
+      for (auto i = 1u; i < vector::size(); i++)
+        if (operator[](i - 1).full_dim(d) != operator[](i).full_dim(d))
+          errorQuda("Dimension %d does not match %d != %d", d, operator[](i - 1).full_dim(d), operator[](i).full_dim(d));
+      return operator[](0).full_dim(d);
+    }
+
     template <class U = T>
     std::enable_if_t<std::is_same_v<std::remove_const_t<U>, ColorSpinorField>, size_t> Length() const
     {

From 6c7bf6396cb4aec08c74f877b351cafda17ca188 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Mon, 8 Jul 2024 09:45:23 +0200
Subject: [PATCH 068/108] Checking against the full lattice size

---
 include/domain_decomposition.h | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/include/domain_decomposition.h b/include/domain_decomposition.h
index aaca447d91..5ce25c1e32 100644
--- a/include/domain_decomposition.h
+++ b/include/domain_decomposition.h
@@ -77,7 +77,7 @@ namespace quda
             if (verbose) printfQuda("block_dim[%d] = %d is not positive \n", i, block_dim[i]);
             return false;
           }
-          int globalDim = comm_dim(i) * field.X(i);
+          int globalDim = comm_dim(i) * field.full_dim(i);
           if (globalDim % block_dim[i] != 0) {
             if (verbose) printfQuda("block_dim[%d] = %d does not divide %d \n", i, block_dim[i], globalDim);
             return false;
@@ -87,6 +87,10 @@ namespace quda
             return false;
           }
         }
+        if (block_dim[0] % 2) {
+          if (verbose) printfQuda("block_dim[0] = %d must be even \n", block_dim[0]);
+          return false;
+        }
       }
 
       return true;

From 14dea268242e984667bb3b9e26d1986619a74203 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Mon, 8 Jul 2024 09:45:39 +0200
Subject: [PATCH 069/108] Adding dedicated tests for DD

---
 tests/CMakeLists.txt   | 88 +++++++++++++++++++++++++++---------------
 tests/dslash_ctest.cpp |  4 +-
 2 files changed, 58 insertions(+), 34 deletions(-)

diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 4d18cec933..09d4238f09 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -417,23 +417,24 @@ foreach(pol IN LISTS DSLASH_POLICIES)
                      --dslash-type ${DIRAC_NAME}
                      --all-partitions 1
                      --test MatPCDagMatPC
-                     --dim 2 4 6 8
+                     --dim 4 2 6 8
                      --gtest_output=xml:dslash_${DIRAC_NAME}_test_pol${pol2}.xml)
     if(polenv)
       set_tests_properties(dslash_${DIRAC_NAME}_policy${pol2} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol})
     endif()
 
-    add_test(NAME dslash_${DIRAC_NAME}_splitgrid_policy${pol2}
+    add_test(NAME dslash_${DIRAC_NAME}_splitgrid_policy${pol2}_DD
              COMMAND ${QUDA_CTEST_LAUNCH} $<TARGET_FILE:dslash_ctest> ${MPIEXEC_POSTFLAGS}
                      --dslash-type ${DIRAC_NAME}
                      --all-partitions 0
+		     --domain-decomposition 1
                      --test Dslash
-                     --dim 2 4 6 8
-                     --gtest_output=xml:dslash_${DIRAC_NAME}_splitgrid_test_pol${pol2}.xml)
+                     --dim 4 2 6 8
+                     --gtest_output=xml:dslash_${DIRAC_NAME}_splitgrid_test_pol${pol2}_DD.xml)
     if(polenv)
-      set_tests_properties(dslash_${DIRAC_NAME}_splitgrid_policy${pol2} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol})
+      set_tests_properties(dslash_${DIRAC_NAME}_splitgrid_policy${pol2}_DD PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol})
     endif()
-    set_tests_properties(dslash_${DIRAC_NAME}_splitgrid_policy${pol2} PROPERTIES ENVIRONMENT QUDA_TEST_GRID_PARTITION=$ENV{QUDA_TEST_GRID_SIZE})
+    set_tests_properties(dslash_${DIRAC_NAME}_splitgrid_policy${pol2}_DD PROPERTIES ENVIRONMENT QUDA_TEST_GRID_PARTITION=$ENV{QUDA_TEST_GRID_SIZE})
 
     add_test(NAME benchmark_dslash_${DIRAC_NAME}_policy${pol2}
              COMMAND ${QUDA_CTEST_LAUNCH} $<TARGET_FILE:dslash_ctest> ${MPIEXEC_POSTFLAGS}
@@ -447,7 +448,20 @@ foreach(pol IN LISTS DSLASH_POLICIES)
     if(polenv)
       set_tests_properties(benchmark_dslash_${DIRAC_NAME}_policy${pol2} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol})
     endif()
-endif()
+    
+    set(TEST_NAME dslash_${DIRAC_NAME}_mat_policy${pol2}_DD)
+    add_test(NAME ${TEST_NAME}
+             COMMAND ${QUDA_CTEST_LAUNCH} $<TARGET_FILE:dslash_ctest> ${MPIEXEC_POSTFLAGS}
+                     --dslash-type ${DIRAC_NAME}
+                     --all-partitions 0
+                     --test Mat
+		     --domain-decomposition 1
+                     --dim 4 2 6 8
+                     --gtest_output=xml:${TEST_NAME}.xml)
+    if(polenv)
+      set_tests_properties(${TEST_NAME} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol})
+    endif()    
+  endif()
 
   if(QUDA_DIRAC_CLOVER)
     set(DIRAC_NAME clover)
@@ -458,20 +472,19 @@ endif()
                      --dslash-type ${DIRAC_NAME}
                      --test MatPCDagMatPC
                      --matpc even-even
-                     --dim 2 4 6 8
+                     --dim 4 2 6 8
                      --gtest_output=xml:dslash_${DIRAC_NAME}_sym_test_pol${pol2}.xml)
     if(polenv)
       set_tests_properties(dslash_${DIRAC_NAME}_sym_policy${pol2} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol})
     endif()
 
     # asymmetric preconditioning
-    set(DIRAC_NAME clover)
     add_test(NAME dslash_${DIRAC_NAME}_asym_policy${pol2}
              COMMAND ${QUDA_CTEST_LAUNCH} $<TARGET_FILE:dslash_ctest> ${MPIEXEC_POSTFLAGS}
                      --dslash-type ${DIRAC_NAME}
                      --test MatPCDagMatPC
                      --matpc even-even-asym
-                     --dim 2 4 6 8
+                     --dim 4 2 6 8
                      --gtest_output=xml:dslash_${DIRAC_NAME}_asym_test_pol${pol2}.xml)
     if(polenv)
       set_tests_properties(dslash_${DIRAC_NAME}_asym_policy${pol2} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol})
@@ -489,6 +502,17 @@ endif()
       set_tests_properties(benchmark_dslash_${DIRAC_NAME}_policy${pol2} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol})
     endif()
 
+    set(TEST_NAME dslash_${DIRAC_NAME}_mat_policy${pol2}_DD)
+    add_test(NAME ${TEST_NAME}
+             COMMAND ${QUDA_CTEST_LAUNCH} $<TARGET_FILE:dslash_ctest> ${MPIEXEC_POSTFLAGS}
+                     --dslash-type ${DIRAC_NAME}
+                     --test Mat
+		     --domain-decomposition 1
+                     --dim 4 2 6 8
+                     --gtest_output=xml:${TEST_NAME}.xml)
+    if(polenv)
+      set_tests_properties(${TEST_NAME} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol})
+    endif()    
   endif()
 
   if(QUDA_DIRAC_CLOVER_HASENBUSCH)
@@ -500,7 +524,7 @@ endif()
                      --dslash-type ${DIRAC_NAME}
                      --test MatPCDagMatPC
                      --matpc even-even
-                     --dim 2 4 6 8
+                     --dim 4 2 6 8
                      --gtest_output=xml:dslash_${DIRAC_NAME}_sym_test_pol${pol2}.xml)
     if(polenv)
       set_tests_properties(dslash_${DIRAC_NAME}_sym_policy${pol2} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol})
@@ -512,7 +536,7 @@ endif()
                      --dslash-type ${DIRAC_NAME}
                      --test MatPCDagMatPC
                      --matpc even-even-asym
-                     --dim 2 4 6 8
+                     --dim 4 2 6 8
                      --gtest_output=xml:dslash_${DIRAC_NAME}_asym_test_pol${pol2}.xml)
     if(polenv)
       set_tests_properties(dslash_${DIRAC_NAME}_asym_policy${pol2} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol})
@@ -541,7 +565,7 @@ endif()
                      --dslash-type ${DIRAC_NAME}
                      --test MatPCDagMatPC
                      --matpc even-even
-                     --dim 2 4 6 8
+                     --dim 4 2 6 8
                      --gtest_output=xml:dslash_${DIRAC_NAME}_sym_test_pol${pol2}.xml)
     if(polenv)
       set_tests_properties(dslash_${DIRAC_NAME}_sym_policy${pol2} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol})
@@ -553,7 +577,7 @@ endif()
                      --dslash-type ${DIRAC_NAME}
                      --test MatPCDagMatPC
                      --matpc even-even-asym
-                     --dim 2 4 6 8
+                     --dim 4 2 6 8
                      --gtest_output=xml:dslash_${DIRAC_NAME}_asym_test_pol${pol2}.xml)
     if(polenv)
       set_tests_properties(dslash_${DIRAC_NAME}_asym_policy${pol2} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol})
@@ -585,7 +609,7 @@ endif()
                      --test MatPCDagMatPC
                      --matpc even-even
                      --flavor nondeg-doublet
-                     --dim 2 4 6 8
+                     --dim 4 2 6 8
         --gtest_output=xml:dslash_ndeg_${DIRAC_NAME}_sym_test_pol${pol2}.xml)
     if(polenv)
       set_tests_properties(dslash_ndeg_${DIRAC_NAME}_sym_policy${pol2}
@@ -599,7 +623,7 @@ endif()
                      --test MatPCDagMatPC
                      --matpc even-even-asym
                      --flavor nondeg-doublet
-                     --dim 2 4 6 8
+                     --dim 4 2 6 8
         --gtest_output=xml:dslash_ndeg_${DIRAC_NAME}_asym_test_pol${pol2}.xml)
     if(polenv)
       set_tests_properties(dslash_ndeg_${DIRAC_NAME}_asym_policy${pol2}
@@ -628,7 +652,7 @@ endif()
                      --dslash-type ${DIRAC_NAME}
                      --test MatPCDagMatPC
                      --matpc even-even
-                     --dim 2 4 6 8
+                     --dim 4 2 6 8
                      --gtest_output=xml:dslash_${DIRAC_NAME}_sym_test_pol${pol2}.xml)
     if(polenv)
       set_tests_properties(dslash_${DIRAC_NAME}_sym_policy${pol2}
@@ -641,7 +665,7 @@ endif()
                      --dslash-type ${DIRAC_NAME}
                      --test MatPCDagMatPC
                      --matpc even-even-asym
-                     --dim 2 4 6 8
+                     --dim 4 2 6 8
                      --gtest_output=xml:dslash_${DIRAC_NAME}_asym_test_pol${pol2}.xml)
     if(polenv)
       set_tests_properties(dslash_${DIRAC_NAME}_asym_policy${pol2}
@@ -674,7 +698,7 @@ endif()
                      --flavor nondeg-doublet
                      --test MatPCDagMatPC
                      --matpc even-even
-                     --dim 2 4 6 8
+                     --dim 4 2 6 8
                      --gtest_output=xml:dslash_ndeg_${DIRAC_NAME}_sym_test_pol${pol2}.xml)
     if(polenv)
       set_tests_properties(dslash_ndeg_${DIRAC_NAME}_sym_policy${pol2}
@@ -688,7 +712,7 @@ endif()
                      --flavor nondeg-doublet
                      --test MatPCDagMatPC
                      --matpc even-even-asym
-                     --dim 2 4 6 8
+                     --dim 4 2 6 8
                      --gtest_output=xml:dslash_ndeg_${DIRAC_NAME}_asym_test_pol${pol2}.xml)
     if(polenv)
       set_tests_properties(dslash_ndeg_${DIRAC_NAME}_asym_policy${pol2}
@@ -719,7 +743,7 @@ endif()
                      --dslash-type ${DIRAC_NAME}
                      --test MatPCDagMatPC
                      --matpc even-even
-                     --dim 2 4 6 8
+                     --dim 4 2 6 8
                      --Lsdim 4
         --gtest_output=xml:dslash_${DIRAC_NAME}_test_pol${pol2}.xml)
     if(polenv)
@@ -748,7 +772,7 @@ endif()
                      --dslash-type ${DIRAC_NAME}
                      --test MatPCDagMatPC
                      --matpc even-even
-                     --dim 2 4 6 8
+                     --dim 4 2 6 8
                      --Lsdim 4
         --gtest_output=xml:dslash_${DIRAC_NAME}_sym_test_pol${pol2}.xml)
     if(polenv)
@@ -761,7 +785,7 @@ endif()
                      --dslash-type ${DIRAC_NAME}
                      --test MatPCDagMatPC
                      --matpc even-even-asym
-                     --dim 2 4 6 8
+                     --dim 4 2 6 8
                      --Lsdim 4
         --gtest_output=xml:dslash_${DIRAC_NAME}_asym_test_pol${pol2}.xml)
     if(polenv)
@@ -790,7 +814,7 @@ endif()
                      --dslash-type ${DIRAC_NAME}
                      --test MatPCDagMatPC
                      --matpc even-even
-                     --dim 2 4 6 8
+                     --dim 4 2 6 8
                      --Lsdim 4
                      --gtest_output=xml:dslash_${DIRAC_NAME}_sym_test_pol${pol2}.xml)
     if(polenv)
@@ -802,7 +826,7 @@ endif()
                      --dslash-type ${DIRAC_NAME}
                      --test MatPCDagMatPC
                      --matpc even-even-asym
-                     --dim 2 4 6 8
+                     --dim 4 2 6 8
                      --Lsdim 4
                      --gtest_output=xml:dslash_${DIRAC_NAME}_asym_test_pol${pol2}.xml)
     if(polenv)
@@ -834,7 +858,7 @@ endif()
                        --dslash-type ${DIRAC_NAME}
                        --test MatPCDagMatPCLocal
                        --matpc even-even
-                       --dim 2 4 6 8
+                       --dim 4 2 6 8
                        --Lsdim 4
                        --gtest_output=xml:dslash_${DIRAC_NAME}_local{pol2}.xml)
     endif()
@@ -847,7 +871,7 @@ endif()
                      --dslash-type ${DIRAC_NAME}
                      --test MatPCDagMatPC
                      --matpc even-even
-                     --dim 2 4 6 8
+                     --dim 4 2 6 8
                      --Lsdim 4
         --gtest_output=xml:dslash_${DIRAC_NAME}_sym_test_pol${pol2}.xml)
     if(polenv)
@@ -859,7 +883,7 @@ endif()
                      --dslash-type ${DIRAC_NAME}
                      --test MatPCDagMatPC
                      --matpc even-even-asym
-                     --dim 2 4 6 8
+                     --dim 4 2 6 8
                      --Lsdim 4
                      --gtest_output=xml:dslash_${DIRAC_NAME}_asym_test_pol${pol2}.xml)
  
@@ -874,7 +898,7 @@ endif()
              COMMAND ${QUDA_CTEST_LAUNCH} $<TARGET_FILE:staggered_dslash_ctest> ${MPIEXEC_POSTFLAGS}
                      --dslash-type ${DIRAC_NAME}
                      --test MatPC
-                     --dim 2 4 6 8
+                     --dim 4 2 6 8
                      --gtest_output=xml:dslash_${DIRAC_NAME}_matpc_test_pol${pol2}.xml)
     if(polenv)
       set_tests_properties(dslash_${DIRAC_NAME}_matpc_policy${pol2} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol2})
@@ -884,7 +908,7 @@ endif()
              COMMAND ${QUDA_CTEST_LAUNCH} $<TARGET_FILE:staggered_dslash_ctest> ${MPIEXEC_POSTFLAGS}
                      --dslash-type ${DIRAC_NAME}
                      --test Mat
-                     --dim 2 4 6 8
+                     --dim 4 2 6 8
                      --gtest_output=xml:dslash_${DIRAC_NAME}_mat_test_pol${pol2}.xml)
     if(polenv)
       set_tests_properties(dslash_${DIRAC_NAME}_mat_policy${pol2} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol2})
@@ -894,7 +918,7 @@ endif()
              COMMAND ${QUDA_CTEST_LAUNCH} $<TARGET_FILE:staggered_dslash_ctest> ${MPIEXEC_POSTFLAGS}
                      --dslash-type ${DIRAC_NAME}
                      --test MatPC
-                     --dim 2 4 6 8
+                     --dim 4 2 6 8
                      --gtest_output=xml:dslash_${DIRAC_NAME}_matpc_test_pol${pol2}.xml)
     if(polenv)
       set_tests_properties(dslash_${DIRAC_NAME}_splitgrid_policy${pol2} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol2})
@@ -968,7 +992,7 @@ endif()
                COMMAND ${QUDA_CTEST_LAUNCH} $<TARGET_FILE:staggered_dslash_ctest> ${MPIEXEC_POSTFLAGS}
                        --dslash-type ${DIRAC_NAME}
                        --test Mat
-                       --dim 2 4 6 8
+                       --dim 4 2 6 8
                        --gtest_output=xml:dslash_${DIRAC_NAME}_mat_test_pol${pol2}.xml)
       if(polenv)
         set_tests_properties(dslash_${DIRAC_NAME}_mat_policy${pol2} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol2})
diff --git a/tests/dslash_ctest.cpp b/tests/dslash_ctest.cpp
index fde827bff1..b2f6ea27ad 100644
--- a/tests/dslash_ctest.cpp
+++ b/tests/dslash_ctest.cpp
@@ -7,7 +7,7 @@ int argc_copy;
 char **argv_copy;
 dslash_test_type dtest_type = dslash_test_type::Dslash;
 bool ctest_all_partitions = false;
-bool ctest_domain_decomposition = true; // currently disabled by default due to unresolved issues
+bool ctest_domain_decomposition = false;
 
 // For googletest names must be non-empty, unique, and may only contain ASCII
 // alphanumeric characters or underscore
@@ -140,7 +140,7 @@ int main(int argc, char **argv)
   auto app = make_app();
   app->add_option("--test", dtest_type, "Test method")->transform(CLI::CheckedTransformer(dtest_type_map));
   app->add_option("--all-partitions", ctest_all_partitions, "Test all instead of reduced combination of partitions");
-  app->add_option("--test-domain-decomposition", ctest_domain_decomposition, "Test domain decomposition");
+  app->add_option("--domain-decomposition", ctest_domain_decomposition, "Test domain decomposition");
   add_comms_option_group(app);
   try {
     app->parse(argc, argv);

From 807b7359bb2467c28bd995897f4209691348158c Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Mon, 8 Jul 2024 09:58:04 +0200
Subject: [PATCH 070/108] trying to fix compiler error  in CI/CD

---
 include/declare_enum.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/include/declare_enum.h b/include/declare_enum.h
index 175c47722e..b82f4e4c0c 100644
--- a/include/declare_enum.h
+++ b/include/declare_enum.h
@@ -30,7 +30,7 @@
 #include <unordered_map>
 
 // Add the definition of this method into a cpp file. (only the declaration in the header)
-static inline const std::vector<std::string> &get_enum_names(const std::string &en_key, const std::string &en_str)
+static inline const std::vector<std::string> &get_enum_names(std::string en_key, std::string en_str)
 {
   static std::unordered_map<std::string, std::vector<std::string>> en_names_map;
   const auto it = en_names_map.find(en_key);
@@ -55,6 +55,6 @@ static inline const std::vector<std::string> &get_enum_names(const std::string &
   enum class ENUM_NAME : unsigned int { __VA_ARGS__ size };                                                            \
   inline std::string to_string(ENUM_NAME en)                                                                           \
   {                                                                                                                    \
-    const auto &names = get_enum_names(#ENUM_NAME #__VA_ARGS__, #__VA_ARGS__);                                         \
+    const auto &names = get_enum_names(#ENUM_NAME, #__VA_ARGS__);                                                      \
     return names[static_cast<std::size_t>(en)];                                                                        \
   }

From 2fdb22a51a336318a56197eb692d0690d41ebd40 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Mon, 8 Jul 2024 10:13:55 +0200
Subject: [PATCH 071/108] trying to fix compiler error  in CI/CD

---
 include/declare_enum.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/include/declare_enum.h b/include/declare_enum.h
index b82f4e4c0c..69b721b343 100644
--- a/include/declare_enum.h
+++ b/include/declare_enum.h
@@ -30,7 +30,7 @@
 #include <unordered_map>
 
 // Add the definition of this method into a cpp file. (only the declaration in the header)
-static inline const std::vector<std::string> &get_enum_names(std::string en_key, std::string en_str)
+static inline const std::vector<std::string> get_enum_names(const std::string &en_key, const std::string &en_str)
 {
   static std::unordered_map<std::string, std::vector<std::string>> en_names_map;
   const auto it = en_names_map.find(en_key);
@@ -55,6 +55,6 @@ static inline const std::vector<std::string> &get_enum_names(std::string en_key,
   enum class ENUM_NAME : unsigned int { __VA_ARGS__ size };                                                            \
   inline std::string to_string(ENUM_NAME en)                                                                           \
   {                                                                                                                    \
-    const auto &names = get_enum_names(#ENUM_NAME, #__VA_ARGS__);                                                      \
+    const auto names = get_enum_names(#ENUM_NAME, #__VA_ARGS__);                                                       \
     return names[static_cast<std::size_t>(en)];                                                                        \
   }

From d71dd756baa27f15fe78a649ba8417014ec59969 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Mon, 8 Jul 2024 13:39:25 +0200
Subject: [PATCH 072/108] Allowing for zero-valued block_dim, i.e. no DD in
 that direction

---
 include/domain_decomposition.h          | 23 +++++++++++++----------
 include/domain_decomposition_helper.cuh |  5 ++++-
 2 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/include/domain_decomposition.h b/include/domain_decomposition.h
index 5ce25c1e32..24e653ac37 100644
--- a/include/domain_decomposition.h
+++ b/include/domain_decomposition.h
@@ -73,18 +73,21 @@ namespace quda
 
       if (type == QUDA_DD_RED_BLACK) {
         for (int i = 0; i < field.Ndim(); i++) {
-          if (block_dim[i] <= 0) {
-            if (verbose) printfQuda("block_dim[%d] = %d is not positive \n", i, block_dim[i]);
+          if (block_dim[i] < 0) {
+            if (verbose) printfQuda("block_dim[%d] = %d is negative\n", i, block_dim[i]);
             return false;
           }
-          int globalDim = comm_dim(i) * field.full_dim(i);
-          if (globalDim % block_dim[i] != 0) {
-            if (verbose) printfQuda("block_dim[%d] = %d does not divide %d \n", i, block_dim[i], globalDim);
-            return false;
-          }
-          if ((globalDim / block_dim[i]) % 2 != 0) {
-            if (verbose) printfQuda("block_dim[%d] = %d does not divide %d **evenly** \n", i, block_dim[i], globalDim);
-            return false;
+          if (block_dim[i] > 0) {
+            int globalDim = comm_dim(i) * field.full_dim(i);
+            if (globalDim % block_dim[i] != 0) {
+              if (verbose) printfQuda("block_dim[%d] = %d does not divide %d \n", i, block_dim[i], globalDim);
+              return false;
+            }
+            if ((globalDim / block_dim[i]) % 2 != 0) {
+              if (verbose)
+                printfQuda("block_dim[%d] = %d does not divide %d **evenly** \n", i, block_dim[i], globalDim);
+              return false;
+            }
           }
         }
         if (block_dim[0] % 2) {
diff --git a/include/domain_decomposition_helper.cuh b/include/domain_decomposition_helper.cuh
index 35a09a5ed7..fcd036f502 100644
--- a/include/domain_decomposition_helper.cuh
+++ b/include/domain_decomposition_helper.cuh
@@ -67,12 +67,15 @@ namespace quda
     template <typename Coord> constexpr bool block_parity(const Coord &x) const
     {
       int block_parity = 0;
-      for (int i = 0; i < x.size(); i++) { block_parity += x.gx[i] / block_dim[i]; }
+      for (int i = 0; i < x.size(); i++) {
+        if (block_dim[i] > 0) block_parity += x.gx[i] / block_dim[i];
+      }
       return block_parity % 2 == 1;
     }
 
     template <typename Coord> constexpr bool on_border(const Coord &x, int mu, int dir) const
     {
+      if (block_dim[mu] == 0) return false;
       int x_mu = x.gx[mu] + dir;
       if (x_mu < 0) x_mu += x.gDim[mu];
       if (x_mu >= x.gDim[mu]) x_mu -= x.gDim[mu];

From 6315e6f8988de74cd7736ffaff9e14ba7fa4c0ed Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Mon, 8 Jul 2024 13:40:24 +0200
Subject: [PATCH 073/108] Restoring original dim of tests and adding dedicated
 tests for DD

---
 tests/CMakeLists.txt | 169 +++++++++++++++++++++++++++++++++++++------
 1 file changed, 146 insertions(+), 23 deletions(-)

diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 09d4238f09..36885f1ecd 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -417,7 +417,7 @@ foreach(pol IN LISTS DSLASH_POLICIES)
                      --dslash-type ${DIRAC_NAME}
                      --all-partitions 1
                      --test MatPCDagMatPC
-                     --dim 4 2 6 8
+                     --dim 2 4 6 8
                      --gtest_output=xml:dslash_${DIRAC_NAME}_test_pol${pol2}.xml)
     if(polenv)
       set_tests_properties(dslash_${DIRAC_NAME}_policy${pol2} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol})
@@ -460,7 +460,7 @@ foreach(pol IN LISTS DSLASH_POLICIES)
                      --gtest_output=xml:${TEST_NAME}.xml)
     if(polenv)
       set_tests_properties(${TEST_NAME} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol})
-    endif()    
+    endif()
   endif()
 
   if(QUDA_DIRAC_CLOVER)
@@ -472,7 +472,7 @@ foreach(pol IN LISTS DSLASH_POLICIES)
                      --dslash-type ${DIRAC_NAME}
                      --test MatPCDagMatPC
                      --matpc even-even
-                     --dim 4 2 6 8
+                     --dim 2 4 6 8
                      --gtest_output=xml:dslash_${DIRAC_NAME}_sym_test_pol${pol2}.xml)
     if(polenv)
       set_tests_properties(dslash_${DIRAC_NAME}_sym_policy${pol2} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol})
@@ -484,7 +484,7 @@ foreach(pol IN LISTS DSLASH_POLICIES)
                      --dslash-type ${DIRAC_NAME}
                      --test MatPCDagMatPC
                      --matpc even-even-asym
-                     --dim 4 2 6 8
+                     --dim 2 4 6 8
                      --gtest_output=xml:dslash_${DIRAC_NAME}_asym_test_pol${pol2}.xml)
     if(polenv)
       set_tests_properties(dslash_${DIRAC_NAME}_asym_policy${pol2} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol})
@@ -512,7 +512,7 @@ foreach(pol IN LISTS DSLASH_POLICIES)
                      --gtest_output=xml:${TEST_NAME}.xml)
     if(polenv)
       set_tests_properties(${TEST_NAME} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol})
-    endif()    
+    endif()
   endif()
 
   if(QUDA_DIRAC_CLOVER_HASENBUSCH)
@@ -524,7 +524,7 @@ foreach(pol IN LISTS DSLASH_POLICIES)
                      --dslash-type ${DIRAC_NAME}
                      --test MatPCDagMatPC
                      --matpc even-even
-                     --dim 4 2 6 8
+                     --dim 2 4 6 8
                      --gtest_output=xml:dslash_${DIRAC_NAME}_sym_test_pol${pol2}.xml)
     if(polenv)
       set_tests_properties(dslash_${DIRAC_NAME}_sym_policy${pol2} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol})
@@ -536,7 +536,7 @@ foreach(pol IN LISTS DSLASH_POLICIES)
                      --dslash-type ${DIRAC_NAME}
                      --test MatPCDagMatPC
                      --matpc even-even-asym
-                     --dim 4 2 6 8
+                     --dim 2 4 6 8
                      --gtest_output=xml:dslash_${DIRAC_NAME}_asym_test_pol${pol2}.xml)
     if(polenv)
       set_tests_properties(dslash_${DIRAC_NAME}_asym_policy${pol2} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol})
@@ -554,6 +554,17 @@ foreach(pol IN LISTS DSLASH_POLICIES)
       set_tests_properties(benchmark_dslash_${DIRAC_NAME}_policy${pol2} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol})
     endif()
 
+    set(TEST_NAME dslash_${DIRAC_NAME}_mat_policy${pol2}_DD)
+    add_test(NAME ${TEST_NAME}
+             COMMAND ${QUDA_CTEST_LAUNCH} $<TARGET_FILE:dslash_ctest> ${MPIEXEC_POSTFLAGS}
+                     --dslash-type ${DIRAC_NAME}
+                     --test Mat
+		     --domain-decomposition 1
+                     --dim 4 2 6 8
+                     --gtest_output=xml:${TEST_NAME}.xml)
+    if(polenv)
+      set_tests_properties(${TEST_NAME} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol})
+    endif()
   endif()
 
   if(QUDA_DIRAC_TWISTED_MASS)
@@ -565,7 +576,7 @@ foreach(pol IN LISTS DSLASH_POLICIES)
                      --dslash-type ${DIRAC_NAME}
                      --test MatPCDagMatPC
                      --matpc even-even
-                     --dim 4 2 6 8
+                     --dim 2 4 6 8
                      --gtest_output=xml:dslash_${DIRAC_NAME}_sym_test_pol${pol2}.xml)
     if(polenv)
       set_tests_properties(dslash_${DIRAC_NAME}_sym_policy${pol2} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol})
@@ -577,7 +588,7 @@ foreach(pol IN LISTS DSLASH_POLICIES)
                      --dslash-type ${DIRAC_NAME}
                      --test MatPCDagMatPC
                      --matpc even-even-asym
-                     --dim 4 2 6 8
+                     --dim 2 4 6 8
                      --gtest_output=xml:dslash_${DIRAC_NAME}_asym_test_pol${pol2}.xml)
     if(polenv)
       set_tests_properties(dslash_${DIRAC_NAME}_asym_policy${pol2} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol})
@@ -595,6 +606,17 @@ foreach(pol IN LISTS DSLASH_POLICIES)
       set_tests_properties(benchmark_dslash_${DIRAC_NAME}_policy${pol2} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol})
     endif()
 
+    set(TEST_NAME dslash_${DIRAC_NAME}_mat_policy${pol2}_DD)
+    add_test(NAME ${TEST_NAME}
+             COMMAND ${QUDA_CTEST_LAUNCH} $<TARGET_FILE:dslash_ctest> ${MPIEXEC_POSTFLAGS}
+                     --dslash-type ${DIRAC_NAME}
+                     --test Mat
+		     --domain-decomposition 1
+                     --dim 4 2 6 8
+                     --gtest_output=xml:${TEST_NAME}.xml)
+    if(polenv)
+      set_tests_properties(${TEST_NAME} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol})
+    endif()
   endif()
 
   if(QUDA_DIRAC_NDEG_TWISTED_MASS)
@@ -609,7 +631,7 @@ foreach(pol IN LISTS DSLASH_POLICIES)
                      --test MatPCDagMatPC
                      --matpc even-even
                      --flavor nondeg-doublet
-                     --dim 4 2 6 8
+                     --dim 2 4 6 8
         --gtest_output=xml:dslash_ndeg_${DIRAC_NAME}_sym_test_pol${pol2}.xml)
     if(polenv)
       set_tests_properties(dslash_ndeg_${DIRAC_NAME}_sym_policy${pol2}
@@ -623,7 +645,7 @@ foreach(pol IN LISTS DSLASH_POLICIES)
                      --test MatPCDagMatPC
                      --matpc even-even-asym
                      --flavor nondeg-doublet
-                     --dim 4 2 6 8
+                     --dim 2 4 6 8
         --gtest_output=xml:dslash_ndeg_${DIRAC_NAME}_asym_test_pol${pol2}.xml)
     if(polenv)
       set_tests_properties(dslash_ndeg_${DIRAC_NAME}_asym_policy${pol2}
@@ -641,6 +663,19 @@ foreach(pol IN LISTS DSLASH_POLICIES)
     if(polenv)
       set_tests_properties(benchmark_dslash_ndeg_${DIRAC_NAME}_policy${pol2} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol})
     endif()
+
+    set(TEST_NAME dslash_ndeg_${DIRAC_NAME}_mat_policy${pol2}_DD)
+    add_test(NAME ${TEST_NAME}
+             COMMAND ${QUDA_CTEST_LAUNCH} $<TARGET_FILE:dslash_ctest> ${MPIEXEC_POSTFLAGS}
+                     --dslash-type ${DIRAC_NAME}
+                     --test Mat
+		     --domain-decomposition 1
+                     --flavor nondeg-doublet
+                     --dim 4 2 6 8
+                     --gtest_output=xml:${TEST_NAME}.xml)
+    if(polenv)
+      set_tests_properties(${TEST_NAME} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol})
+    endif()
   endif()
 
   if(QUDA_DIRAC_TWISTED_CLOVER)
@@ -652,7 +687,7 @@ foreach(pol IN LISTS DSLASH_POLICIES)
                      --dslash-type ${DIRAC_NAME}
                      --test MatPCDagMatPC
                      --matpc even-even
-                     --dim 4 2 6 8
+                     --dim 2 4 6 8
                      --gtest_output=xml:dslash_${DIRAC_NAME}_sym_test_pol${pol2}.xml)
     if(polenv)
       set_tests_properties(dslash_${DIRAC_NAME}_sym_policy${pol2}
@@ -665,7 +700,7 @@ foreach(pol IN LISTS DSLASH_POLICIES)
                      --dslash-type ${DIRAC_NAME}
                      --test MatPCDagMatPC
                      --matpc even-even-asym
-                     --dim 4 2 6 8
+                     --dim 2 4 6 8
                      --gtest_output=xml:dslash_${DIRAC_NAME}_asym_test_pol${pol2}.xml)
     if(polenv)
       set_tests_properties(dslash_${DIRAC_NAME}_asym_policy${pol2}
@@ -684,6 +719,17 @@ foreach(pol IN LISTS DSLASH_POLICIES)
       set_tests_properties(benchmark_dslash_${DIRAC_NAME}_policy${pol2} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol})
     endif()
 
+    set(TEST_NAME dslash_${DIRAC_NAME}_mat_policy${pol2}_DD)
+    add_test(NAME ${TEST_NAME}
+             COMMAND ${QUDA_CTEST_LAUNCH} $<TARGET_FILE:dslash_ctest> ${MPIEXEC_POSTFLAGS}
+                     --dslash-type ${DIRAC_NAME}
+                     --test Mat
+		     --domain-decomposition 1
+                     --dim 4 2 6 8
+                     --gtest_output=xml:${TEST_NAME}.xml)
+    if(polenv)
+      set_tests_properties(${TEST_NAME} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol})
+    endif()
   endif()
 
   if(QUDA_DIRAC_NDEG_TWISTED_CLOVER)
@@ -698,7 +744,7 @@ foreach(pol IN LISTS DSLASH_POLICIES)
                      --flavor nondeg-doublet
                      --test MatPCDagMatPC
                      --matpc even-even
-                     --dim 4 2 6 8
+                     --dim 2 4 6 8
                      --gtest_output=xml:dslash_ndeg_${DIRAC_NAME}_sym_test_pol${pol2}.xml)
     if(polenv)
       set_tests_properties(dslash_ndeg_${DIRAC_NAME}_sym_policy${pol2}
@@ -712,7 +758,7 @@ foreach(pol IN LISTS DSLASH_POLICIES)
                      --flavor nondeg-doublet
                      --test MatPCDagMatPC
                      --matpc even-even-asym
-                     --dim 4 2 6 8
+                     --dim 2 4 6 8
                      --gtest_output=xml:dslash_ndeg_${DIRAC_NAME}_asym_test_pol${pol2}.xml)
     if(polenv)
       set_tests_properties(dslash_ndeg_${DIRAC_NAME}_asym_policy${pol2}
@@ -732,6 +778,18 @@ foreach(pol IN LISTS DSLASH_POLICIES)
       set_tests_properties(benchmark_dslash_ndeg_${DIRAC_NAME}_policy${pol2} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol})
     endif()
 
+    set(TEST_NAME dslash_ndeg_${DIRAC_NAME}_mat_policy${pol2}_DD)
+    add_test(NAME ${TEST_NAME}
+             COMMAND ${QUDA_CTEST_LAUNCH} $<TARGET_FILE:dslash_ctest> ${MPIEXEC_POSTFLAGS}
+                     --dslash-type ${DIRAC_NAME}
+                     --test Mat
+		     --domain-decomposition 1
+                     --flavor nondeg-doublet
+                     --dim 4 2 6 8
+                     --gtest_output=xml:${TEST_NAME}.xml)
+    if(polenv)
+      set_tests_properties(${TEST_NAME} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol})
+    endif()
   endif()
 
   if(QUDA_DIRAC_DOMAIN_WALL)
@@ -743,7 +801,7 @@ foreach(pol IN LISTS DSLASH_POLICIES)
                      --dslash-type ${DIRAC_NAME}
                      --test MatPCDagMatPC
                      --matpc even-even
-                     --dim 4 2 6 8
+                     --dim 2 4 6 8
                      --Lsdim 4
         --gtest_output=xml:dslash_${DIRAC_NAME}_test_pol${pol2}.xml)
     if(polenv)
@@ -763,6 +821,19 @@ foreach(pol IN LISTS DSLASH_POLICIES)
       set_tests_properties(benchmark_dslash_${DIRAC_NAME}_policy${pol2} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol})
     endif()
 
+    set(TEST_NAME dslash_ndeg_${DIRAC_NAME}_mat_policy${pol2}_DD)
+    add_test(NAME ${TEST_NAME}
+             COMMAND ${QUDA_CTEST_LAUNCH} $<TARGET_FILE:dslash_ctest> ${MPIEXEC_POSTFLAGS}
+                     --dslash-type ${DIRAC_NAME}
+                     --test Mat
+		     --domain-decomposition 1
+                     --flavor nondeg-doublet
+                     --dim 4 2 6 8
+                     --gtest_output=xml:${TEST_NAME}.xml)
+    if(polenv)
+      set_tests_properties(${TEST_NAME} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol})
+    endif()
+
     # 4-d preconditioned
     set(DIRAC_NAME domain-wall-4d)
 
@@ -772,7 +843,7 @@ foreach(pol IN LISTS DSLASH_POLICIES)
                      --dslash-type ${DIRAC_NAME}
                      --test MatPCDagMatPC
                      --matpc even-even
-                     --dim 4 2 6 8
+                     --dim 2 4 6 8
                      --Lsdim 4
         --gtest_output=xml:dslash_${DIRAC_NAME}_sym_test_pol${pol2}.xml)
     if(polenv)
@@ -785,7 +856,7 @@ foreach(pol IN LISTS DSLASH_POLICIES)
                      --dslash-type ${DIRAC_NAME}
                      --test MatPCDagMatPC
                      --matpc even-even-asym
-                     --dim 4 2 6 8
+                     --dim 2 4 6 8
                      --Lsdim 4
         --gtest_output=xml:dslash_${DIRAC_NAME}_asym_test_pol${pol2}.xml)
     if(polenv)
@@ -805,6 +876,19 @@ foreach(pol IN LISTS DSLASH_POLICIES)
       set_tests_properties(benchmark_dslash_${DIRAC_NAME}_policy${pol2} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol})
     endif()
 
+    set(TEST_NAME dslash_${DIRAC_NAME}_mat_policy${pol2}_DD)
+    add_test(NAME ${TEST_NAME}
+             COMMAND ${QUDA_CTEST_LAUNCH} $<TARGET_FILE:dslash_ctest> ${MPIEXEC_POSTFLAGS}
+                     --dslash-type ${DIRAC_NAME}
+                     --all-partitions 0
+                     --test Mat
+		     --domain-decomposition 1
+                     --dim 4 2 6 8
+                     --gtest_output=xml:${TEST_NAME}.xml)
+    if(polenv)
+      set_tests_properties(${TEST_NAME} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol})
+    endif()
+
     # Mobius fermions
     set(DIRAC_NAME mobius)
 
@@ -814,7 +898,7 @@ foreach(pol IN LISTS DSLASH_POLICIES)
                      --dslash-type ${DIRAC_NAME}
                      --test MatPCDagMatPC
                      --matpc even-even
-                     --dim 4 2 6 8
+                     --dim 2 4 6 8
                      --Lsdim 4
                      --gtest_output=xml:dslash_${DIRAC_NAME}_sym_test_pol${pol2}.xml)
     if(polenv)
@@ -826,7 +910,7 @@ foreach(pol IN LISTS DSLASH_POLICIES)
                      --dslash-type ${DIRAC_NAME}
                      --test MatPCDagMatPC
                      --matpc even-even-asym
-                     --dim 4 2 6 8
+                     --dim 2 4 6 8
                      --Lsdim 4
                      --gtest_output=xml:dslash_${DIRAC_NAME}_asym_test_pol${pol2}.xml)
     if(polenv)
@@ -858,7 +942,7 @@ foreach(pol IN LISTS DSLASH_POLICIES)
                        --dslash-type ${DIRAC_NAME}
                        --test MatPCDagMatPCLocal
                        --matpc even-even
-                       --dim 4 2 6 8
+                       --dim 2 4 6 8
                        --Lsdim 4
                        --gtest_output=xml:dslash_${DIRAC_NAME}_local{pol2}.xml)
     endif()
@@ -871,7 +955,7 @@ foreach(pol IN LISTS DSLASH_POLICIES)
                      --dslash-type ${DIRAC_NAME}
                      --test MatPCDagMatPC
                      --matpc even-even
-                     --dim 4 2 6 8
+                     --dim 2 4 6 8
                      --Lsdim 4
         --gtest_output=xml:dslash_${DIRAC_NAME}_sym_test_pol${pol2}.xml)
     if(polenv)
@@ -883,13 +967,26 @@ foreach(pol IN LISTS DSLASH_POLICIES)
                      --dslash-type ${DIRAC_NAME}
                      --test MatPCDagMatPC
                      --matpc even-even-asym
-                     --dim 4 2 6 8
+                     --dim 2 4 6 8
                      --Lsdim 4
                      --gtest_output=xml:dslash_${DIRAC_NAME}_asym_test_pol${pol2}.xml)
  
     if(polenv)
       set_tests_properties(dslash_${DIRAC_NAME}_asym_policy${pol2} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol})
     endif()
+
+    set(TEST_NAME dslash_${DIRAC_NAME}_mat_policy${pol2}_DD)
+    add_test(NAME ${TEST_NAME}
+             COMMAND ${QUDA_CTEST_LAUNCH} $<TARGET_FILE:dslash_ctest> ${MPIEXEC_POSTFLAGS}
+                     --dslash-type ${DIRAC_NAME}
+                     --all-partitions 0
+                     --test Mat
+		     --domain-decomposition 1
+                     --dim 4 2 6 8
+                     --gtest_output=xml:${TEST_NAME}.xml)
+    if(polenv)
+      set_tests_properties(${TEST_NAME} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol})
+    endif()
   endif()
 
   if(QUDA_DIRAC_STAGGERED)
@@ -937,6 +1034,19 @@ foreach(pol IN LISTS DSLASH_POLICIES)
       set_tests_properties(benchmark_dslash_${DIRAC_NAME}_policy${pol2} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol})
     endif()
 
+    set(TEST_NAME dslash_${DIRAC_NAME}_mat_policy${pol2}_DD)
+    add_test(NAME ${TEST_NAME}
+             COMMAND ${QUDA_CTEST_LAUNCH} $<TARGET_FILE:staggered_dslash_ctest> ${MPIEXEC_POSTFLAGS}
+                     --dslash-type ${DIRAC_NAME}
+                     --all-partitions 0
+                     --test Mat
+		     --domain-decomposition 1
+                     --dim 4 2 6 8
+                     --gtest_output=xml:${TEST_NAME}.xml)
+    if(polenv)
+      set_tests_properties(${TEST_NAME} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol})
+    endif()
+
     set(DIRAC_NAME asqtad)
     add_test(NAME dslash_${DIRAC_NAME}_matpc_policy${pol2}
              COMMAND ${QUDA_CTEST_LAUNCH} $<TARGET_FILE:staggered_dslash_ctest> ${MPIEXEC_POSTFLAGS}
@@ -986,6 +1096,19 @@ foreach(pol IN LISTS DSLASH_POLICIES)
       set_tests_properties(dslash_${DIRAC_NAME}_build_policy${pol2} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol2})
     endif()
 
+    set(TEST_NAME dslash_${DIRAC_NAME}_mat_policy${pol2}_DD)
+    add_test(NAME ${TEST_NAME}
+             COMMAND ${QUDA_CTEST_LAUNCH} $<TARGET_FILE:staggered_dslash_ctest> ${MPIEXEC_POSTFLAGS}
+                     --dslash-type ${DIRAC_NAME}
+                     --all-partitions 0
+                     --test Mat
+		     --domain-decomposition 1
+                     --dim 8 6 10 12
+                     --gtest_output=xml:${TEST_NAME}.xml)
+    if(polenv)
+      set_tests_properties(${TEST_NAME} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol})
+    endif()
+
     if(QUDA_LAPLACE)
       set(DIRAC_NAME laplace)
       add_test(NAME dslash_${DIRAC_NAME}_mat_policy${pol2}

From bdc7d10df64da2fd4d8159a224e02edf6286249e Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Mon, 8 Jul 2024 13:41:40 +0200
Subject: [PATCH 074/108] Cleanup and removing usage of Tmp2 and Tmp3 spinors

---
 tests/dslash_ctest.cpp    |  3 ++
 tests/dslash_test_utils.h | 61 +++++++++++++++------------------------
 2 files changed, 26 insertions(+), 38 deletions(-)

diff --git a/tests/dslash_ctest.cpp b/tests/dslash_ctest.cpp
index b2f6ea27ad..bf49ec8b2d 100644
--- a/tests/dslash_ctest.cpp
+++ b/tests/dslash_ctest.cpp
@@ -42,6 +42,7 @@ class DslashTest : public ::testing::TestWithParam<::testing::tuple<int, int, in
                                                   true, false, false, false, true, false, true, true};
     if (!ctest_all_partitions && !partition_enabled[::testing::get<2>(GetParam())]) return true;
 
+    if (::testing::get<3>(GetParam()) == 0 && ::testing::get<4>(GetParam()) > 0) return true;
     if (!ctest_domain_decomposition && ::testing::get<3>(GetParam())>0) return true;
 
     return false;
@@ -191,6 +192,8 @@ std::string getdslashtestname(testing::TestParamInfo<::testing::tuple<int, int,
     case 2: ss << "_red_black"; break;
     case 3: ss << "_black_black"; break;
     }
+  } else if (col > 0) {
+    ss << "_skipped" << col;
   }
   return ss.str();
 }
diff --git a/tests/dslash_test_utils.h b/tests/dslash_test_utils.h
index 31cef86e9e..d2a8e8d422 100644
--- a/tests/dslash_test_utils.h
+++ b/tests/dslash_test_utils.h
@@ -63,8 +63,6 @@ struct DslashTestWrapper {
   std::vector<ColorSpinorField> cudaSpinor;
   std::vector<ColorSpinorField> cudaSpinorOut;
   std::vector<ColorSpinorField> cudaSpinorTmp;
-  std::vector<ColorSpinorField> cudaSpinorTmp2;
-  std::vector<ColorSpinorField> cudaSpinorTmp3;
 
   // Dirac pointers
   quda::Dirac *dirac = nullptr;
@@ -336,16 +334,10 @@ struct DslashTestWrapper {
       csParam.setPrecision(inv_param.cuda_prec, inv_param.cuda_prec, true);
 
       printfQuda("Creating cudaSpinor with nParity = %d\n", csParam.siteSubset);
-      cudaSpinor.resize(Nsrc);
-      for (int i = 0; i < Nsrc; i++) cudaSpinor[i] = ColorSpinorField(csParam);
+      resize(cudaSpinor, Nsrc, csParam);
       printfQuda("Creating cudaSpinorOut with nParity = %d\n", csParam.siteSubset);
-      cudaSpinorOut.resize(Nsrc);
-      for (int i = 0; i < Nsrc; i++) cudaSpinorOut[i] = ColorSpinorField(csParam);
-      if (test_domain_decomposition) {
-        resize(cudaSpinorTmp, Nsrc, csParam);
-        resize(cudaSpinorTmp2, Nsrc, csParam);
-        resize(cudaSpinorTmp3, Nsrc, csParam);
-      }
+      resize(cudaSpinorOut, Nsrc, csParam);
+      if (test_domain_decomposition) { resize(cudaSpinorTmp, Nsrc, csParam); }
 
       printfQuda("Sending spinor field to GPU\n");
       cudaSpinor = spinor;
@@ -864,17 +856,15 @@ struct DslashTestWrapper {
     } else if (test_domain_decomposition) {
 
       vector_ref<ColorSpinorField> spinor = cudaSpinor;
+      vector_ref<ColorSpinorField> out = cudaSpinorOut;
       vector_ref<ColorSpinorField> tmp = cudaSpinorTmp;
-      vector_ref<ColorSpinorField> tmp2 = cudaSpinorTmp2;
-      vector_ref<ColorSpinorField> tmp3 = cudaSpinorTmp3;
 
       if (dd_red_black) {
         for (int n = 0; n < Nsrc; n++) {
           for (auto i = 0u; i < 4; i++) {
             cudaSpinor[n].DD().block_dim[i] = dd_block_size[i];
+            cudaSpinorOut[n].DD().block_dim[i] = dd_block_size[i];
             cudaSpinorTmp[n].DD().block_dim[i] = dd_block_size[i];
-            cudaSpinorTmp2[n].DD().block_dim[i] = dd_block_size[i];
-            cudaSpinorTmp3[n].DD().block_dim[i] = dd_block_size[i];
           }
         }
 
@@ -882,48 +872,43 @@ struct DslashTestWrapper {
         blas::zero(cudaSpinorTmp);
 
         spinor.DD(DD::reset, DD::red_black_type, dd_col % 2 == 0 ? DD::red_active : DD::black_active);
-        tmp.DD(DD::reset, DD::red_black_type, dd_col / 2 == 0 ? DD::red_active : DD::black_active);
+        out.DD(DD::reset, DD::red_black_type, dd_col / 2 == 0 ? DD::red_active : DD::black_active);
 
         switch (dtest_type) {
-        case dslash_test_type::Dslash: dirac->Dslash(cudaSpinorTmp, cudaSpinor, parity); break;
+        case dslash_test_type::Dslash: dirac->Dslash(cudaSpinorOut, cudaSpinor, parity); break;
         case dslash_test_type::MatPC:
-        case dslash_test_type::Mat: dirac->M(cudaSpinorTmp, cudaSpinor); break;
+        case dslash_test_type::Mat: dirac->M(cudaSpinorOut, cudaSpinor); break;
         case dslash_test_type::MatPCDagMatPC:
-        case dslash_test_type::MatDagMat: dirac->MdagM(cudaSpinorTmp, cudaSpinor); break;
+        case dslash_test_type::MatDagMat: dirac->MdagM(cudaSpinorOut, cudaSpinor); break;
         default:
           errorQuda("Test type %s not support for current Dslash", get_string(dtest_type_map, dtest_type).c_str());
         }
 
         spinor.DD(DD::reset);
-        tmp.DD(DD::reset);
+        out.DD(DD::reset);
+        spinorOut = cudaSpinorOut;
 
         // We also test that Dyx is same as D applied to projected in and out spinors
-        blas::copy(tmp2, cudaSpinor);
-        tmp2.DD(DD::reset, DD::red_black_type, dd_col % 2 == 0 ? DD::red_active : DD::black_active);
-        tmp2.projectDD();
-        tmp2.DD(DD::reset);
+        blas::copy(tmp, cudaSpinor);
+        tmp.DD(DD::reset, DD::red_black_type, dd_col % 2 == 0 ? DD::red_active : DD::black_active);
+        tmp.projectDD();
+        tmp.DD(DD::reset);
 
         switch (dtest_type) {
-        case dslash_test_type::Dslash: dirac->Dslash(cudaSpinorTmp3, cudaSpinorTmp2, parity); break;
+        case dslash_test_type::Dslash: dirac->Dslash(cudaSpinorOut, cudaSpinorTmp, parity); break;
         case dslash_test_type::MatPC:
-        case dslash_test_type::Mat: dirac->M(cudaSpinorTmp3, cudaSpinorTmp2); break;
+        case dslash_test_type::Mat: dirac->M(cudaSpinorOut, cudaSpinorTmp); break;
         case dslash_test_type::MatPCDagMatPC:
-        case dslash_test_type::MatDagMat: dirac->MdagM(cudaSpinorTmp3, cudaSpinorTmp2); break;
+        case dslash_test_type::MatDagMat: dirac->MdagM(cudaSpinorOut, cudaSpinorTmp); break;
         default:
           errorQuda("Test type %s not support for current Dslash", get_string(dtest_type_map, dtest_type).c_str());
         }
 
-        tmp3.DD(DD::reset, DD::red_black_type, dd_col / 2 == 0 ? DD::red_active : DD::black_active);
-        tmp3.projectDD();
-        tmp3.DD(DD::reset);
-
-        if (dd_test_projection)
-          blas::xpy(cudaSpinorTmp3, cudaSpinorOut);
-        else
-          blas::xpy(cudaSpinorTmp, cudaSpinorOut);
+        out.DD(DD::reset, DD::red_black_type, dd_col / 2 == 0 ? DD::red_active : DD::black_active);
+        out.projectDD();
+        out.DD(DD::reset);
 
-        spinorTmp = cudaSpinorTmp;
-        spinorOut = cudaSpinorTmp3;
+        spinorRef = cudaSpinorOut;
 
       } else {
         errorQuda("Test dd type not supported");
@@ -1211,7 +1196,7 @@ struct DslashTestWrapper {
       }
     } else if (test_domain_decomposition) {
       for (int n = 0; n < Nsrc; n++) {
-        auto deviation = std::pow(10, -(double)(ColorSpinorField::Compare(spinorTmp[n], spinorOut[n])));
+        auto deviation = std::pow(10, -(double)(ColorSpinorField::Compare(spinorRef[n], spinorOut[n])));
         printfQuda("Deviation for (D-PDP)_{%d,%d}*spinor is %e\n", dd_col % 2, dd_col / 2, deviation);
         double tol = getTolerance(cuda_prec);
         EXPECT_LE(deviation, tol) << "Projected Dirac and project spinors do not agree";

From 1234bbbcc03eccb15032c8a09e1366fa6bcc013a Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Mon, 8 Jul 2024 13:42:04 +0200
Subject: [PATCH 075/108] Adding DD also to staggered_dslash_ctest

---
 tests/staggered_dslash_ctest.cpp    |  62 ++++++++++++---
 tests/staggered_dslash_test_utils.h | 115 ++++++++++++++++++++++++++--
 2 files changed, 158 insertions(+), 19 deletions(-)

diff --git a/tests/staggered_dslash_ctest.cpp b/tests/staggered_dslash_ctest.cpp
index 65edd69124..f4db942dcd 100644
--- a/tests/staggered_dslash_ctest.cpp
+++ b/tests/staggered_dslash_ctest.cpp
@@ -3,6 +3,7 @@
 using namespace quda;
 
 bool ctest_all_partitions = false;
+bool ctest_domain_decomposition = false;
 
 using ::testing::Bool;
 using ::testing::Combine;
@@ -10,10 +11,10 @@ using ::testing::Range;
 using ::testing::TestWithParam;
 using ::testing::Values;
 
-class StaggeredDslashTest : public ::testing::TestWithParam<::testing::tuple<int, int, int>>
+class StaggeredDslashTest : public ::testing::TestWithParam<::testing::tuple<int, int, int, int, int>>
 {
 protected:
-  ::testing::tuple<int, int, int> param;
+  ::testing::tuple<int, int, int, int, int> param;
 
   bool skip()
   {
@@ -31,6 +32,10 @@ class StaggeredDslashTest : public ::testing::TestWithParam<::testing::tuple<int
     if (!ctest_all_partitions && !partition_enabled[::testing::get<2>(GetParam())]) return true;
 
     if (::testing::get<2>(GetParam()) > 0 && dslash_test_wrapper.test_split_grid) { return true; }
+
+    if (::testing::get<3>(GetParam()) == 0 && ::testing::get<4>(GetParam()) > 0) return true;
+    if (!ctest_domain_decomposition && ::testing::get<3>(GetParam()) > 0) return true;
+
     return false;
   }
 
@@ -42,6 +47,11 @@ class StaggeredDslashTest : public ::testing::TestWithParam<::testing::tuple<int
     printfQuda("prec recon   test_type     dagger   S_dim         T_dimension\n");
     printfQuda("%s   %s       %s           %d       %d/%d/%d        %d \n", get_prec_str(prec),
                get_recon_str(link_recon), get_string(dtest_type_map, dtest_type).c_str(), dagger, xdim, ydim, zdim, tdim);
+    if (dslash_test_wrapper.test_domain_decomposition) {
+      if (dd_red_black)
+        printfQuda("Testing DD Red Black with block: %d  %d  %d  %d\n", dd_block_size[0], dd_block_size[1],
+                   dd_block_size[2], dd_block_size[3]);
+    }
   }
 
 public:
@@ -58,7 +68,10 @@ class StaggeredDslashTest : public ::testing::TestWithParam<::testing::tuple<int
     }
     updateR();
 
-    dslash_test_wrapper.init_ctest(prec, recon);
+    int dd_value = ::testing::get<3>(GetParam());
+    int dd_color = ::testing::get<4>(GetParam());
+
+    dslash_test_wrapper.init_ctest(prec, recon, dd_value, dd_color);
     display_test_info(prec, recon);
   }
 
@@ -108,6 +121,7 @@ int main(int argc, char **argv)
   auto app = make_app();
   app->add_option("--test", dtest_type, "Test method")->transform(CLI::CheckedTransformer(dtest_type_map));
   app->add_option("--all-partitions", ctest_all_partitions, "Test all instead of reduced combination of partitions");
+  app->add_option("--domain-decomposition", ctest_domain_decomposition, "Test domain decomposition");
   add_comms_option_group(app);
   try {
     app->parse(argc, argv);
@@ -154,29 +168,55 @@ int main(int argc, char **argv)
   return test_rc;
 }
 
-std::string getstaggereddslashtestname(testing::TestParamInfo<::testing::tuple<int, int, int>> param)
+std::string getstaggereddslashtestname(testing::TestParamInfo<::testing::tuple<int, int, int, int, int>> param)
 {
   const int prec = ::testing::get<0>(param.param);
   const int recon = ::testing::get<1>(param.param);
   const int part = ::testing::get<2>(param.param);
+  const int dd = ::testing::get<3>(param.param);
+  const int col = ::testing::get<4>(param.param);
   std::stringstream ss;
   // ss << get_dslash_str(dslash_type) << "_";
   ss << get_prec_str(getPrecision(prec));
   ss << "_r" << recon;
   ss << "_partition" << part;
+  if (dd > 0) {
+    switch (dd) {
+    case 1: ss << "_dd_local"; break;
+    case 2: ss << "_dd_global"; break;
+    }
+    switch (col) {
+    case 0: ss << "_red_red"; break;
+    case 1: ss << "_black_red"; break;
+    case 2: ss << "_red_black"; break;
+    case 3: ss << "_black_black"; break;
+    }
+  } else if (col > 0) {
+    ss << "_skipped" << col;
+  }
   return ss.str();
 }
 
 #ifdef MULTI_GPU
-INSTANTIATE_TEST_SUITE_P(QUDA, StaggeredDslashTest,
-                         Combine(Range(0, 4),
-                                 ::testing::Values(QUDA_RECONSTRUCT_NO, QUDA_RECONSTRUCT_12, QUDA_RECONSTRUCT_8),
-                                 Range(0, 16)),
-                         getstaggereddslashtestname);
+#define N_PARTITIONS 16
+#else
+#define N_PARTITIONS 1
+#endif
+
+#ifdef GPU_DD_DIRAC
+#define N_DD_TESTS 3
+#define N_DD_COLS 4
 #else
+#define N_DD_TESTS 1
+#define N_DD_COLS 1
+#endif
+
 INSTANTIATE_TEST_SUITE_P(QUDA, StaggeredDslashTest,
                          Combine(Range(0, 4),
                                  ::testing::Values(QUDA_RECONSTRUCT_NO, QUDA_RECONSTRUCT_12, QUDA_RECONSTRUCT_8),
-                                 ::testing::Values(0)),
+                                 Range(0, N_PARTITIONS), Range(0, N_DD_TESTS), Range(0, N_DD_COLS)),
                          getstaggereddslashtestname);
-#endif
+
+#undef N_PARTITIONS
+#undef N_DD_TESTS
+#undef N_DD_COLS
diff --git a/tests/staggered_dslash_test_utils.h b/tests/staggered_dslash_test_utils.h
index 4d336d8d4f..b195c79638 100644
--- a/tests/staggered_dslash_test_utils.h
+++ b/tests/staggered_dslash_test_utils.h
@@ -57,6 +57,7 @@ struct StaggeredDslashTestWrapper {
 
   std::vector<ColorSpinorField> cudaSpinor;
   std::vector<ColorSpinorField> cudaSpinorOut;
+  std::vector<ColorSpinorField> cudaSpinorTmp;
 
   static inline std::vector<ColorSpinorField> vp_spinor;
   static inline std::vector<ColorSpinorField> vp_spinor_out;
@@ -75,8 +76,11 @@ struct StaggeredDslashTestWrapper {
 
   // Split grid options
   static inline bool test_split_grid = false;
+  static inline bool test_domain_decomposition = false;
   int num_src = 1;
 
+  static inline int dd_col = 0;
+
   void staggeredDslashRef()
   {
     // compare to dslash reference implementation
@@ -98,7 +102,7 @@ struct StaggeredDslashTestWrapper {
     }
   }
 
-  void init_ctest(int precision, QudaReconstructType link_recon_)
+  void init_ctest(int precision, QudaReconstructType link_recon_, int dd_value, int dd_color)
   {
     gauge_param = newQudaGaugeParam();
     inv_param = newQudaInvertParam();
@@ -121,6 +125,8 @@ struct StaggeredDslashTestWrapper {
       init_host();
       first_time = false;
     }
+
+    init_domain_decomposition(dd_value, dd_color);
     init();
   }
 
@@ -216,6 +222,36 @@ struct StaggeredDslashTestWrapper {
     setVerbosity(verbosity);
   }
 
+  void init_domain_decomposition(int value, int color)
+  {
+    if (value == 0) {
+      test_domain_decomposition = false;
+      return;
+    }
+    test_domain_decomposition = true;
+    dd_col = color;
+
+    if (value < 3) {
+      dd_red_black = true;
+
+      // dd_block_size is half of the local lattice
+      if (value == 1) {
+        for (auto i = 0u; i < 4; i++) dd_block_size[i] = gauge_param.X[i] / 2;
+        return;
+      }
+
+      // dd_block_size is half of the global lattice
+      if (value == 2) {
+        for (auto i = 0u; i < 4; i++) dd_block_size[i] = (gauge_param.X[i] * comm_dim(i)) / 2;
+        return;
+      }
+
+    } else {
+      dd_red_black = false;
+    }
+    errorQuda("Unexpected value for domain decomposition (%d)", value);
+  }
+
   void init()
   {
 
@@ -258,13 +294,10 @@ struct StaggeredDslashTestWrapper {
     csParam.setPrecision(inv_param.cuda_prec);
     csParam.location = QUDA_CUDA_FIELD_LOCATION;
 
-    cudaSpinor.resize(Nsrc);
-    cudaSpinorOut.resize(Nsrc);
-    for (auto i = 0; i < Nsrc; i++) {
-      cudaSpinor[i] = ColorSpinorField(csParam);
-      cudaSpinorOut[i] = ColorSpinorField(csParam);
-      cudaSpinor[i] = spinor[i];
-    }
+    resize(cudaSpinor, Nsrc, csParam);
+    resize(cudaSpinorOut, Nsrc, csParam);
+    if (test_domain_decomposition) { resize(cudaSpinorTmp, Nsrc, csParam); }
+    cudaSpinor = spinor;
 
     bool pc = (dtest_type == dslash_test_type::MatPC); // For test_type 0, can use either pc or not pc
     // because both call the same "Dslash" directly.
@@ -334,6 +367,65 @@ struct StaggeredDslashTestWrapper {
       }
       dslashMultiSrcQuda(_hp_x.data(), _hp_b.data(), &inv_param, parity);
 
+    } else if (test_domain_decomposition) {
+
+      vector_ref<ColorSpinorField> spinor = cudaSpinor;
+      vector_ref<ColorSpinorField> out = cudaSpinorOut;
+      vector_ref<ColorSpinorField> tmp = cudaSpinorTmp;
+
+      if (dd_red_black) {
+        for (int n = 0; n < Nsrc; n++) {
+          for (auto i = 0u; i < 4; i++) {
+            cudaSpinor[n].DD().block_dim[i] = dd_block_size[i];
+            cudaSpinorOut[n].DD().block_dim[i] = dd_block_size[i];
+            cudaSpinorTmp[n].DD().block_dim[i] = dd_block_size[i];
+          }
+        }
+
+        blas::zero(cudaSpinorOut);
+        blas::zero(cudaSpinorTmp);
+
+        spinor.DD(DD::reset, DD::red_black_type, dd_col % 2 == 0 ? DD::red_active : DD::black_active);
+        out.DD(DD::reset, DD::red_black_type, dd_col / 2 == 0 ? DD::red_active : DD::black_active);
+
+        switch (dtest_type) {
+        case dslash_test_type::Dslash: dirac->Dslash(cudaSpinorOut, cudaSpinor, parity); break;
+        case dslash_test_type::MatPC:
+        case dslash_test_type::Mat: dirac->M(cudaSpinorOut, cudaSpinor); break;
+        case dslash_test_type::MatDagMat: dirac->MdagM(cudaSpinorOut, cudaSpinor); break;
+        default:
+          errorQuda("Test type %s not support for current Dslash", get_string(dtest_type_map, dtest_type).c_str());
+        }
+
+        spinor.DD(DD::reset);
+        out.DD(DD::reset);
+        spinorOut = cudaSpinorOut;
+
+        // We also test that Dyx is same as D applied to projected in and out spinors
+        blas::copy(tmp, cudaSpinor);
+        tmp.DD(DD::reset, DD::red_black_type, dd_col % 2 == 0 ? DD::red_active : DD::black_active);
+        tmp.projectDD();
+        tmp.DD(DD::reset);
+
+        switch (dtest_type) {
+        case dslash_test_type::Dslash: dirac->Dslash(cudaSpinorOut, cudaSpinorTmp, parity); break;
+        case dslash_test_type::MatPC:
+        case dslash_test_type::Mat: dirac->M(cudaSpinorOut, cudaSpinorTmp); break;
+        case dslash_test_type::MatDagMat: dirac->MdagM(cudaSpinorOut, cudaSpinorTmp); break;
+        default:
+          errorQuda("Test type %s not support for current Dslash", get_string(dtest_type_map, dtest_type).c_str());
+        }
+
+        out.DD(DD::reset, DD::red_black_type, dd_col / 2 == 0 ? DD::red_active : DD::black_active);
+        out.projectDD();
+        out.DD(DD::reset);
+
+        spinorRef = cudaSpinorOut;
+
+      } else {
+        errorQuda("Test dd type not supported");
+      }
+
     } else {
 
       for (int i = 0; i < niter; i++) {
@@ -442,6 +534,13 @@ struct StaggeredDslashTestWrapper {
         deviation = std::max(deviation, pow(10.0, -(double)(ColorSpinorField::Compare(spinorRef[0], vp_spinor_out[n]))));
         if (failed) { deviation = 1.0; }
       }
+    } else if (test_domain_decomposition) {
+      for (int n = 0; n < Nsrc; n++) {
+        auto deviation = std::pow(10, -(double)(ColorSpinorField::Compare(spinorRef[n], spinorOut[n])));
+        printfQuda("Deviation for (D-PDP)_{%d,%d}*spinor is %e\n", dd_col % 2, dd_col / 2, deviation);
+        double tol = getTolerance(cuda_prec);
+        EXPECT_LE(deviation, tol) << "Projected Dirac and project spinors do not agree";
+      }
     } else {
       for (int i = 0; i < Nsrc; i++) {
         auto spinor_ref_norm = blas::norm2(spinorRef[i]);

From 90b4c5c648596f70d728f69ce73ae5f8e4efaed8 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Wed, 24 Jul 2024 14:10:29 +0200
Subject: [PATCH 076/108] fixing benchmarking

---
 tests/dslash_test_utils.h | 71 ++++++++++++++++++++++-----------------
 1 file changed, 41 insertions(+), 30 deletions(-)

diff --git a/tests/dslash_test_utils.h b/tests/dslash_test_utils.h
index d2a8e8d422..50ad8b57c4 100644
--- a/tests/dslash_test_utils.h
+++ b/tests/dslash_test_utils.h
@@ -872,44 +872,57 @@ struct DslashTestWrapper {
         blas::zero(cudaSpinorTmp);
 
         spinor.DD(DD::reset, DD::red_black_type, dd_col % 2 == 0 ? DD::red_active : DD::black_active);
-        out.DD(DD::reset, DD::red_black_type, dd_col / 2 == 0 ? DD::red_active : DD::black_active);
+        out.DD(DD::reset, DD::red_black_type, dd_col / 2 == 1 ? DD::red_active : DD::black_active);
 
-        switch (dtest_type) {
-        case dslash_test_type::Dslash: dirac->Dslash(cudaSpinorOut, cudaSpinor, parity); break;
-        case dslash_test_type::MatPC:
-        case dslash_test_type::Mat: dirac->M(cudaSpinorOut, cudaSpinor); break;
-        case dslash_test_type::MatPCDagMatPC:
-        case dslash_test_type::MatDagMat: dirac->MdagM(cudaSpinorOut, cudaSpinor); break;
-        default:
-          errorQuda("Test type %s not support for current Dslash", get_string(dtest_type_map, dtest_type).c_str());
+        for (int i = 0; i < niter; i++) {
+          host_timer.start();
+          switch (dtest_type) {
+          case dslash_test_type::Dslash: dirac->Dslash(cudaSpinorOut, cudaSpinor, parity); break;
+          case dslash_test_type::MatPC:
+          case dslash_test_type::Mat: dirac->M(cudaSpinorOut, cudaSpinor); break;
+          case dslash_test_type::MatPCDagMatPC:
+          case dslash_test_type::MatDagMat: dirac->MdagM(cudaSpinorOut, cudaSpinor); break;
+          default:
+            errorQuda("Test type %s not support for current Dslash", get_string(dtest_type_map, dtest_type).c_str());
+          }
+          host_timer.stop();
+
+          dslash_time.cpu_time += host_timer.last();
+          // skip first and last iterations since they may skew these metrics if comms are not synchronous
+          if (i > 0 && i < niter) {
+            dslash_time.cpu_min = std::min(dslash_time.cpu_min, host_timer.last());
+            dslash_time.cpu_max = std::max(dslash_time.cpu_max, host_timer.last());
+          }
         }
 
         spinor.DD(DD::reset);
         out.DD(DD::reset);
         spinorOut = cudaSpinorOut;
 
-        // We also test that Dyx is same as D applied to projected in and out spinors
-        blas::copy(tmp, cudaSpinor);
-        tmp.DD(DD::reset, DD::red_black_type, dd_col % 2 == 0 ? DD::red_active : DD::black_active);
-        tmp.projectDD();
-        tmp.DD(DD::reset);
-
-        switch (dtest_type) {
-        case dslash_test_type::Dslash: dirac->Dslash(cudaSpinorOut, cudaSpinorTmp, parity); break;
-        case dslash_test_type::MatPC:
-        case dslash_test_type::Mat: dirac->M(cudaSpinorOut, cudaSpinorTmp); break;
-        case dslash_test_type::MatPCDagMatPC:
-        case dslash_test_type::MatDagMat: dirac->MdagM(cudaSpinorOut, cudaSpinorTmp); break;
-        default:
-          errorQuda("Test type %s not support for current Dslash", get_string(dtest_type_map, dtest_type).c_str());
-        }
+        if (niter > 2) { // HACK: when benchmarking we do not produce reference solution
+          // We also test that Dyx is same as D applied to projected in and out spinors
+          blas::copy(tmp, cudaSpinor);
+          tmp.DD(DD::reset, DD::red_black_type, dd_col % 2 == 0 ? DD::red_active : DD::black_active);
+          tmp.projectDD();
+          tmp.DD(DD::reset);
 
-        out.DD(DD::reset, DD::red_black_type, dd_col / 2 == 0 ? DD::red_active : DD::black_active);
-        out.projectDD();
-        out.DD(DD::reset);
+          blas::zero(cudaSpinorOut);
+          switch (dtest_type) {
+          case dslash_test_type::Dslash: dirac->Dslash(cudaSpinorOut, cudaSpinorTmp, parity); break;
+          case dslash_test_type::MatPC:
+          case dslash_test_type::Mat: dirac->M(cudaSpinorOut, cudaSpinorTmp); break;
+          case dslash_test_type::MatPCDagMatPC:
+          case dslash_test_type::MatDagMat: dirac->MdagM(cudaSpinorOut, cudaSpinorTmp); break;
+          default:
+            errorQuda("Test type %s not support for current Dslash", get_string(dtest_type_map, dtest_type).c_str());
+          }
 
-        spinorRef = cudaSpinorOut;
+          out.DD(DD::reset, DD::red_black_type, dd_col / 2 == 0 ? DD::red_active : DD::black_active);
+          out.projectDD();
+          out.DD(DD::reset);
 
+          spinorRef = cudaSpinorOut;
+        }
       } else {
         errorQuda("Test dd type not supported");
       }
@@ -1198,8 +1211,6 @@ struct DslashTestWrapper {
       for (int n = 0; n < Nsrc; n++) {
         auto deviation = std::pow(10, -(double)(ColorSpinorField::Compare(spinorRef[n], spinorOut[n])));
         printfQuda("Deviation for (D-PDP)_{%d,%d}*spinor is %e\n", dd_col % 2, dd_col / 2, deviation);
-        double tol = getTolerance(cuda_prec);
-        EXPECT_LE(deviation, tol) << "Projected Dirac and project spinors do not agree";
       }
     } else {
       for (int n = 0; n < Nsrc; n++) {

From 9fc2025ab1ed9842257fef8a047e196dec17c739 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Tue, 3 Sep 2024 14:28:45 +0200
Subject: [PATCH 077/108] Changing QUDA_DIRAC_DOMAIN_DECOMPOSITION to bit value
 for multiple kind of DD (+ some renaming)

---
 CMakeLists.txt                   | 2 +-
 include/quda_define.h.in         | 9 +++------
 tests/dslash_ctest.cpp           | 4 ++--
 tests/dslash_test.cpp            | 2 +-
 tests/staggered_dslash_ctest.cpp | 2 +-
 5 files changed, 8 insertions(+), 11 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 7187789fde..8cb0d1c034 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -151,8 +151,8 @@ option(QUDA_DIRAC_CLOVER_HASENBUSCH "build clover Hasenbusch twist operators" ${
 option(QUDA_DIRAC_NDEG_TWISTED_MASS "build non-degenerate twisted mass Dirac operators" ${QUDA_DIRAC_DEFAULT})
 option(QUDA_DIRAC_NDEG_TWISTED_CLOVER "build non-degenerate twisted clover Dirac operators" ${QUDA_DIRAC_DEFAULT})
 option(QUDA_DIRAC_LAPLACE "build laplace operator" ${QUDA_DIRAC_DEFAULT})
-option(QUDA_DIRAC_DOMAIN_DECOMPOSITION "build code for domain decomposition of the Dirac operator" OFF)
 option(QUDA_DIRAC_DISTANCE_PRECONDITIONING "build code for distance preconditioned Wilson/clover Dirac operators" OFF)
+set(QUDA_DOMAIN_DECOMPOSITION "0" CACHE STRING "which domain decomposition to instantiate in QUDA (1-bit number - RedBlack)")
 
 option(QUDA_COVDEV "build code for covariant derivative" OFF)
 
diff --git a/include/quda_define.h.in b/include/quda_define.h.in
index fd2577deb1..9852e404da 100644
--- a/include/quda_define.h.in
+++ b/include/quda_define.h.in
@@ -110,14 +110,11 @@
 #define GPU_LAPLACE
 #endif
 
-#cmakedefine QUDA_DIRAC_DOMAIN_DECOMPOSITION
-#ifdef QUDA_DIRAC_DOMAIN_DECOMPOSITION
 /**
- * @def   GPU_DD_DIRAC
- * @brief This macro is set when we have Domain Decomposition (DD)-aware Dirac operator enabled
+ * @def   QUDA_DOMAIN_DECOMPOSITION
+ * @brief This macro sets the type of Domain Decomposition (DD)-aware Dirac operator enabled
  */
-#define GPU_DD_DIRAC
-#endif
+#define QUDA_DOMAIN_DECOMPOSITION @QUDA_DOMAIN_DECOMPOSITION@
 
 #cmakedefine QUDA_DIRAC_DISTANCE_PRECONDITIONING
 #ifdef QUDA_DIRAC_DISTANCE_PRECONDITIONING
diff --git a/tests/dslash_ctest.cpp b/tests/dslash_ctest.cpp
index bf49ec8b2d..21141f93a2 100644
--- a/tests/dslash_ctest.cpp
+++ b/tests/dslash_ctest.cpp
@@ -116,7 +116,7 @@ class DslashTest : public ::testing::TestWithParam<::testing::tuple<int, int, in
 
 TEST_P(DslashTest, verify)
 {
-  dslash_test_wrapper.dslashRef();
+  if (not dslash_test_wrapper.test_domain_decomposition) dslash_test_wrapper.dslashRef();
   dslash_test_wrapper.run_test(2);
 
   double deviation = dslash_test_wrapper.verify();
@@ -211,7 +211,7 @@ INSTANTIATE_TEST_SUITE_P(Regular, DslashTest,
                                  Range(0, N_PARTITIONS), ::testing::Values(0), ::testing::Values(0)),
                          getdslashtestname);
 
-#ifdef GPU_DD_DIRAC
+#if QUDA_DOMAIN_DECOMPOSITION > 0
 #define N_DD_TESTS 3
 
 // DD tests
diff --git a/tests/dslash_test.cpp b/tests/dslash_test.cpp
index e3290c2e2a..afc1462764 100644
--- a/tests/dslash_test.cpp
+++ b/tests/dslash_test.cpp
@@ -69,7 +69,7 @@ TEST_F(DslashTest, verify)
 {
   if (!verify_results) GTEST_SKIP();
 
-  dslash_test_wrapper.dslashRef();
+  if (not dslash_test_wrapper.test_domain_decomposition) dslash_test_wrapper.dslashRef();
   dslash_test_wrapper.run_test(2);
 
   double deviation = dslash_test_wrapper.verify();
diff --git a/tests/staggered_dslash_ctest.cpp b/tests/staggered_dslash_ctest.cpp
index f4db942dcd..f840494f8c 100644
--- a/tests/staggered_dslash_ctest.cpp
+++ b/tests/staggered_dslash_ctest.cpp
@@ -203,7 +203,7 @@ std::string getstaggereddslashtestname(testing::TestParamInfo<::testing::tuple<i
 #define N_PARTITIONS 1
 #endif
 
-#ifdef GPU_DD_DIRAC
+#if QUDA_DOMAIN_DECOMPOSITION > 0
 #define N_DD_TESTS 3
 #define N_DD_COLS 4
 #else

From b495ffedf637dca22a276b52afbe9645922f3283 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Tue, 3 Sep 2024 14:29:35 +0200
Subject: [PATCH 078/108] Adding is_enabled for DD and precision_type_mapper

---
 include/instantiate.h        | 32 ++++++++++++++++++++++++++++++++
 include/instantiate_dslash.h |  7 ++++---
 2 files changed, 36 insertions(+), 3 deletions(-)

diff --git a/include/instantiate.h b/include/instantiate.h
index 8eee6ad269..9b40ede127 100644
--- a/include/instantiate.h
+++ b/include/instantiate.h
@@ -84,6 +84,24 @@ namespace quda
     }
   }
 
+  /**
+     @brief precision_type_mapper Struct used to convert QudaPrecision to data-type.
+  */
+  template <QudaPrecision precision> struct precision_type_mapper {
+  };
+  template <> struct precision_type_mapper<QUDA_DOUBLE_PRECISION> {
+    using type = double;
+  };
+  template <> struct precision_type_mapper<QUDA_SINGLE_PRECISION> {
+    using type = float;
+  };
+  template <> struct precision_type_mapper<QUDA_HALF_PRECISION> {
+    using type = short;
+  };
+  template <> struct precision_type_mapper<QUDA_QUARTER_PRECISION> {
+    using type = int8_t;
+  };
+
   /**
      @brief Helper function for returning if a given reconstruct is enabled
      @tparam reconstruct The reconstruct requested
@@ -97,6 +115,20 @@ namespace quda
   template <> constexpr bool is_enabled<QUDA_RECONSTRUCT_8>() { return (QUDA_RECONSTRUCT & 1) ? true : false; }
   template <> constexpr bool is_enabled<QUDA_RECONSTRUCT_10>() { return true; }
 
+  /**
+     @brief Helper function for returning if a given domain decomposition is enabled
+     @tparam DD The domain decomposition requested
+     @return True if enabled, false if not
+  */
+  constexpr bool is_enabled(QudaDDType DD)
+  {
+    switch (DD) {
+    case QUDA_DD_NO: return true;
+    case QUDA_DD_RED_BLACK: return (QUDA_DOMAIN_DECOMPOSITION & 1) ? true : false;
+    default: return false;
+    }
+  }
+
   struct ReconstructFull {
     static constexpr std::array<QudaReconstructType, 6> recon
       = {QUDA_RECONSTRUCT_NO, QUDA_RECONSTRUCT_13, QUDA_RECONSTRUCT_12, QUDA_RECONSTRUCT_9, QUDA_RECONSTRUCT_8, QUDA_RECONSTRUCT_10};
diff --git a/include/instantiate_dslash.h b/include/instantiate_dslash.h
index bc248a7865..587f36e7ee 100644
--- a/include/instantiate_dslash.h
+++ b/include/instantiate_dslash.h
@@ -56,10 +56,11 @@ namespace quda
   {
     if (out.DD().type == QUDA_DD_NO and in.DD().type == QUDA_DD_NO) {
       instantiate<Apply, Recon, Float, 3, DDNo>(out, in, x, U, args...);
-#ifdef GPU_DD_DIRAC
     } else if (out.DD().type == QUDA_DD_RED_BLACK or in.DD().type == QUDA_DD_RED_BLACK) {
-      instantiate<Apply, Recon, Float, 3, DDRedBlack>(out, in, x, U, args...);
-#endif
+      if constexpr (is_enabled(QUDA_DD_RED_BLACK))
+        instantiate<Apply, Recon, Float, 3, DDRedBlack>(out, in, x, U, args...);
+      else
+        errorQuda("QUDA_DOMAIN_DECOMPOSITION=%d does not enable RedBlack", QUDA_DOMAIN_DECOMPOSITION);
     } else {
       errorQuda("Unsupported DD type %d\n", out.DD().type);
     }

From 029bdf73fafe297cb110c4bc8b25e0c4ef13e427 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Tue, 3 Sep 2024 14:30:09 +0200
Subject: [PATCH 079/108] Splitting compilation of dslash_wilson into multiple
 compilations units

---
 lib/CMakeLists.txt      | 69 +++++++++++++++++++++++++++++++++++++++++
 lib/dslash_wilson.cu    |  2 ++
 lib/dslash_wilson.hpp   |  4 +++
 lib/dslash_wilson.in.cu | 21 +++++++++++++
 4 files changed, 96 insertions(+)
 create mode 100644 lib/dslash_wilson.in.cu

diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt
index 4be29d2f68..1ef46b1e6d 100644
--- a/lib/CMakeLists.txt
+++ b/lib/CMakeLists.txt
@@ -186,6 +186,75 @@ configure_file(color_spinor_util.in.cu color_spinor_util.cu @ONLY)
 configure_file(dslash_coarse_mma.in.hpp dslash_coarse_mma.hpp @ONLY)
 configure_file(block_transpose.in.cu block_transpose.cu @ONLY)
 
+# Function to check if a specific bit is set
+function(check_bit number bit)
+  # Perform bitwise AND with (1 << bit) to check if the bit is set
+  math(EXPR result "${number} & (1 << ${bit})")
+  set(BIT_CHECK "${result}" PARENT_SCOPE)
+endfunction()
+
+set(QUDA_DSLASH_PREC_LIST "")
+set(QUDA_DSLASH_NCOLOR_LIST "3")
+set(QUDA_DSLASH_DDARG_LIST "DDNo")
+set(QUDA_DSLASH_RECONI_LIST "")
+
+check_bit(${QUDA_PRECISION} 0)
+if(BIT_CHECK)
+  list(APPEND QUDA_DSLASH_PREC_LIST "QUARTER")
+endif()
+
+check_bit(${QUDA_PRECISION} 1)
+if(BIT_CHECK)
+  list(APPEND QUDA_DSLASH_PREC_LIST "HALF")
+endif()
+
+check_bit(${QUDA_PRECISION} 2)
+if(BIT_CHECK)
+  list(APPEND QUDA_DSLASH_PREC_LIST "SINGLE")
+endif()
+
+check_bit(${QUDA_PRECISION} 3)
+if(BIT_CHECK)
+  list(APPEND QUDA_DSLASH_PREC_LIST "DOUBLE")
+endif()
+
+check_bit(${QUDA_RECONSTRUCT} 0)
+if(BIT_CHECK)
+  list(APPEND QUDA_DSLASH_RECONI_LIST "2")
+endif()
+
+check_bit(${QUDA_RECONSTRUCT} 1)
+if(BIT_CHECK)
+  list(APPEND QUDA_DSLASH_RECONI_LIST "1")
+endif()
+
+check_bit(${QUDA_RECONSTRUCT} 2)
+if(BIT_CHECK)
+  list(APPEND QUDA_DSLASH_RECONI_LIST "0")
+endif()
+
+check_bit(${QUDA_DOMAIN_DECOMPOSITION} 0)
+if(BIT_CHECK)
+  list(APPEND QUDA_DSLASH_DDARG_LIST "DDRedBlack")
+endif()
+
+message(STATUS "QUDA_DSLASH_PREC_LIST=${QUDA_DSLASH_PREC_LIST}")
+message(STATUS "QUDA_DSLASH_NCOLOR_LIST=${QUDA_DSLASH_NCOLOR_LIST}")
+message(STATUS "QUDA_DSLASH_DDARG_LIST=${QUDA_DSLASH_DDARG_LIST}")
+message(STATUS "QUDA_DSLASH_RECONI_LIST=${QUDA_DSLASH_RECONI_LIST}")
+
+foreach(QUDA_DSLASH_PREC ${QUDA_DSLASH_PREC_LIST})
+  foreach(QUDA_DSLASH_NCOLOR ${QUDA_DSLASH_NCOLOR_LIST})
+    foreach(QUDA_DSLASH_DDARG ${QUDA_DSLASH_DDARG_LIST})
+      foreach(QUDA_DSLASH_RECONI ${QUDA_DSLASH_RECONI_LIST})
+        configure_file(dslash_wilson.in.cu "dslash_wilson_${QUDA_DSLASH_PREC}_${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_${QUDA_DSLASH_RECONI}.cu" @ONLY)
+        list(PREPEND QUDA_CU_OBJS "dslash_wilson_${QUDA_DSLASH_PREC}_${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_${QUDA_DSLASH_RECONI}.cu")
+      endforeach()
+    endforeach()
+  endforeach()
+endforeach()
+
+
 if(QUDA_MULTIGRID)
   string(REPLACE "," ";" QUDA_MULTIGRID_NVEC_LIST_SEMICOLON "${QUDA_MULTIGRID_NVEC_LIST}")
   string(REPLACE "," ";" QUDA_MULTIGRID_MRHS_LIST_SEMICOLON "${QUDA_MULTIGRID_MRHS_LIST}")
diff --git a/lib/dslash_wilson.cu b/lib/dslash_wilson.cu
index 0dc7f8ebb1..414cb19495 100644
--- a/lib/dslash_wilson.cu
+++ b/lib/dslash_wilson.cu
@@ -1,4 +1,6 @@
+#define SIGNATURE_ONLY
 #include <dslash_wilson.hpp>
+#undef SIGNATURE_ONLY
 
 namespace quda
 {
diff --git a/lib/dslash_wilson.hpp b/lib/dslash_wilson.hpp
index 63efcba966..3ccc35bc13 100644
--- a/lib/dslash_wilson.hpp
+++ b/lib/dslash_wilson.hpp
@@ -43,6 +43,9 @@ namespace quda
     WilsonApply(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
                 cvector_ref<const ColorSpinorField> &x, const GaugeField &U, double a, double alpha0, int t0,
                 int parity, bool dagger, const int *comm_override, DistanceType<distance_pc>, TimeProfile &profile)
+#ifdef SIGNATURE_ONLY
+      ;
+#else
     {
       constexpr int nDim = 4;
       auto halo = ColorSpinorField::create_comms_batch(in);
@@ -51,6 +54,7 @@ namespace quda
       Wilson<decltype(arg)> wilson(arg, out, in, halo);
       dslash::DslashPolicyTune<decltype(wilson)> policy(wilson, in, halo, profile);
     }
+#endif
   };
 
 } // namespace quda
diff --git a/lib/dslash_wilson.in.cu b/lib/dslash_wilson.in.cu
new file mode 100644
index 0000000000..eb775b7f59
--- /dev/null
+++ b/lib/dslash_wilson.in.cu
@@ -0,0 +1,21 @@
+#include <dslash_wilson.hpp>
+
+namespace quda
+{
+
+  constexpr QudaPrecision precision = QUDA_@QUDA_DSLASH_PREC@_PRECISION;
+  constexpr int nColor = @QUDA_DSLASH_NCOLOR@;
+  constexpr int reconI = @QUDA_DSLASH_RECONI@;
+  constexpr bool distance_pc = false;
+
+  typedef @QUDA_DSLASH_DDARG@DDArg;
+  typedef precision_type_mapper<precision>::type Float;
+
+  template struct WilsonApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>;
+
+  template WilsonApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>::WilsonApply<distance_pc>(
+    cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in, cvector_ref<const ColorSpinorField> &x,
+    const GaugeField &U, double a, double alpha0, int t0, int parity, bool dagger, const int *comm_override,
+    DistanceType<distance_pc>, TimeProfile &profile);
+
+} // namespace quda

From c2dd7978eb4ab1228a5137d3802f3f35eae31082 Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Mon, 16 Sep 2024 12:15:52 +0200
Subject: [PATCH 080/108] Fixing typo

---
 lib/dslash_wilson.in.cu | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/dslash_wilson.in.cu b/lib/dslash_wilson.in.cu
index eb775b7f59..f824aba4b8 100644
--- a/lib/dslash_wilson.in.cu
+++ b/lib/dslash_wilson.in.cu
@@ -8,7 +8,7 @@ namespace quda
   constexpr int reconI = @QUDA_DSLASH_RECONI@;
   constexpr bool distance_pc = false;
 
-  typedef @QUDA_DSLASH_DDARG@DDArg;
+  typedef @QUDA_DSLASH_DDARG@ DDArg;
   typedef precision_type_mapper<precision>::type Float;
 
   template struct WilsonApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>;

From 855f64e81ba6dc2a0548eb1224327d74468c2ffc Mon Sep 17 00:00:00 2001
From: Simone Bacchio <s.bacchio@gmail.com>
Date: Mon, 16 Sep 2024 14:32:08 +0200
Subject: [PATCH 081/108] extending single compilation units to distance
 preconditioning

---
 lib/CMakeLists.txt            | 13 +++++++++++--
 lib/dslash_wilson.hpp         |  2 +-
 lib/dslash_wilson.in.cu       |  2 +-
 lib/dslash_wilson_distance.cu |  2 ++
 4 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt
index 1ef46b1e6d..7cfaf378f7 100644
--- a/lib/CMakeLists.txt
+++ b/lib/CMakeLists.txt
@@ -197,6 +197,11 @@ set(QUDA_DSLASH_PREC_LIST "")
 set(QUDA_DSLASH_NCOLOR_LIST "3")
 set(QUDA_DSLASH_DDARG_LIST "DDNo")
 set(QUDA_DSLASH_RECONI_LIST "")
+set(QUDA_DSLASH_DISTANCE_LIST "false")
+
+if(QUDA_DIRAC_DISTANCE_PRECONDITIONING)
+  list(APPEND QUDA_DSLASH_DISTANCE_LIST "true")
+endif()
 
 check_bit(${QUDA_PRECISION} 0)
 if(BIT_CHECK)
@@ -242,13 +247,17 @@ message(STATUS "QUDA_DSLASH_PREC_LIST=${QUDA_DSLASH_PREC_LIST}")
 message(STATUS "QUDA_DSLASH_NCOLOR_LIST=${QUDA_DSLASH_NCOLOR_LIST}")
 message(STATUS "QUDA_DSLASH_DDARG_LIST=${QUDA_DSLASH_DDARG_LIST}")
 message(STATUS "QUDA_DSLASH_RECONI_LIST=${QUDA_DSLASH_RECONI_LIST}")
+message(STATUS "QUDA_DSLASH_DISTANCE_LIST=${QUDA_DSLASH_DISTANCE_LIST}")
 
 foreach(QUDA_DSLASH_PREC ${QUDA_DSLASH_PREC_LIST})
   foreach(QUDA_DSLASH_NCOLOR ${QUDA_DSLASH_NCOLOR_LIST})
     foreach(QUDA_DSLASH_DDARG ${QUDA_DSLASH_DDARG_LIST})
       foreach(QUDA_DSLASH_RECONI ${QUDA_DSLASH_RECONI_LIST})
-        configure_file(dslash_wilson.in.cu "dslash_wilson_${QUDA_DSLASH_PREC}_${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_${QUDA_DSLASH_RECONI}.cu" @ONLY)
-        list(PREPEND QUDA_CU_OBJS "dslash_wilson_${QUDA_DSLASH_PREC}_${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_${QUDA_DSLASH_RECONI}.cu")
+	foreach(QUDA_DSLASH_RECONI ${QUDA_DSLASH_RECONI_LIST})
+	  set(TMP_FILENAME "dslash_wilson_${QUDA_DSLASH_PREC}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
+          configure_file(dslash_wilson.in.cu ${TMP_FILENAME} @ONLY)
+          list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
+	endforeach()
       endforeach()
     endforeach()
   endforeach()
diff --git a/lib/dslash_wilson.hpp b/lib/dslash_wilson.hpp
index 3ccc35bc13..78a864f640 100644
--- a/lib/dslash_wilson.hpp
+++ b/lib/dslash_wilson.hpp
@@ -43,7 +43,7 @@ namespace quda
     WilsonApply(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
                 cvector_ref<const ColorSpinorField> &x, const GaugeField &U, double a, double alpha0, int t0,
                 int parity, bool dagger, const int *comm_override, DistanceType<distance_pc>, TimeProfile &profile)
-#ifdef SIGNATURE_ONLY
+#ifdef SIGNATURE_ONLY //Used to hide from the compiler the implementation of the function
       ;
 #else
     {
diff --git a/lib/dslash_wilson.in.cu b/lib/dslash_wilson.in.cu
index f824aba4b8..a912d4cf03 100644
--- a/lib/dslash_wilson.in.cu
+++ b/lib/dslash_wilson.in.cu
@@ -6,7 +6,7 @@ namespace quda
   constexpr QudaPrecision precision = QUDA_@QUDA_DSLASH_PREC@_PRECISION;
   constexpr int nColor = @QUDA_DSLASH_NCOLOR@;
   constexpr int reconI = @QUDA_DSLASH_RECONI@;
-  constexpr bool distance_pc = false;
+  constexpr bool distance_pc = @QUDA_DSLASH_DISTANCE@;
 
   typedef @QUDA_DSLASH_DDARG@ DDArg;
   typedef precision_type_mapper<precision>::type Float;
diff --git a/lib/dslash_wilson_distance.cu b/lib/dslash_wilson_distance.cu
index 414924f992..aad9d3a7e6 100644
--- a/lib/dslash_wilson_distance.cu
+++ b/lib/dslash_wilson_distance.cu
@@ -1,4 +1,6 @@
+#define SIGNATURE_ONLY
 #include <dslash_wilson.hpp>
+#undef SIGNATURE_ONLY
 
 namespace quda
 {

From 962554f939cc7ce16ae20a62a9ed99c144286921 Mon Sep 17 00:00:00 2001
From: Ferenc Pittler <pittler@hiskp.uni-bonn.de>
Date: Tue, 8 Oct 2024 20:35:45 +0200
Subject: [PATCH 082/108] correcting travis-ci

---
 ci/docker/Dockerfile.build | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ci/docker/Dockerfile.build b/ci/docker/Dockerfile.build
index 5ca1d7a3dc..3019b28c38 100644
--- a/ci/docker/Dockerfile.build
+++ b/ci/docker/Dockerfile.build
@@ -39,6 +39,7 @@ RUN  QUDA_TEST_GRID_SIZE="1 1 1 2" cmake -S /quda/src \
     -DQUDA_MULTIGRID_NVEC_LIST=6 \
     -DQUDA_MDW_FUSED_LS_LIST=4 \
     -DQUDA_MPI=ON  \
+    -DQUDA_DSLASH_DISTANCE=1 \
     -DQUDA_DIRAC_DEFAULT_OFF=ON \
     -DQUDA_DIRAC_WILSON=ON \
     -DQUDA_DIRAC_CLOVER=ON \

From bce6914d713c1c6d3027fc437af18b3726e1da90 Mon Sep 17 00:00:00 2001
From: Ferenc Pittler <pittler@hiskp.uni-bonn.de>
Date: Wed, 9 Oct 2024 08:28:59 +0200
Subject: [PATCH 083/108] correcting typo

---
 lib/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt
index 7cfaf378f7..6bcb7c7dff 100644
--- a/lib/CMakeLists.txt
+++ b/lib/CMakeLists.txt
@@ -253,7 +253,7 @@ foreach(QUDA_DSLASH_PREC ${QUDA_DSLASH_PREC_LIST})
   foreach(QUDA_DSLASH_NCOLOR ${QUDA_DSLASH_NCOLOR_LIST})
     foreach(QUDA_DSLASH_DDARG ${QUDA_DSLASH_DDARG_LIST})
       foreach(QUDA_DSLASH_RECONI ${QUDA_DSLASH_RECONI_LIST})
-	foreach(QUDA_DSLASH_RECONI ${QUDA_DSLASH_RECONI_LIST})
+	foreach(QUDA_DSLASH_DISTANCE ${QUDA_DSLASH_DISTANCE_LIST})
 	  set(TMP_FILENAME "dslash_wilson_${QUDA_DSLASH_PREC}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
           configure_file(dslash_wilson.in.cu ${TMP_FILENAME} @ONLY)
           list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})

From 279c12339c9a7efeeaa66221e2c2aa2bb3a72364 Mon Sep 17 00:00:00 2001
From: Ferenc Pittler <pittler@hiskp.uni-bonn.de>
Date: Sat, 12 Oct 2024 13:20:41 +0200
Subject: [PATCH 084/108] seprate compilation for twisted clover

---
 lib/CMakeLists.txt              |  3 ++
 lib/dslash_twisted_clover.cu    | 78 ++-------------------------
 lib/dslash_twisted_clover.hpp   | 93 +++++++++++++++++++++++++++++++++
 lib/dslash_twisted_clover.in.cu | 23 ++++++++
 4 files changed, 122 insertions(+), 75 deletions(-)
 create mode 100644 lib/dslash_twisted_clover.hpp
 create mode 100644 lib/dslash_twisted_clover.in.cu

diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt
index 6bcb7c7dff..eeb6772917 100644
--- a/lib/CMakeLists.txt
+++ b/lib/CMakeLists.txt
@@ -257,6 +257,9 @@ foreach(QUDA_DSLASH_PREC ${QUDA_DSLASH_PREC_LIST})
 	  set(TMP_FILENAME "dslash_wilson_${QUDA_DSLASH_PREC}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
           configure_file(dslash_wilson.in.cu ${TMP_FILENAME} @ONLY)
           list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
+	  set(TMP_FILENAME "dslash_twisted_clover_${QUDA_DSLASH_PREC}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
+          configure_file(dslash_twisted_clover.in.cu ${TMP_FILENAME} @ONLY)
+          list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
 	endforeach()
       endforeach()
     endforeach()
diff --git a/lib/dslash_twisted_clover.cu b/lib/dslash_twisted_clover.cu
index 4a1f8f6291..7e9b4747b1 100644
--- a/lib/dslash_twisted_clover.cu
+++ b/lib/dslash_twisted_clover.cu
@@ -1,11 +1,7 @@
-#include <gauge_field.h>
-#include <color_spinor_field.h>
-#include <clover_field.h>
-#include <dslash.h>
-#include <worker.h>
+#define SIGNATURE_ONLY
+#include <dslash_twisted_clover.hpp>
+#undef SIGNATURE_ONLY
 
-#include <dslash_policy.hpp>
-#include <kernels/dslash_wilson_clover.cuh>
 
 /**
    This is the basic gauged twisted-clover operator
@@ -14,74 +10,6 @@
 namespace quda
 {
 
-  template <typename Arg> class TwistedClover : public Dslash<wilsonClover, Arg>
-  {
-    using Dslash = Dslash<wilsonClover, Arg>;
-    using Dslash::arg;
-    using Dslash::halo;
-    using Dslash::in;
-
-  public:
-    TwistedClover(Arg &arg, cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
-                  const ColorSpinorField &halo) :
-      Dslash(arg, out, in, halo)
-    {
-    }
-
-    void apply(const qudaStream_t &stream)
-    {
-      TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
-      Dslash::setParam(tp);
-      if (arg.xpay)
-        this->template instantiate<packShmem, true>(tp, stream);
-      else
-        errorQuda("Twisted-clover operator only defined for xpay=true");
-    }
-
-    long long flops() const
-    {
-      int clover_flops = 504 + 48;
-      long long flops = Dslash::flops();
-      switch (arg.kernel_type) {
-      case INTERIOR_KERNEL:
-      case UBER_KERNEL:
-      case KERNEL_POLICY: flops += clover_flops * halo.Volume(); break;
-      default: break; // all clover flops are in the interior kernel
-      }
-      return flops;
-    }
-
-    long long bytes() const
-    {
-      int clover_bytes = 72 * in.Precision() + (isFixed<typename Arg::Float>::value ? 2 * sizeof(float) : 0);
-
-      long long bytes = Dslash::bytes();
-      switch (arg.kernel_type) {
-      case INTERIOR_KERNEL:
-      case UBER_KERNEL:
-      case KERNEL_POLICY: bytes += clover_bytes * halo.Volume(); break;
-      default: break;
-      }
-
-      return bytes;
-    }
-  };
-
-  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon> struct TwistedCloverApply {
-
-    TwistedCloverApply(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
-                       cvector_ref<const ColorSpinorField> &x, const GaugeField &U, const CloverField &C, double a,
-                       double b, int parity, bool dagger, const int *comm_override, TimeProfile &profile)
-    {
-      constexpr int nDim = 4;
-      auto halo = ColorSpinorField::create_comms_batch(in);
-      WilsonCloverArg<Float, nColor, nDim, DDArg, recon, true> arg(out, in, halo, U, C, a, b, x, parity, dagger,
-                                                                   comm_override);
-      TwistedClover<decltype(arg)> twisted(arg, out, in, halo);
-      dslash::DslashPolicyTune<decltype(twisted)> policy(twisted, in, halo, profile);
-    }
-  };
-
   // Apply the twisted-mass Dslash operator
   // out(x) = M*in = (A + i*b*gamma_5)*in(x) + a*\sum_mu U_{-\mu}(x)in(x+mu) + U^\dagger_mu(x-mu)in(x-mu)
   // Uses the kappa normalization for the Wilson operator, with a = -kappa.
diff --git a/lib/dslash_twisted_clover.hpp b/lib/dslash_twisted_clover.hpp
new file mode 100644
index 0000000000..b37ac649d7
--- /dev/null
+++ b/lib/dslash_twisted_clover.hpp
@@ -0,0 +1,93 @@
+#include <gauge_field.h>
+#include <color_spinor_field.h>
+#include <clover_field.h>
+#include <dslash.h>
+#include <worker.h>
+
+#include <dslash_policy.hpp>
+#include <kernels/dslash_wilson_clover.cuh>
+
+/**
+   This is the basic gauged twisted-clover operator
+*/
+
+namespace quda
+{
+
+  template <typename Arg> class TwistedClover : public Dslash<wilsonClover, Arg>
+  {
+    using Dslash = Dslash<wilsonClover, Arg>;
+    using Dslash::arg;
+    using Dslash::halo;
+    using Dslash::in;
+
+  public:
+    TwistedClover(Arg &arg, cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+                  const ColorSpinorField &halo) :
+      Dslash(arg, out, in, halo)
+    {
+    }
+
+    void apply(const qudaStream_t &stream)
+    {
+      TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
+      Dslash::setParam(tp);
+      if (arg.xpay)
+        Dslash::template instantiate<packShmem, true>(tp, stream);
+      else
+        errorQuda("Twisted-clover operator only defined for xpay=true");
+    }
+
+    long long flops() const
+    {
+      int clover_flops = 504 + 48;
+      long long flops = Dslash::flops();
+      switch (arg.kernel_type) {
+      case INTERIOR_KERNEL:
+      case UBER_KERNEL:
+      case KERNEL_POLICY: flops += clover_flops * halo.Volume(); break;
+      default: break; // all clover flops are in the interior kernel
+      }
+      return flops;
+    }
+
+    long long bytes() const
+    {
+      int clover_bytes = 72 * in.Precision() + (isFixed<typename Arg::Float>::value ? 2 * sizeof(float) : 0);
+
+      long long bytes = Dslash::bytes();
+      switch (arg.kernel_type) {
+      case INTERIOR_KERNEL:
+      case UBER_KERNEL:
+      case KERNEL_POLICY: bytes += clover_bytes * halo.Volume(); break;
+      default: break;
+      }
+
+      return bytes;
+    }
+  };
+
+  template <bool distance_pc> struct DistanceType {
+  };
+
+
+  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon> struct TwistedCloverApply {
+    template <bool distance_pc>
+    TwistedCloverApply(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+                       cvector_ref<const ColorSpinorField> &x, const GaugeField &U, const CloverField &C, double a,
+                       double b, int parity, bool dagger, const int *comm_override, DistanceType<distance_pc>,TimeProfile &profile)
+#ifdef SIGNATURE_ONLY
+    ;
+#else
+    {
+      constexpr int nDim = 4;
+      auto halo = ColorSpinorField::create_comms_batch(in);
+      WilsonCloverArg<Float, nColor, nDim, DDArg, recon, true,distance_pc> arg(out, in, halo, U, C, a, b, x, parity, dagger,
+                                                                   comm_override);
+      TwistedClover<decltype(arg)> twisted(arg, out, in, halo);
+      dslash::DslashPolicyTune<decltype(twisted)> policy(twisted, in, halo, profile);
+    }
+#endif
+  };
+
+} // namespace quda
diff --git a/lib/dslash_twisted_clover.in.cu b/lib/dslash_twisted_clover.in.cu
new file mode 100644
index 0000000000..8a597c5038
--- /dev/null
+++ b/lib/dslash_twisted_clover.in.cu
@@ -0,0 +1,23 @@
+#include <dslash_twisted_clover.hpp>
+
+namespace quda
+{
+
+  constexpr QudaPrecision precision = QUDA_@QUDA_DSLASH_PREC@_PRECISION;
+  constexpr int nColor = @QUDA_DSLASH_NCOLOR@;
+  constexpr int reconI = @QUDA_DSLASH_RECONI@;
+  constexpr bool distance_pc = @QUDA_DSLASH_DISTANCE@;
+
+  typedef @QUDA_DSLASH_DDARG@ DDArg;
+  typedef precision_type_mapper<precision>::type Float;
+
+  template struct TwistedCloverApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>;
+  
+  template TwistedCloverApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>::TwistedCloverApply<distance_pc>(
+		  cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in, 
+		  cvector_ref<const ColorSpinorField> &x,const GaugeField &U, 
+		  const CloverField &C, double a, double b, int parity, 
+		  bool dagger, const int *comm_override,
+		  DistanceType<distance_pc>, TimeProfile &profile);
+
+} // namespace quda

From 268ca0b9bce7f64546b32f211c12e89effe3f1e0 Mon Sep 17 00:00:00 2001
From: Ferenc Pittler <pittler@hiskp.uni-bonn.de>
Date: Sat, 12 Oct 2024 19:09:56 +0200
Subject: [PATCH 085/108] correcting error

---
 lib/dslash_twisted_clover.cu | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/dslash_twisted_clover.cu b/lib/dslash_twisted_clover.cu
index 7e9b4747b1..a00d2ad1e6 100644
--- a/lib/dslash_twisted_clover.cu
+++ b/lib/dslash_twisted_clover.cu
@@ -19,7 +19,8 @@ namespace quda
                           TimeProfile &profile)
   {
     if constexpr (is_enabled<QUDA_TWISTED_CLOVER_DSLASH>()) {
-      instantiate<TwistedCloverApply>(out, in, x, U, C, a, b, parity, dagger, comm_override, profile);
+      auto dummy = DistanceType<false>();
+      instantiate<TwistedCloverApply>(out, in, x, U, C, a, b, parity, dagger, comm_override, dummy, profile);
     } else {
       errorQuda("Twisted-clover operator has not been built");
     }

From cc669f54cc70adc6e3cee26e3e35e7b608931950 Mon Sep 17 00:00:00 2001
From: Ferenc Pittler <pittler@hiskp.uni-bonn.de>
Date: Mon, 14 Oct 2024 09:06:35 +0200
Subject: [PATCH 086/108] separate compilation units for several dslash

---
 lib/CMakeLists.txt                            |  22 ++-
 lib/dslash_ndeg_twisted_clover.cu             |  89 +----------
 lib/dslash_ndeg_twisted_clover.hpp            | 100 ++++++++++++
 lib/dslash_ndeg_twisted_clover.in.cu          |  20 +++
 ...lash_ndeg_twisted_clover_preconditioned.cu | 110 +-------------
 ...ash_ndeg_twisted_clover_preconditioned.hpp | 124 +++++++++++++++
 ...h_ndeg_twisted_clover_preconditioned.in.cu |  24 +++
 lib/dslash_ndeg_twisted_mass.cu               |  66 +-------
 lib/dslash_ndeg_twisted_mass.hpp              |  77 ++++++++++
 lib/dslash_ndeg_twisted_mass.in.cu            |  20 +++
 ...dslash_ndeg_twisted_mass_preconditioned.cu | 114 +-------------
 ...slash_ndeg_twisted_mass_preconditioned.hpp | 126 ++++++++++++++++
 ...ash_ndeg_twisted_mass_preconditioned.in.cu |  23 +++
 lib/dslash_twisted_clover_preconditioned.cu   | 129 +---------------
 lib/dslash_twisted_clover_preconditioned.hpp  | 142 ++++++++++++++++++
 ...dslash_twisted_clover_preconditioned.in.cu |  24 +++
 lib/dslash_twisted_mass.cu                    |  68 +--------
 lib/dslash_twisted_mass.hpp                   |  74 +++++++++
 lib/dslash_twisted_mass.in.cu                 |  20 +++
 lib/dslash_wilson_clover.cu                   |   2 +
 lib/dslash_wilson_clover.hpp                  |   5 +-
 lib/dslash_wilson_clover.in.cu                |  27 ++++
 22 files changed, 858 insertions(+), 548 deletions(-)
 create mode 100644 lib/dslash_ndeg_twisted_clover.hpp
 create mode 100644 lib/dslash_ndeg_twisted_clover.in.cu
 create mode 100644 lib/dslash_ndeg_twisted_clover_preconditioned.hpp
 create mode 100644 lib/dslash_ndeg_twisted_clover_preconditioned.in.cu
 create mode 100644 lib/dslash_ndeg_twisted_mass.hpp
 create mode 100644 lib/dslash_ndeg_twisted_mass.in.cu
 create mode 100644 lib/dslash_ndeg_twisted_mass_preconditioned.hpp
 create mode 100644 lib/dslash_ndeg_twisted_mass_preconditioned.in.cu
 create mode 100644 lib/dslash_twisted_clover_preconditioned.hpp
 create mode 100644 lib/dslash_twisted_clover_preconditioned.in.cu
 create mode 100644 lib/dslash_twisted_mass.hpp
 create mode 100644 lib/dslash_twisted_mass.in.cu
 create mode 100644 lib/dslash_wilson_clover.in.cu

diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt
index eeb6772917..287c19ff99 100644
--- a/lib/CMakeLists.txt
+++ b/lib/CMakeLists.txt
@@ -260,13 +260,33 @@ foreach(QUDA_DSLASH_PREC ${QUDA_DSLASH_PREC_LIST})
 	  set(TMP_FILENAME "dslash_twisted_clover_${QUDA_DSLASH_PREC}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
           configure_file(dslash_twisted_clover.in.cu ${TMP_FILENAME} @ONLY)
           list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
+	  set(TMP_FILENAME "dslash_wilson_clover_${QUDA_DSLASH_PREC}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
+          configure_file(dslash_wilson_clover.in.cu ${TMP_FILENAME} @ONLY)
+          list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
+          set(TMP_FILENAME "dslash_twisted_mass_${QUDA_DSLASH_PREC}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
+          configure_file(dslash_twisted_mass.in.cu ${TMP_FILENAME} @ONLY)
+          list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
+          set(TMP_FILENAME "dslash_ndeg_twisted_clover_${QUDA_DSLASH_PREC}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
+          configure_file(dslash_ndeg_twisted_clover.in.cu ${TMP_FILENAME} @ONLY)
+          list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
+          set(TMP_FILENAME "dslash_ndeg_twisted_clover_preconditioned.${QUDA_DSLASH_PREC}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
+          configure_file(dslash_ndeg_twisted_clover_preconditioned.in.cu ${TMP_FILENAME} @ONLY)
+          list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
+	  set(TMP_FILENAME "dslash_ndeg_twisted_mass_preconditioned.${QUDA_DSLASH_PREC}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
+          configure_file(dslash_ndeg_twisted_mass_preconditioned.in.cu ${TMP_FILENAME} @ONLY)
+          list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
+          set(TMP_FILENAME "dslash_ndeg_twisted_mass_${QUDA_DSLASH_PREC}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
+          configure_file(dslash_ndeg_twisted_mass.in.cu ${TMP_FILENAME} @ONLY)
+          list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
+          set(TMP_FILENAME "dslash_twisted_clover_preconditioned.${QUDA_DSLASH_PREC}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
+          configure_file(dslash_twisted_clover_preconditioned.in.cu ${TMP_FILENAME} @ONLY)
+          list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
 	endforeach()
       endforeach()
     endforeach()
   endforeach()
 endforeach()
 
-
 if(QUDA_MULTIGRID)
   string(REPLACE "," ";" QUDA_MULTIGRID_NVEC_LIST_SEMICOLON "${QUDA_MULTIGRID_NVEC_LIST}")
   string(REPLACE "," ";" QUDA_MULTIGRID_MRHS_LIST_SEMICOLON "${QUDA_MULTIGRID_MRHS_LIST}")
diff --git a/lib/dslash_ndeg_twisted_clover.cu b/lib/dslash_ndeg_twisted_clover.cu
index c4016a53f5..04e9bb94f1 100644
--- a/lib/dslash_ndeg_twisted_clover.cu
+++ b/lib/dslash_ndeg_twisted_clover.cu
@@ -1,11 +1,6 @@
-#include <gauge_field.h>
-#include <color_spinor_field.h>
-#include <clover_field.h>
-#include <dslash.h>
-#include <worker.h>
-
-#include <dslash_policy.hpp>
-#include <kernels/dslash_ndeg_twisted_clover.cuh>
+#define SIGNATURE_ONLY
+#include <dslash_ndeg_twisted_clover.hpp>
+#undef SIGNATURE_ONLY
 
 /**
    This is the gauged non-degenerate twisted-clover operator acting on a 
@@ -15,88 +10,14 @@
 namespace quda
 {
 
-  template <typename Arg> class NdegTwistedClover : public Dslash<nDegTwistedClover, Arg>
-    {
-      using Dslash = Dslash<nDegTwistedClover, Arg>;
-      using Dslash::arg;
-      using Dslash::halo;
-      using Dslash::in;
-
-      unsigned int sharedBytesPerThread() const
-      {
-        return 2 * in.Ncolor() * 4 * sizeof(typename mapper<typename Arg::Float>::type);
-      }
-
-    public:
-      NdegTwistedClover(Arg &arg, cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
-                        const ColorSpinorField &halo) :
-        Dslash(arg, out, in, halo)
-      {
-        TunableKernel3D::resizeStep(2, 1);
-      }
-
-      void apply(const qudaStream_t &stream)
-      {
-        TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
-        Dslash::setParam(tp);
-        if (arg.xpay)
-          Dslash::template instantiate<packShmem, true>(tp, stream);
-        else
-          errorQuda("Non-degenerate twisted-clover operator only defined for xpay=true");
-      }
-      
-      long long flops() const
-      {
-        int clover_flops = 504;
-        long long flops = Dslash::flops();
-        switch (arg.kernel_type) {
-        case INTERIOR_KERNEL:
-        case KERNEL_POLICY:
-          // b and c multiply (= 2 * 48 * in.Volume())
-          flops += 2 * in.Ncolor() * 4 * 4 * halo.Volume(); // complex * Nc * Ns * fma * vol
-          flops += clover_flops * halo.Volume();
-          break;
-        default: break; // twisted-mass flops are in the interior kernel
-        }
-        return flops;
-      }
-      long long bytes() const
-      {
-        int clover_bytes = 72 * in.Precision() + (isFixed<typename Arg::Float>::value ? 2 * sizeof(float) : 0);
-        
-        long long bytes = Dslash::bytes();
-        switch (arg.kernel_type) {
-        case INTERIOR_KERNEL:
-        case KERNEL_POLICY: bytes += clover_bytes * halo.Volume(); break;
-        default: break;
-        }
-        
-        return bytes;
-      }
-    };
-
-    template <typename Float, int nColor, typename DDArg, QudaReconstructType recon> struct NdegTwistedCloverApply {
-
-      NdegTwistedCloverApply(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
-                             cvector_ref<const ColorSpinorField> &x, const GaugeField &U, const CloverField &A, double a,
-                             double b, double c, int parity, bool dagger, const int *comm_override, TimeProfile &profile)
-      {
-        constexpr int nDim = 4;
-        auto halo = ColorSpinorField::create_comms_batch(in);
-        NdegTwistedCloverArg<Float, nColor, nDim, DDArg, recon> arg(out, in, halo, U, A, a, b, c, x, parity, dagger,
-                                                                    comm_override);
-        NdegTwistedClover<decltype(arg)> twisted(arg, out, in, halo);
-        dslash::DslashPolicyTune<decltype(twisted)> policy(twisted, in, halo, profile);
-      }
-    };
-
     void ApplyNdegTwistedClover(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
                                 const GaugeField &U, const CloverField &A, double a, double b, double c,
                                 cvector_ref<const ColorSpinorField> &x, int parity, bool dagger,
                                 const int *comm_override, TimeProfile &profile)
     {
       if constexpr (is_enabled<QUDA_TWISTED_CLOVER_DSLASH>()) {
-        instantiate<NdegTwistedCloverApply>(out, in, x, U, A, a, b, c, parity, dagger, comm_override, profile);
+        auto dummy = DistanceType<false>();
+        instantiate<NdegTwistedCloverApply>(out, in, x, U, A, a, b, c, parity, dagger, comm_override, dummy, profile);
       } else {
         errorQuda("Non-degenerate twisted-clover operator has not been built");
       }
diff --git a/lib/dslash_ndeg_twisted_clover.hpp b/lib/dslash_ndeg_twisted_clover.hpp
new file mode 100644
index 0000000000..285aab6c3c
--- /dev/null
+++ b/lib/dslash_ndeg_twisted_clover.hpp
@@ -0,0 +1,100 @@
+#include <gauge_field.h>
+#include <color_spinor_field.h>
+#include <clover_field.h>
+#include <dslash.h>
+#include <worker.h>
+
+#include <dslash_policy.hpp>
+#include <kernels/dslash_ndeg_twisted_clover.cuh>
+
+/**
+   This is the gauged non-degenerate twisted-clover operator acting on a 
+   quark doublet.
+*/
+
+namespace quda
+{
+
+  template <typename Arg> class NdegTwistedClover : public Dslash<nDegTwistedClover, Arg>
+    {
+      using Dslash = Dslash<nDegTwistedClover, Arg>;
+      using Dslash::arg;
+      using Dslash::halo;
+      using Dslash::in;
+
+      unsigned int sharedBytesPerThread() const
+      {
+        return 2 * in.Ncolor() * 4 * sizeof(typename mapper<typename Arg::Float>::type);
+      }
+
+    public:
+      NdegTwistedClover(Arg &arg, cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+                        const ColorSpinorField &halo) :
+        Dslash(arg, out, in, halo)
+      {
+        TunableKernel3D::resizeStep(2, 1);
+      }
+
+      void apply(const qudaStream_t &stream)
+      {
+        TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
+        Dslash::setParam(tp);
+        if (arg.xpay)
+          Dslash::template instantiate<packShmem, true>(tp, stream);
+        else
+          errorQuda("Non-degenerate twisted-clover operator only defined for xpay=true");
+      }
+      
+      long long flops() const
+      {
+        int clover_flops = 504;
+        long long flops = Dslash::flops();
+        switch (arg.kernel_type) {
+        case INTERIOR_KERNEL:
+        case KERNEL_POLICY:
+          // b and c multiply (= 2 * 48 * in.Volume())
+          flops += 2 * in.Ncolor() * 4 * 4 * halo.Volume(); // complex * Nc * Ns * fma * vol
+          flops += clover_flops * halo.Volume();
+          break;
+        default: break; // twisted-mass flops are in the interior kernel
+        }
+        return flops;
+      }
+      long long bytes() const
+      {
+        int clover_bytes = 72 * in.Precision() + (isFixed<typename Arg::Float>::value ? 2 * sizeof(float) : 0);
+        
+        long long bytes = Dslash::bytes();
+        switch (arg.kernel_type) {
+        case INTERIOR_KERNEL:
+        case KERNEL_POLICY: bytes += clover_bytes * halo.Volume(); break;
+        default: break;
+        }
+        
+        return bytes;
+      }
+    };
+
+    template <bool distance_pc> struct DistanceType {
+    };
+
+    template <typename Float, int nColor, typename DDArg, QudaReconstructType recon> struct NdegTwistedCloverApply {
+      template <bool distance_pc>
+      NdegTwistedCloverApply(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+                             cvector_ref<const ColorSpinorField> &x, const GaugeField &U, const CloverField &A, double a,
+                             double b, double c, int parity, bool dagger, const int *comm_override,DistanceType<distance_pc>,TimeProfile &profile)
+#ifdef SIGNATURE_ONLY
+    ;
+#else
+      {
+        constexpr int nDim = 4;
+        auto halo = ColorSpinorField::create_comms_batch(in);
+        NdegTwistedCloverArg<Float, nColor, nDim, DDArg, recon> arg(out, in, halo, U, A, a, b, c, x, parity, dagger,
+                                                                    comm_override);
+        NdegTwistedClover<decltype(arg)> twisted(arg, out, in, halo);
+        dslash::DslashPolicyTune<decltype(twisted)> policy(twisted, in, halo, profile);
+      }
+#endif
+    };
+
+} // namespace quda
diff --git a/lib/dslash_ndeg_twisted_clover.in.cu b/lib/dslash_ndeg_twisted_clover.in.cu
new file mode 100644
index 0000000000..7e428d30c7
--- /dev/null
+++ b/lib/dslash_ndeg_twisted_clover.in.cu
@@ -0,0 +1,20 @@
+#include <dslash_ndeg_twisted_clover.hpp>
+
+namespace quda
+{
+
+    constexpr QudaPrecision precision = QUDA_@QUDA_DSLASH_PREC@_PRECISION;
+    constexpr int nColor = @QUDA_DSLASH_NCOLOR@;
+    constexpr int reconI = @QUDA_DSLASH_RECONI@;
+    constexpr bool distance_pc = @QUDA_DSLASH_DISTANCE@;
+
+    typedef @QUDA_DSLASH_DDARG@ DDArg;
+    typedef precision_type_mapper<precision>::type Float;
+
+    template struct NdegTwistedCloverApply<Float,nColor, DDArg, ReconstructWilson::recon[reconI]>;
+
+    template NdegTwistedCloverApply<Float,nColor, DDArg, ReconstructWilson::recon[reconI]>::NdegTwistedCloverApply<distance_pc>(
+	    cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+            cvector_ref<const ColorSpinorField> &x, const GaugeField &U, const CloverField &A, double a,
+	    double b, double c, int parity, bool dagger, const int *comm_override, DistanceType<distance_pc>, TimeProfile &profile);
+} // namespace quda
diff --git a/lib/dslash_ndeg_twisted_clover_preconditioned.cu b/lib/dslash_ndeg_twisted_clover_preconditioned.cu
index 1cfbc280b7..b01ff7ee77 100644
--- a/lib/dslash_ndeg_twisted_clover_preconditioned.cu
+++ b/lib/dslash_ndeg_twisted_clover_preconditioned.cu
@@ -1,11 +1,6 @@
-#include <gauge_field.h>
-#include <color_spinor_field.h>
-#include <clover_field.h>
-#include <dslash.h>
-#include <worker.h>
-
-#include <dslash_policy.hpp>
-#include <kernels/dslash_ndeg_twisted_clover_preconditioned.cuh>
+#define SIGNATURE_ONLY
+#include <dslash_ndeg_twisted_clover_preconditioned.hpp>
+#undef SIGNATURE_ONLY
 
 /**
    This is the gauged preconditioned twisted-clover operator 
@@ -14,102 +9,6 @@
 
 namespace quda
 {
-  template <typename Arg> class NdegTwistedCloverPreconditioned : public Dslash<nDegTwistedCloverPreconditioned, Arg>
-    {
-      using Dslash = Dslash<nDegTwistedCloverPreconditioned, Arg>;
-      using Dslash::arg;
-      using Dslash::halo;
-      using Dslash::in;
-
-      unsigned int sharedBytesPerThread() const
-      {
-        return (in.Nspin() / 2) * in.Ncolor() * 2 * sizeof(typename mapper<typename Arg::Float>::type);
-      }
-      
-    public:
-      NdegTwistedCloverPreconditioned(Arg &arg, cvector_ref<ColorSpinorField> &out,
-                                      cvector_ref<const ColorSpinorField> &in, const ColorSpinorField &halo) :
-        Dslash(arg, out, in, halo)
-      {
-        TunableKernel3D::resizeStep(2, 1); // this will force flavor to be contained in the block
-      }
-      
-      void apply(const qudaStream_t &stream)
-      {
-        TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
-        Dslash::setParam(tp);
-        if (arg.nParity != 1) errorQuda("Preconditioned non-degenerate twisted-clover operator not defined nParity=%d", arg.nParity);
-       
-        if (arg.xpay){
-          if (arg.dagger) errorQuda("xpay operator not only defined for not dagger");
-          Dslash::template instantiate<packShmem, 1, false, true>(tp, stream);
-        } else {
-          if (arg.dagger)
-            Dslash::template instantiate<packShmem, 1, true, false>(tp, stream);
-          else
-            Dslash::template instantiate<packShmem, 1, false, false>(tp, stream);
-        }
-      }
-
-      void initTuneParam(TuneParam &param) const
-      {
-        Dslash::initTuneParam(param);
-        param.shared_bytes = sharedBytesPerThread() * param.block.x * param.block.y * param.block.z;
-      }
-      
-      void defaultTuneParam(TuneParam &param) const
-      {
-        Dslash::defaultTuneParam(param);
-        param.shared_bytes = sharedBytesPerThread() * param.block.x * param.block.y * param.block.z;
-      }
-      
-      long long flops() const
-      {
-        int clover_flops = 504;
-        long long flops = Dslash::flops();
-        switch (arg.kernel_type) {
-        case INTERIOR_KERNEL:
-        case KERNEL_POLICY:
-          // b and c multiply (= 2 * 48 * in.Volume())
-          flops += 2 * in.Ncolor() * 4 * 4 * halo.Volume(); // complex * Nc * Ns * fma * vol
-          flops += clover_flops * halo.Volume();
-          break;
-        default: break; // twisted-mass flops are in the interior kernel
-        }
-        return flops;
-      }
-      
-      long long bytes() const
-      {
-        int clover_bytes = 72 * in.Precision() + (isFixed<typename Arg::Float>::value ? 2 * sizeof(float) : 0);
-        
-        long long bytes = Dslash::bytes();
-        switch (arg.kernel_type) {
-        case INTERIOR_KERNEL:
-        case KERNEL_POLICY: bytes += clover_bytes * halo.Volume(); break;
-        default: break;
-        }
-        
-        return bytes;
-      }
-    };
-
-    template <typename Float, int nColor, typename DDArg, QudaReconstructType recon>
-    struct NdegTwistedCloverPreconditionedApply {
-
-      NdegTwistedCloverPreconditionedApply(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
-                                           cvector_ref<const ColorSpinorField> &x, const GaugeField &U,
-                                           const CloverField &A, double a, double b, double c, bool xpay, int parity,
-                                           bool dagger, const int *comm_override, TimeProfile &profile)
-      {
-        constexpr int nDim = 4;
-        auto halo = ColorSpinorField::create_comms_batch(in);
-        NdegTwistedCloverPreconditionedArg<Float, nColor, nDim, DDArg, recon> arg(out, in, halo, U, A, a, b, c, xpay, x,
-                                                                                  parity, dagger, comm_override);
-        NdegTwistedCloverPreconditioned<decltype(arg)> twisted(arg, out, in, halo);
-        dslash::DslashPolicyTune<decltype(twisted)> policy(twisted, in, halo, profile);
-      }
-    };
 
     void ApplyNdegTwistedCloverPreconditioned(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
                                               const GaugeField &U, const CloverField &A, double a, double b, double c,
@@ -117,8 +16,9 @@ namespace quda
                                               bool dagger, const int *comm_override, TimeProfile &profile)
     {
       if constexpr (is_enabled<QUDA_TWISTED_CLOVER_DSLASH>()) {
+        auto dummy = DistanceType<false>();
         instantiate<NdegTwistedCloverPreconditionedApply>(out, in, x, U, A, a, b, c, xpay, parity, dagger,
-                                                          comm_override, profile);
+                                                          comm_override, dummy, profile);
       } else {
         errorQuda("Non-degenerate preconditioned twisted-clover operator has not been built");
       }
diff --git a/lib/dslash_ndeg_twisted_clover_preconditioned.hpp b/lib/dslash_ndeg_twisted_clover_preconditioned.hpp
new file mode 100644
index 0000000000..a2763a5bca
--- /dev/null
+++ b/lib/dslash_ndeg_twisted_clover_preconditioned.hpp
@@ -0,0 +1,124 @@
+#include <gauge_field.h>
+#include <color_spinor_field.h>
+#include <clover_field.h>
+#include <dslash.h>
+#include <worker.h>
+
+#include <dslash_policy.hpp>
+#include <kernels/dslash_ndeg_twisted_clover_preconditioned.cuh>
+
+/**
+   This is the gauged preconditioned twisted-clover operator 
+   acting on a non-degenerate quark doublet.
+*/
+
+namespace quda
+{
+  template <typename Arg> class NdegTwistedCloverPreconditioned : public Dslash<nDegTwistedCloverPreconditioned, Arg>
+    {
+      using Dslash = Dslash<nDegTwistedCloverPreconditioned, Arg>;
+      using Dslash::arg;
+      using Dslash::halo;
+      using Dslash::in;
+
+      unsigned int sharedBytesPerThread() const
+      {
+        return (in.Nspin() / 2) * in.Ncolor() * 2 * sizeof(typename mapper<typename Arg::Float>::type);
+      }
+      
+    public:
+      NdegTwistedCloverPreconditioned(Arg &arg, cvector_ref<ColorSpinorField> &out,
+                                      cvector_ref<const ColorSpinorField> &in, const ColorSpinorField &halo) :
+        Dslash(arg, out, in, halo)
+      {
+        TunableKernel3D::resizeStep(2, 1); // this will force flavor to be contained in the block
+      }
+      
+      void apply(const qudaStream_t &stream)
+      {
+        TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
+        Dslash::setParam(tp);
+        if (arg.nParity != 1) errorQuda("Preconditioned non-degenerate twisted-clover operator not defined nParity=%d", arg.nParity);
+       
+        if (arg.xpay){
+          if (arg.dagger) errorQuda("xpay operator not only defined for not dagger");
+          Dslash::template instantiate<packShmem, 1, false, true>(tp, stream);
+        } else {
+          if (arg.dagger)
+            Dslash::template instantiate<packShmem, 1, true, false>(tp, stream);
+          else
+            Dslash::template instantiate<packShmem, 1, false, false>(tp, stream);
+        }
+      }
+
+      void initTuneParam(TuneParam &param) const
+      {
+        Dslash::initTuneParam(param);
+        param.shared_bytes = sharedBytesPerThread() * param.block.x * param.block.y * param.block.z;
+      }
+      
+      void defaultTuneParam(TuneParam &param) const
+      {
+        Dslash::defaultTuneParam(param);
+        param.shared_bytes = sharedBytesPerThread() * param.block.x * param.block.y * param.block.z;
+      }
+      
+      long long flops() const
+      {
+        int clover_flops = 504;
+        long long flops = Dslash::flops();
+        switch (arg.kernel_type) {
+        case INTERIOR_KERNEL:
+        case KERNEL_POLICY:
+          // b and c multiply (= 2 * 48 * in.Volume())
+          flops += 2 * in.Ncolor() * 4 * 4 * halo.Volume(); // complex * Nc * Ns * fma * vol
+          flops += clover_flops * halo.Volume();
+          break;
+        default: break; // twisted-mass flops are in the interior kernel
+        }
+        return flops;
+      }
+      
+      long long bytes() const
+      {
+        int clover_bytes = 72 * in.Precision() + (isFixed<typename Arg::Float>::value ? 2 * sizeof(float) : 0);
+        
+        long long bytes = Dslash::bytes();
+        switch (arg.kernel_type) {
+        case INTERIOR_KERNEL:
+        case KERNEL_POLICY: bytes += clover_bytes * halo.Volume(); break;
+        default: break;
+        }
+        
+        return bytes;
+      }
+    };
+
+
+    template <bool distance_pc> struct DistanceType {
+    };
+
+
+    template <typename Float, int nColor, typename DDArg, QudaReconstructType recon>
+    struct NdegTwistedCloverPreconditionedApply {
+      template <bool distance_pc>
+      NdegTwistedCloverPreconditionedApply(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+                                           cvector_ref<const ColorSpinorField> &x, const GaugeField &U,
+                                           const CloverField &A, double a, double b, double c, bool xpay, int parity,
+                                           bool dagger, const int *comm_override,DistanceType<distance_pc>,TimeProfile &profile)
+
+#ifdef SIGNATURE_ONLY
+    ;
+#else
+      {
+        constexpr int nDim = 4;
+        auto halo = ColorSpinorField::create_comms_batch(in);
+        NdegTwistedCloverPreconditionedArg<Float, nColor, nDim, DDArg, recon> arg(out, in, halo, U, A, a, b, c, xpay, x,
+                                                                                  parity, dagger, comm_override);
+        NdegTwistedCloverPreconditioned<decltype(arg)> twisted(arg, out, in, halo);
+        dslash::DslashPolicyTune<decltype(twisted)> policy(twisted, in, halo, profile);
+      }
+#endif
+    };
+} // namespace quda
+
diff --git a/lib/dslash_ndeg_twisted_clover_preconditioned.in.cu b/lib/dslash_ndeg_twisted_clover_preconditioned.in.cu
new file mode 100644
index 0000000000..90b6173be0
--- /dev/null
+++ b/lib/dslash_ndeg_twisted_clover_preconditioned.in.cu
@@ -0,0 +1,24 @@
+#include <dslash_ndeg_twisted_clover_preconditioned.hpp>
+
+namespace quda
+{
+
+    constexpr QudaPrecision precision = QUDA_@QUDA_DSLASH_PREC@_PRECISION;
+    constexpr int nColor = @QUDA_DSLASH_NCOLOR@;
+    constexpr int reconI = @QUDA_DSLASH_RECONI@;
+    constexpr bool distance_pc = @QUDA_DSLASH_DISTANCE@;
+
+    typedef @QUDA_DSLASH_DDARG@ DDArg;
+    typedef precision_type_mapper<precision>::type Float;
+
+    template struct NdegTwistedCloverPreconditionedApply<Float,nColor, DDArg, ReconstructWilson::recon[reconI]>;
+
+
+    template NdegTwistedCloverPreconditionedApply<Float,nColor, DDArg, ReconstructWilson::recon[reconI]>::NdegTwistedCloverPreconditionedApply<distance_pc>(
+		    cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+                    cvector_ref<const ColorSpinorField> &x, const GaugeField &U,
+                    const CloverField &A, double a, double b, double c, bool xpay, int parity,
+                    bool dagger, const int *comm_override, DistanceType<distance_pc>, TimeProfile &profile);
+    
+} // namespace quda
+
diff --git a/lib/dslash_ndeg_twisted_mass.cu b/lib/dslash_ndeg_twisted_mass.cu
index a71cda8cbf..2c0bda1fe1 100644
--- a/lib/dslash_ndeg_twisted_mass.cu
+++ b/lib/dslash_ndeg_twisted_mass.cu
@@ -1,10 +1,6 @@
-#include <gauge_field.h>
-#include <color_spinor_field.h>
-#include <dslash.h>
-#include <worker.h>
-
-#include <dslash_policy.hpp>
-#include <kernels/dslash_ndeg_twisted_mass.cuh>
+#define SIGNATURE_ONLY
+#include <dslash_ndeg_twisted_mass.hpp>
+#undef SIGNATURE_ONLY
 
 /**
    This is the gauged twisted-mass operator acting on a non-generate
@@ -14,65 +10,13 @@
 namespace quda
 {
 
-  template <typename Arg> class NdegTwistedMass : public Dslash<nDegTwistedMass, Arg>
-  {
-    using Dslash = Dslash<nDegTwistedMass, Arg>;
-    using Dslash::arg;
-    using Dslash::in;
-
-  public:
-    NdegTwistedMass(Arg &arg, cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
-                    const ColorSpinorField &halo) :
-      Dslash(arg, out, in, halo)
-    {
-    }
-
-    void apply(const qudaStream_t &stream)
-    {
-      TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
-      Dslash::setParam(tp);
-      if (arg.xpay)
-        Dslash::template instantiate<packShmem, true>(tp, stream);
-      else
-        errorQuda("Non-degenerate twisted-mass operator only defined for xpay=true");
-    }
-
-    long long flops() const
-    {
-      long long flops = Dslash::flops();
-      switch (arg.kernel_type) {
-      case INTERIOR_KERNEL:
-      case UBER_KERNEL:
-      case KERNEL_POLICY:
-        flops += in.size() * 2 * in.Ncolor() * 4 * 4 * in.Volume(); // complex * Nc * Ns * fma * vol
-        break;
-      default: break; // twisted-mass flops are in the interior kernel
-      }
-      return flops;
-    }
-  };
-
-  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon> struct NdegTwistedMassApply {
-
-    NdegTwistedMassApply(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
-                         cvector_ref<const ColorSpinorField> &x, const GaugeField &U, double a, double b, double c,
-                         int parity, bool dagger, const int *comm_override, TimeProfile &profile)
-    {
-      constexpr int nDim = 4;
-      auto halo = ColorSpinorField::create_comms_batch(in);
-      NdegTwistedMassArg<Float, nColor, nDim, DDArg, recon> arg(out, in, halo, U, a, b, c, x, parity, dagger,
-                                                                comm_override);
-      NdegTwistedMass<decltype(arg)> twisted(arg, out, in, halo);
-      dslash::DslashPolicyTune<decltype(twisted)> policy(twisted, in, halo, profile);
-    }
-  };
-
   void ApplyNdegTwistedMass(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
                             const GaugeField &U, double a, double b, double c, cvector_ref<const ColorSpinorField> &x,
                             int parity, bool dagger, const int *comm_override, TimeProfile &profile)
   {
     if constexpr (is_enabled<QUDA_TWISTED_MASS_DSLASH>()) {
-      instantiate<NdegTwistedMassApply>(out, in, x, U, a, b, c, parity, dagger, comm_override, profile);
+      auto dummy = DistanceType<false>();
+      instantiate<NdegTwistedMassApply>(out, in, x, U, a, b, c, parity, dagger, comm_override,dummy,profile);
     } else {
       errorQuda("Non-degenerate twisted-mass operator has not been built");
     }
diff --git a/lib/dslash_ndeg_twisted_mass.hpp b/lib/dslash_ndeg_twisted_mass.hpp
new file mode 100644
index 0000000000..432b73791a
--- /dev/null
+++ b/lib/dslash_ndeg_twisted_mass.hpp
@@ -0,0 +1,77 @@
+#include <gauge_field.h>
+#include <color_spinor_field.h>
+#include <dslash.h>
+#include <worker.h>
+
+#include <dslash_policy.hpp>
+#include <kernels/dslash_ndeg_twisted_mass.cuh>
+
+/**
+   This is the gauged twisted-mass operator acting on a non-generate
+   quark doublet.
+*/
+
+namespace quda
+{
+
+  template <typename Arg> class NdegTwistedMass : public Dslash<nDegTwistedMass, Arg>
+  {
+    using Dslash = Dslash<nDegTwistedMass, Arg>;
+    using Dslash::arg;
+    using Dslash::in;
+
+  public:
+    NdegTwistedMass(Arg &arg, cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+                    const ColorSpinorField &halo) :
+      Dslash(arg, out, in, halo)
+    {
+    }
+
+    void apply(const qudaStream_t &stream)
+    {
+      TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
+      Dslash::setParam(tp);
+      if (arg.xpay)
+        Dslash::template instantiate<packShmem, true>(tp, stream);
+      else
+        errorQuda("Non-degenerate twisted-mass operator only defined for xpay=true");
+    }
+
+    long long flops() const
+    {
+      long long flops = Dslash::flops();
+      switch (arg.kernel_type) {
+      case INTERIOR_KERNEL:
+      case UBER_KERNEL:
+      case KERNEL_POLICY:
+        flops += in.size() * 2 * in.Ncolor() * 4 * 4 * in.Volume(); // complex * Nc * Ns * fma * vol
+        break;
+      default: break; // twisted-mass flops are in the interior kernel
+      }
+      return flops;
+    }
+  };
+
+  template <bool distance_pc> struct DistanceType {
+  };
+
+
+  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon> struct NdegTwistedMassApply {
+    template <bool distance_pc>
+    NdegTwistedMassApply(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+                         cvector_ref<const ColorSpinorField> &x, const GaugeField &U, double a, double b, double c,
+                         int parity, bool dagger, const int *comm_override,DistanceType<distance_pc>, TimeProfile &profile)
+#ifdef SIGNATURE_ONLY
+    ;
+#else
+    {
+      constexpr int nDim = 4;
+      auto halo = ColorSpinorField::create_comms_batch(in);
+      NdegTwistedMassArg<Float, nColor, nDim, DDArg, recon> arg(out, in, halo, U, a, b, c, x, parity, dagger,
+                                                                comm_override);
+      NdegTwistedMass<decltype(arg)> twisted(arg, out, in, halo);
+      dslash::DslashPolicyTune<decltype(twisted)> policy(twisted, in, halo, profile);
+    }
+#endif
+  };
+} // namespace quda
diff --git a/lib/dslash_ndeg_twisted_mass.in.cu b/lib/dslash_ndeg_twisted_mass.in.cu
new file mode 100644
index 0000000000..9cb709ad32
--- /dev/null
+++ b/lib/dslash_ndeg_twisted_mass.in.cu
@@ -0,0 +1,20 @@
+#include <dslash_ndeg_twisted_mass.hpp>
+  
+namespace quda
+{
+
+  constexpr QudaPrecision precision = QUDA_@QUDA_DSLASH_PREC@_PRECISION;
+  constexpr int nColor = @QUDA_DSLASH_NCOLOR@;
+  constexpr int reconI = @QUDA_DSLASH_RECONI@;
+  constexpr bool distance_pc = @QUDA_DSLASH_DISTANCE@;
+
+  typedef @QUDA_DSLASH_DDARG@ DDArg;
+  typedef precision_type_mapper<precision>::type Float;
+
+  template struct NdegTwistedMassApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>;
+
+  template NdegTwistedMassApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>::NdegTwistedMassApply<distance_pc>(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+                         cvector_ref<const ColorSpinorField> &x, const GaugeField &U, double a, double b, double c,
+                         int parity, bool dagger, const int *comm_override,DistanceType<distance_pc>, TimeProfile &profile);
+
+} // namespace quda
diff --git a/lib/dslash_ndeg_twisted_mass_preconditioned.cu b/lib/dslash_ndeg_twisted_mass_preconditioned.cu
index 7f2a3e868a..9af7437df4 100644
--- a/lib/dslash_ndeg_twisted_mass_preconditioned.cu
+++ b/lib/dslash_ndeg_twisted_mass_preconditioned.cu
@@ -1,10 +1,6 @@
-#include <gauge_field.h>
-#include <color_spinor_field.h>
-#include <dslash.h>
-#include <worker.h>
-
-#include <dslash_policy.hpp>
-#include <kernels/dslash_ndeg_twisted_mass_preconditioned.cuh>
+#define SIGNATURE_ONLY
+#include <dslash_ndeg_twisted_mass_preconditioned.hpp>
+#undef SIGNATURE_ONLY
 
 /**
    This is the preconditioned twisted-mass operator acting on a non-generate
@@ -14,107 +10,6 @@
 namespace quda
 {
 
-  // trait to ensure we don't instantiate asymmetric & xpay
-  template <bool symmetric> constexpr bool xpay_() { return true; }
-  template <> constexpr bool xpay_<true>() { return false; }
-
-  // trait to ensure we don't instantiate asymmetric & !dagger
-  template <bool symmetric> constexpr bool not_dagger_() { return false; }
-  template <> constexpr bool not_dagger_<true>() { return true; }
-
-  template <typename Arg> class NdegTwistedMassPreconditioned : public Dslash<nDegTwistedMassPreconditioned, Arg>
-  {
-    using Dslash = Dslash<nDegTwistedMassPreconditioned, Arg>;
-    using Dslash::arg;
-    using Dslash::halo;
-    using Dslash::in;
-
-  protected:
-    bool shared;
-    unsigned int sharedBytesPerThread() const
-    {
-      return shared ? 2 * in.Ncolor() * 4 * sizeof(typename mapper<typename Arg::Float>::type) : 0;
-    }
-
-  public:
-    NdegTwistedMassPreconditioned(Arg &arg, cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
-                                  const ColorSpinorField &halo) :
-      Dslash(arg, out, in, halo), shared(arg.asymmetric || !arg.dagger)
-    {
-      if (shared) TunableKernel3D::resizeStep(2, 1); // this will force flavor to be contained in the block
-    }
-
-    void apply(const qudaStream_t &stream)
-    {
-      TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
-      Dslash::setParam(tp);
-      if (arg.asymmetric && !arg.dagger) errorQuda("asymmetric operator only defined for dagger");
-      if (arg.asymmetric && arg.xpay) errorQuda("asymmetric operator not defined for xpay");
-      if (arg.nParity != 1) errorQuda("Preconditioned non-degenerate twisted-mass operator not defined nParity=%d", arg.nParity);
-
-      if (arg.dagger) {
-        if (arg.xpay)
-          Dslash::template instantiate<packShmem, 1, true, xpay_<Arg::asymmetric>()>(tp, stream);
-        else
-          Dslash::template instantiate<packShmem, 1, true, false>(tp, stream);
-      } else {
-        if (arg.xpay)
-          Dslash::template instantiate<packShmem, 1, not_dagger_<Arg::asymmetric>(), xpay_<Arg::asymmetric>()>(tp, stream);
-        else
-          Dslash::template instantiate<packShmem, 1, not_dagger_<Arg::asymmetric>(), false>(tp, stream);
-      }
-    }
-
-    void initTuneParam(TuneParam &param) const
-    {
-      Dslash::initTuneParam(param);
-      if (shared) param.shared_bytes = sharedBytesPerThread() * param.block.x * param.block.y * param.block.z;
-    }
-
-    void defaultTuneParam(TuneParam &param) const
-    {
-      Dslash::defaultTuneParam(param);
-      if (shared) param.shared_bytes = sharedBytesPerThread() * param.block.x * param.block.y * param.block.z;
-    }
-
-    long long flops() const
-    {
-      long long flops = Dslash::flops();
-      switch (arg.kernel_type) {
-      case INTERIOR_KERNEL:
-      case UBER_KERNEL:
-      case KERNEL_POLICY:
-        flops += 2 * in.Ncolor() * 4 * 4 * halo.Volume(); // complex * Nc * Ns * fma * vol
-        break;
-      default: break; // twisted-mass flops are in the interior kernel
-      }
-      return flops;
-    }
-  };
-
-  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon>
-  struct NdegTwistedMassPreconditionedApply {
-
-    NdegTwistedMassPreconditionedApply(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
-                                       cvector_ref<const ColorSpinorField> &x, const GaugeField &U, double a, double b,
-                                       double c, bool xpay, int parity, bool dagger, bool asymmetric,
-                                       const int *comm_override, TimeProfile &profile)
-    {
-      constexpr int nDim = 4;
-      auto halo = ColorSpinorField::create_comms_batch(in);
-      if (asymmetric) {
-        NdegTwistedMassArg<Float, nColor, nDim, DDArg, recon, true> arg(out, in, halo, U, a, b, c, xpay, x, parity,
-                                                                        dagger, comm_override);
-        NdegTwistedMassPreconditioned<decltype(arg)> twisted(arg, out, in, halo);
-        dslash::DslashPolicyTune<decltype(twisted)> policy(twisted, in, halo, profile);
-      } else {
-        NdegTwistedMassArg<Float, nColor, nDim, DDArg, recon, false> arg(out, in, halo, U, a, b, c, xpay, x, parity,
-                                                                         dagger, comm_override);
-        NdegTwistedMassPreconditioned<decltype(arg)> twisted(arg, out, in, halo);
-        dslash::DslashPolicyTune<decltype(twisted)> policy(twisted, in, halo, profile);
-      }
-    }
-  };
 
   // Apply the non-degenerate twisted-mass Dslash operator
   // out(x) = M*in = a*(1 + i*b*gamma_5*tau_3 + c*tau_1)*D + x
@@ -125,8 +20,9 @@ namespace quda
                                           bool asymmetric, const int *comm_override, TimeProfile &profile)
   {
     if constexpr (is_enabled<QUDA_TWISTED_MASS_DSLASH>()) {
+      auto dummy = DistanceType<false>();
       instantiate<NdegTwistedMassPreconditionedApply>(out, in, x, U, a, b, c, xpay, parity, dagger, asymmetric,
-                                                      comm_override, profile);
+                                                      comm_override, dummy, profile);
     } else {
       errorQuda("Non-degenerate preconditioned twisted-mass dslash has not been built");
     }
diff --git a/lib/dslash_ndeg_twisted_mass_preconditioned.hpp b/lib/dslash_ndeg_twisted_mass_preconditioned.hpp
new file mode 100644
index 0000000000..47e3029965
--- /dev/null
+++ b/lib/dslash_ndeg_twisted_mass_preconditioned.hpp
@@ -0,0 +1,126 @@
+#include <gauge_field.h>
+#include <color_spinor_field.h>
+#include <dslash.h>
+#include <worker.h>
+
+#include <dslash_policy.hpp>
+#include <kernels/dslash_ndeg_twisted_mass_preconditioned.cuh>
+
+/**
+   This is the preconditioned twisted-mass operator acting on a non-generate
+   quark doublet.
+*/
+
+namespace quda
+{
+
+  // trait to ensure we don't instantiate asymmetric & xpay
+  template <bool symmetric> constexpr bool xpay_() { return true; }
+  template <> constexpr bool xpay_<true>() { return false; }
+
+  // trait to ensure we don't instantiate asymmetric & !dagger
+  template <bool symmetric> constexpr bool not_dagger_() { return false; }
+  template <> constexpr bool not_dagger_<true>() { return true; }
+
+  template <typename Arg> class NdegTwistedMassPreconditioned : public Dslash<nDegTwistedMassPreconditioned, Arg>
+  {
+    using Dslash = Dslash<nDegTwistedMassPreconditioned, Arg>;
+    using Dslash::arg;
+    using Dslash::halo;
+    using Dslash::in;
+
+  protected:
+    bool shared;
+    unsigned int sharedBytesPerThread() const
+    {
+      return shared ? 2 * in.Ncolor() * 4 * sizeof(typename mapper<typename Arg::Float>::type) : 0;
+    }
+
+  public:
+    NdegTwistedMassPreconditioned(Arg &arg, cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+                                  const ColorSpinorField &halo) :
+      Dslash(arg, out, in, halo), shared(arg.asymmetric || !arg.dagger)
+    {
+      if (shared) TunableKernel3D::resizeStep(2, 1); // this will force flavor to be contained in the block
+    }
+
+    void apply(const qudaStream_t &stream)
+    {
+      TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
+      Dslash::setParam(tp);
+      if (arg.asymmetric && !arg.dagger) errorQuda("asymmetric operator only defined for dagger");
+      if (arg.asymmetric && arg.xpay) errorQuda("asymmetric operator not defined for xpay");
+      if (arg.nParity != 1) errorQuda("Preconditioned non-degenerate twisted-mass operator not defined nParity=%d", arg.nParity);
+
+      if (arg.dagger) {
+        if (arg.xpay)
+          Dslash::template instantiate<packShmem, 1, true, xpay_<Arg::asymmetric>()>(tp, stream);
+        else
+          Dslash::template instantiate<packShmem, 1, true, false>(tp, stream);
+      } else {
+        if (arg.xpay)
+          Dslash::template instantiate<packShmem, 1, not_dagger_<Arg::asymmetric>(), xpay_<Arg::asymmetric>()>(tp, stream);
+        else
+          Dslash::template instantiate<packShmem, 1, not_dagger_<Arg::asymmetric>(), false>(tp, stream);
+      }
+    }
+
+    void initTuneParam(TuneParam &param) const
+    {
+      Dslash::initTuneParam(param);
+      if (shared) param.shared_bytes = sharedBytesPerThread() * param.block.x * param.block.y * param.block.z;
+    }
+
+    void defaultTuneParam(TuneParam &param) const
+    {
+      Dslash::defaultTuneParam(param);
+      if (shared) param.shared_bytes = sharedBytesPerThread() * param.block.x * param.block.y * param.block.z;
+    }
+
+    long long flops() const
+    {
+      long long flops = Dslash::flops();
+      switch (arg.kernel_type) {
+      case INTERIOR_KERNEL:
+      case UBER_KERNEL:
+      case KERNEL_POLICY:
+        flops += 2 * in.Ncolor() * 4 * 4 * halo.Volume(); // complex * Nc * Ns * fma * vol
+        break;
+      default: break; // twisted-mass flops are in the interior kernel
+      }
+      return flops;
+    }
+  };
+
+  template <bool distance_pc> struct DistanceType {
+  };
+
+  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon>
+  struct NdegTwistedMassPreconditionedApply {
+    template <bool distance_pc>
+    NdegTwistedMassPreconditionedApply(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+                                       cvector_ref<const ColorSpinorField> &x, const GaugeField &U, double a, double b,
+                                       double c, bool xpay, int parity, bool dagger, bool asymmetric,
+                                       const int *comm_override, DistanceType<distance_pc>, TimeProfile &profile)
+#ifdef SIGNATURE_ONLY
+    ;
+#else
+    {
+      constexpr int nDim = 4;
+      auto halo = ColorSpinorField::create_comms_batch(in);
+      if (asymmetric) {
+        NdegTwistedMassArg<Float, nColor, nDim, DDArg, recon, true> arg(out, in, halo, U, a, b, c, xpay, x, parity,
+                                                                        dagger, comm_override);
+        NdegTwistedMassPreconditioned<decltype(arg)> twisted(arg, out, in, halo);
+        dslash::DslashPolicyTune<decltype(twisted)> policy(twisted, in, halo, profile);
+      } else {
+        NdegTwistedMassArg<Float, nColor, nDim, DDArg, recon, false> arg(out, in, halo, U, a, b, c, xpay, x, parity,
+                                                                         dagger, comm_override);
+        NdegTwistedMassPreconditioned<decltype(arg)> twisted(arg, out, in, halo);
+        dslash::DslashPolicyTune<decltype(twisted)> policy(twisted, in, halo, profile);
+      }
+    }
+#endif
+  };
+
+} // namespace quda
diff --git a/lib/dslash_ndeg_twisted_mass_preconditioned.in.cu b/lib/dslash_ndeg_twisted_mass_preconditioned.in.cu
new file mode 100644
index 0000000000..0ae75055b2
--- /dev/null
+++ b/lib/dslash_ndeg_twisted_mass_preconditioned.in.cu
@@ -0,0 +1,23 @@
+#include <dslash_ndeg_twisted_mass_preconditioned.hpp>
+
+namespace quda
+{
+
+
+    constexpr QudaPrecision precision = QUDA_@QUDA_DSLASH_PREC@_PRECISION;
+    constexpr int nColor = @QUDA_DSLASH_NCOLOR@;
+    constexpr int reconI = @QUDA_DSLASH_RECONI@;
+    constexpr bool distance_pc = @QUDA_DSLASH_DISTANCE@;
+
+    typedef @QUDA_DSLASH_DDARG@ DDArg;
+    typedef precision_type_mapper<precision>::type Float;
+
+
+    template struct NdegTwistedMassPreconditionedApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>;
+    
+    template NdegTwistedMassPreconditionedApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>::NdegTwistedMassPreconditionedApply<distance_pc>(
+		    cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+                    cvector_ref<const ColorSpinorField> &x, const GaugeField &U, double a, double b,
+                    double c, bool xpay, int parity, bool dagger, bool asymmetric,
+                    const int *comm_override, DistanceType<distance_pc>,TimeProfile &profile);
+} // namespace quda
diff --git a/lib/dslash_twisted_clover_preconditioned.cu b/lib/dslash_twisted_clover_preconditioned.cu
index df6cd3213d..d5a3883c88 100644
--- a/lib/dslash_twisted_clover_preconditioned.cu
+++ b/lib/dslash_twisted_clover_preconditioned.cu
@@ -1,11 +1,6 @@
-#include <gauge_field.h>
-#include <color_spinor_field.h>
-#include <clover_field.h>
-#include <dslash.h>
-#include <worker.h>
-
-#include <dslash_policy.hpp>
-#include <kernels/dslash_twisted_clover_preconditioned.cuh>
+#define SIGNATURE_ONLY
+#include <dslash_twisted_clover_preconditioned.hpp>
+#undef SIGNATURE_ONLY
 
 /**
    This is the preconditioned gauged twisted-mass operator
@@ -14,121 +9,6 @@
 namespace quda
 {
 
-  template <typename Arg> class TwistedCloverPreconditioned : public Dslash<twistedCloverPreconditioned, Arg>
-  {
-    using Dslash = Dslash<twistedCloverPreconditioned, Arg>;
-    using Dslash::arg;
-    using Dslash::halo;
-    using Dslash::in;
-
-  public:
-    TwistedCloverPreconditioned(Arg &arg, cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
-                                const ColorSpinorField &halo) :
-      Dslash(arg, out, in, halo)
-    {
-    }
-
-    void apply(const qudaStream_t &stream)
-    {
-      TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
-      Dslash::setParam(tp);
-      // specialize here to constrain the template instantiation
-      if (arg.nParity == 1) {
-        if (arg.xpay) {
-          if (arg.dagger) errorQuda("xpay operator only defined for not dagger");
-          Dslash::template instantiate<packShmem, 1, false, true>(tp, stream);
-        } else {
-          if (arg.dagger)
-            Dslash::template instantiate<packShmem, 1, true, false>(tp, stream);
-          else
-            Dslash::template instantiate<packShmem, 1, false, false>(tp, stream);
-        }
-      } else {
-        errorQuda("Preconditioned twisted-clover operator not defined nParity=%d", arg.nParity);
-      }
-    }
-
-    long long flops() const
-    {
-      int clover_flops = 504 + 48;
-      long long flops = Dslash::flops();
-      switch (arg.kernel_type) {
-      case EXTERIOR_KERNEL_X:
-      case EXTERIOR_KERNEL_Y:
-      case EXTERIOR_KERNEL_Z:
-      case EXTERIOR_KERNEL_T: flops += clover_flops * 2 * halo.GhostFace()[arg.kernel_type]; break;
-      case EXTERIOR_KERNEL_ALL:
-        flops
-          += clover_flops * 2 * (halo.GhostFace()[0] + halo.GhostFace()[1] + halo.GhostFace()[2] + halo.GhostFace()[3]);
-        break;
-      case INTERIOR_KERNEL:
-      case UBER_KERNEL:
-      case KERNEL_POLICY:
-        flops += clover_flops * halo.Volume();
-
-        if (arg.kernel_type == KERNEL_POLICY) break;
-        // now correct for flops done by exterior kernel
-        long long ghost_sites = 0;
-        for (int d = 0; d < 4; d++)
-          if (arg.commDim[d]) ghost_sites += 2 * halo.GhostFace()[d];
-        flops -= clover_flops * ghost_sites;
-
-        break;
-      }
-      return flops;
-    }
-
-    long long bytes() const
-    {
-      int clover_bytes = 72 * in.Precision() + (isFixed<typename Arg::Float>::value ? 2 * sizeof(float) : 0);
-      if (!arg.dynamic_clover) clover_bytes *= 2;
-
-      long long bytes = Dslash::bytes();
-      switch (arg.kernel_type) {
-      case EXTERIOR_KERNEL_X:
-      case EXTERIOR_KERNEL_Y:
-      case EXTERIOR_KERNEL_Z:
-      case EXTERIOR_KERNEL_T: bytes += clover_bytes * 2 * halo.GhostFace()[arg.kernel_type]; break;
-      case EXTERIOR_KERNEL_ALL:
-        bytes
-          += clover_bytes * 2 * (halo.GhostFace()[0] + halo.GhostFace()[1] + halo.GhostFace()[2] + halo.GhostFace()[3]);
-        break;
-      case INTERIOR_KERNEL:
-      case UBER_KERNEL:
-      case KERNEL_POLICY:
-        bytes += clover_bytes * halo.Volume();
-
-        if (arg.kernel_type == KERNEL_POLICY) break;
-        // now correct for bytes done by exterior kernel
-        long long ghost_sites = 0;
-        for (int d = 0; d < 4; d++)
-          if (arg.commDim[d]) ghost_sites += 2 * halo.GhostFace()[d];
-        bytes -= clover_bytes * ghost_sites;
-
-        break;
-      }
-
-      return bytes;
-    }
-  };
-
-  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon>
-  struct TwistedCloverPreconditionedApply {
-
-    TwistedCloverPreconditionedApply(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
-                                     cvector_ref<const ColorSpinorField> &x, const GaugeField &U, const CloverField &C,
-                                     double a, double b, bool xpay, int parity, bool dagger, const int *comm_override,
-                                     TimeProfile &profile)
-    {
-      constexpr int nDim = 4;
-      auto halo = ColorSpinorField::create_comms_batch(in);
-      TwistedCloverArg<Float, nColor, nDim, DDArg, recon> arg(out, in, halo, U, C, a, b, xpay, x, parity, dagger,
-                                                              comm_override);
-      TwistedCloverPreconditioned<decltype(arg)> twisted(arg, out, in, halo);
-      dslash::DslashPolicyTune<decltype(twisted)> policy(twisted, in, halo, profile);
-    }
-  };
-
   /*
     Apply the preconditioned twisted-mass Dslash operator
 
@@ -140,7 +20,8 @@ namespace quda
                                         const int *comm_override, TimeProfile &profile)
   {
     if constexpr (is_enabled<QUDA_TWISTED_CLOVER_DSLASH>()) {
-      instantiate<TwistedCloverPreconditionedApply>(out, in, x, U, C, a, b, xpay, parity, dagger, comm_override, profile);
+      auto dummy = DistanceType<false>();
+      instantiate<TwistedCloverPreconditionedApply>(out, in, x, U, C, a, b, xpay, parity, dagger, comm_override, dummy, profile);
     } else {
       errorQuda("Twisted-clover operator has not been built");
     }
diff --git a/lib/dslash_twisted_clover_preconditioned.hpp b/lib/dslash_twisted_clover_preconditioned.hpp
new file mode 100644
index 0000000000..b6fbaec92a
--- /dev/null
+++ b/lib/dslash_twisted_clover_preconditioned.hpp
@@ -0,0 +1,142 @@
+#include <gauge_field.h>
+#include <color_spinor_field.h>
+#include <clover_field.h>
+#include <dslash.h>
+#include <worker.h>
+
+#include <dslash_policy.hpp>
+#include <kernels/dslash_twisted_clover_preconditioned.cuh>
+
+/**
+   This is the preconditioned gauged twisted-mass operator
+*/
+
+namespace quda
+{
+
+  template <typename Arg> class TwistedCloverPreconditioned : public Dslash<twistedCloverPreconditioned, Arg>
+  {
+    using Dslash = Dslash<twistedCloverPreconditioned, Arg>;
+    using Dslash::arg;
+    using Dslash::halo;
+    using Dslash::in;
+
+  public:
+    TwistedCloverPreconditioned(Arg &arg, cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+                                const ColorSpinorField &halo) :
+      Dslash(arg, out, in, halo)
+    {
+    }
+
+    void apply(const qudaStream_t &stream)
+    {
+      TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
+      Dslash::setParam(tp);
+      // specialize here to constrain the template instantiation
+      if (arg.nParity == 1) {
+        if (arg.xpay) {
+          if (arg.dagger) errorQuda("xpay operator only defined for not dagger");
+          Dslash::template instantiate<packShmem, 1, false, true>(tp, stream);
+        } else {
+          if (arg.dagger)
+            Dslash::template instantiate<packShmem, 1, true, false>(tp, stream);
+          else
+            Dslash::template instantiate<packShmem, 1, false, false>(tp, stream);
+        }
+      } else {
+        errorQuda("Preconditioned twisted-clover operator not defined nParity=%d", arg.nParity);
+      }
+    }
+
+    long long flops() const
+    {
+      int clover_flops = 504 + 48;
+      long long flops = Dslash::flops();
+      switch (arg.kernel_type) {
+      case EXTERIOR_KERNEL_X:
+      case EXTERIOR_KERNEL_Y:
+      case EXTERIOR_KERNEL_Z:
+      case EXTERIOR_KERNEL_T: flops += clover_flops * 2 * halo.GhostFace()[arg.kernel_type]; break;
+      case EXTERIOR_KERNEL_ALL:
+        flops
+          += clover_flops * 2 * (halo.GhostFace()[0] + halo.GhostFace()[1] + halo.GhostFace()[2] + halo.GhostFace()[3]);
+        break;
+      case INTERIOR_KERNEL:
+      case UBER_KERNEL:
+      case KERNEL_POLICY:
+        flops += clover_flops * halo.Volume();
+
+        if (arg.kernel_type == KERNEL_POLICY) break;
+        // now correct for flops done by exterior kernel
+        long long ghost_sites = 0;
+        for (int d = 0; d < 4; d++)
+          if (arg.commDim[d]) ghost_sites += 2 * halo.GhostFace()[d];
+        flops -= clover_flops * ghost_sites;
+
+        break;
+      }
+      return flops;
+    }
+
+    long long bytes() const
+    {
+      int clover_bytes = 72 * in.Precision() + (isFixed<typename Arg::Float>::value ? 2 * sizeof(float) : 0);
+      if (!arg.dynamic_clover) clover_bytes *= 2;
+
+      long long bytes = Dslash::bytes();
+      switch (arg.kernel_type) {
+      case EXTERIOR_KERNEL_X:
+      case EXTERIOR_KERNEL_Y:
+      case EXTERIOR_KERNEL_Z:
+      case EXTERIOR_KERNEL_T: bytes += clover_bytes * 2 * halo.GhostFace()[arg.kernel_type]; break;
+      case EXTERIOR_KERNEL_ALL:
+        bytes
+          += clover_bytes * 2 * (halo.GhostFace()[0] + halo.GhostFace()[1] + halo.GhostFace()[2] + halo.GhostFace()[3]);
+        break;
+      case INTERIOR_KERNEL:
+      case UBER_KERNEL:
+      case KERNEL_POLICY:
+        bytes += clover_bytes * halo.Volume();
+
+        if (arg.kernel_type == KERNEL_POLICY) break;
+        // now correct for bytes done by exterior kernel
+        long long ghost_sites = 0;
+        for (int d = 0; d < 4; d++)
+          if (arg.commDim[d]) ghost_sites += 2 * halo.GhostFace()[d];
+        bytes -= clover_bytes * ghost_sites;
+
+        break;
+      }
+
+      return bytes;
+    }
+  };
+
+  template <bool distance_pc> struct DistanceType {
+  };
+
+
+
+  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon>
+  struct TwistedCloverPreconditionedApply {
+    template <bool distance_pc>
+    TwistedCloverPreconditionedApply(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+                                     cvector_ref<const ColorSpinorField> &x, const GaugeField &U, const CloverField &C,
+                                     double a, double b, bool xpay, int parity, bool dagger, const int *comm_override,DistanceType<distance_pc>,
+                                     TimeProfile &profile)
+#ifdef SIGNATURE_ONLY
+    ;
+#else
+
+    {
+      constexpr int nDim = 4;
+      auto halo = ColorSpinorField::create_comms_batch(in);
+      TwistedCloverArg<Float, nColor, nDim, DDArg, recon> arg(out, in, halo, U, C, a, b, xpay, x, parity, dagger,
+                                                              comm_override);
+      TwistedCloverPreconditioned<decltype(arg)> twisted(arg, out, in, halo);
+      dslash::DslashPolicyTune<decltype(twisted)> policy(twisted, in, halo, profile);
+    }
+#endif
+  };
+
+} // namespace quda
diff --git a/lib/dslash_twisted_clover_preconditioned.in.cu b/lib/dslash_twisted_clover_preconditioned.in.cu
new file mode 100644
index 0000000000..e59bc56d26
--- /dev/null
+++ b/lib/dslash_twisted_clover_preconditioned.in.cu
@@ -0,0 +1,24 @@
+#include <dslash_twisted_clover_preconditioned.hpp>
+
+
+namespace quda
+{
+
+  constexpr QudaPrecision precision = QUDA_@QUDA_DSLASH_PREC@_PRECISION;
+  constexpr int nColor = @QUDA_DSLASH_NCOLOR@;
+  constexpr int reconI = @QUDA_DSLASH_RECONI@;
+  constexpr bool distance_pc = @QUDA_DSLASH_DISTANCE@;
+
+  typedef @QUDA_DSLASH_DDARG@ DDArg;
+  typedef precision_type_mapper<precision>::type Float;
+
+  template struct TwistedCloverPreconditionedApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>;
+
+  template TwistedCloverPreconditionedApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>::TwistedCloverPreconditionedApply<distance_pc>(
+                  cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+                  cvector_ref<const ColorSpinorField> &x,const GaugeField &U,
+                  const CloverField &C, double a, double b,bool xpay, int parity,
+                  bool dagger, const int *comm_override,
+                  DistanceType<distance_pc>, TimeProfile &profile);
+
+} // namespace quda
diff --git a/lib/dslash_twisted_mass.cu b/lib/dslash_twisted_mass.cu
index a686c8ec6b..6194a52c63 100644
--- a/lib/dslash_twisted_mass.cu
+++ b/lib/dslash_twisted_mass.cu
@@ -1,69 +1,10 @@
-#include <gauge_field.h>
-#include <color_spinor_field.h>
-#include <dslash.h>
-#include <worker.h>
-
-#include <dslash_policy.hpp>
-#include <kernels/dslash_twisted_mass.cuh>
-
-/**
-   This is the basic gauged twisted-mass operator
-*/
+#define SIGNATURE_ONLY
+#include <dslash_twisted_mass.hpp>
+#undef SIGNATURE_ONLY
 
 namespace quda
 {
 
-  template <typename Arg> class TwistedMass : public Dslash<twistedMass, Arg>
-  {
-    using Dslash = Dslash<twistedMass, Arg>;
-    using Dslash::arg;
-    using Dslash::in;
-
-  public:
-    TwistedMass(Arg &arg, cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
-                const ColorSpinorField &halo) :
-      Dslash(arg, out, in, halo)
-    {
-    }
-
-    void apply(const qudaStream_t &stream)
-    {
-      TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
-      Dslash::setParam(tp);
-      if (arg.xpay)
-        Dslash::template instantiate<packShmem, true>(tp, stream);
-      else
-        errorQuda("Twisted-mass operator only defined for xpay=true");
-    }
-
-    long long flops() const
-    {
-      long long flops = Dslash::flops();
-      switch (arg.kernel_type) {
-      case INTERIOR_KERNEL:
-      case UBER_KERNEL:
-      case KERNEL_POLICY:
-        flops += in.size() * 2 * in.Ncolor() * 4 * 2 * in.Volume(); // complex * Nc * Ns * fma * vol
-        break;
-      default: break; // twisted-mass flops are in the interior kernel
-      }
-      return flops;
-    }
-  };
-
-  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon> struct TwistedMassApply {
-
-    TwistedMassApply(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
-                     cvector_ref<const ColorSpinorField> &x, const GaugeField &U, double a, double b, int parity,
-                     bool dagger, const int *comm_override, TimeProfile &profile)
-    {
-      constexpr int nDim = 4;
-      auto halo = ColorSpinorField::create_comms_batch(in);
-      TwistedMassArg<Float, nColor, nDim, DDArg, recon> arg(out, in, halo, U, a, b, x, parity, dagger, comm_override);
-      TwistedMass<decltype(arg)> twisted(arg, out, in, halo);
-      dslash::DslashPolicyTune<decltype(twisted)> policy(twisted, in, halo, profile);
-    }
-  };
 
   // Apply the twisted-mass Dslash operator
   // out(x) = M*in = (1 + i*b*gamma_5)*in(x) + a*\sum_mu U_{-\mu}(x)in(x+mu) + U^\dagger_mu(x-mu)in(x-mu)
@@ -73,7 +14,8 @@ namespace quda
                         bool dagger, const int *comm_override, TimeProfile &profile)
   {
     if constexpr (is_enabled<QUDA_TWISTED_MASS_DSLASH>()) {
-      instantiate<TwistedMassApply>(out, in, x, U, a, b, parity, dagger, comm_override, profile);
+      auto dummy = DistanceType<false>();
+      instantiate<TwistedMassApply>(out, in, x, U, a, b, parity, dagger, comm_override, dummy, profile);
     } else {
       errorQuda("Twisted-mass operator has not been built");
     }
diff --git a/lib/dslash_twisted_mass.hpp b/lib/dslash_twisted_mass.hpp
new file mode 100644
index 0000000000..ff8d210a7b
--- /dev/null
+++ b/lib/dslash_twisted_mass.hpp
@@ -0,0 +1,74 @@
+#include <gauge_field.h>
+#include <color_spinor_field.h>
+#include <dslash.h>
+#include <worker.h>
+
+#include <dslash_policy.hpp>
+#include <kernels/dslash_twisted_mass.cuh>
+
+/**
+   This is the basic gauged twisted-mass operator
+*/
+
+namespace quda
+{
+
+  template <typename Arg> class TwistedMass : public Dslash<twistedMass, Arg>
+  {
+    using Dslash = Dslash<twistedMass, Arg>;
+    using Dslash::arg;
+    using Dslash::in;
+
+  public:
+    TwistedMass(Arg &arg, cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+                const ColorSpinorField &halo) :
+      Dslash(arg, out, in, halo)
+    {
+    }
+
+    void apply(const qudaStream_t &stream)
+    {
+      TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
+      Dslash::setParam(tp);
+      if (arg.xpay)
+        Dslash::template instantiate<packShmem, true>(tp, stream);
+      else
+        errorQuda("Twisted-mass operator only defined for xpay=true");
+    }
+
+    long long flops() const
+    {
+      long long flops = Dslash::flops();
+      switch (arg.kernel_type) {
+      case INTERIOR_KERNEL:
+      case UBER_KERNEL:
+      case KERNEL_POLICY:
+        flops += in.size() * 2 * in.Ncolor() * 4 * 2 * in.Volume(); // complex * Nc * Ns * fma * vol
+        break;
+      default: break; // twisted-mass flops are in the interior kernel
+      }
+      return flops;
+    }
+  };
+  template <bool distance_pc> struct DistanceType {
+  };
+
+  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon> struct TwistedMassApply {
+    template <bool distance_pc>
+    TwistedMassApply(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+                     cvector_ref<const ColorSpinorField> &x, const GaugeField &U, double a, double b, int parity,
+                     bool dagger, const int *comm_override, DistanceType<distance_pc>, TimeProfile &profile)
+#ifdef SIGNATURE_ONLY
+    ;
+#else
+    {
+      constexpr int nDim = 4;
+      auto halo = ColorSpinorField::create_comms_batch(in);
+      TwistedMassArg<Float, nColor, nDim, DDArg, recon> arg(out, in, halo, U, a, b, x, parity, dagger, comm_override);
+      TwistedMass<decltype(arg)> twisted(arg, out, in, halo);
+      dslash::DslashPolicyTune<decltype(twisted)> policy(twisted, in, halo, profile);
+    }
+#endif
+  };
+
+} // namespace quda
diff --git a/lib/dslash_twisted_mass.in.cu b/lib/dslash_twisted_mass.in.cu
new file mode 100644
index 0000000000..656bcf83ed
--- /dev/null
+++ b/lib/dslash_twisted_mass.in.cu
@@ -0,0 +1,20 @@
+#include <dslash_twisted_mass.hpp>
+
+namespace quda
+{
+  constexpr QudaPrecision precision = QUDA_@QUDA_DSLASH_PREC@_PRECISION;
+  constexpr int nColor = @QUDA_DSLASH_NCOLOR@;
+  constexpr int reconI = @QUDA_DSLASH_RECONI@;
+
+  typedef @QUDA_DSLASH_DDARG@ DDArg;
+  typedef precision_type_mapper<precision>::type Float;
+  constexpr bool distance_pc = @QUDA_DSLASH_DISTANCE@;
+
+  template struct TwistedMassApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>; 
+
+  template TwistedMassApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>::TwistedMassApply<distance_pc>(
+		  cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+                  cvector_ref<const ColorSpinorField> &x, const GaugeField &U, double a, double b, int parity,
+                  bool dagger, const int *comm_override, DistanceType<distance_pc>, TimeProfile &profile);
+
+} // namespace quda
diff --git a/lib/dslash_wilson_clover.cu b/lib/dslash_wilson_clover.cu
index 96413eeba8..9992b64d27 100644
--- a/lib/dslash_wilson_clover.cu
+++ b/lib/dslash_wilson_clover.cu
@@ -1,4 +1,6 @@
+#define SIGNATURE_ONLY
 #include <dslash_wilson_clover.hpp>
+#undef SIGNATURE_ONLY
 
 /**
    This is the Wilson-clover linear operator
diff --git a/lib/dslash_wilson_clover.hpp b/lib/dslash_wilson_clover.hpp
index 7caa9df101..59ae5b6a96 100644
--- a/lib/dslash_wilson_clover.hpp
+++ b/lib/dslash_wilson_clover.hpp
@@ -72,12 +72,14 @@ namespace quda
   };
 
   template <typename Float, int nColor, typename DDArg, QudaReconstructType recon> struct WilsonCloverApply {
-
     template <bool distance_pc>
     WilsonCloverApply(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
                       cvector_ref<const ColorSpinorField> &x, const GaugeField &U, const CloverField &A, double a,
                       double alpha0, int t0, int parity, bool dagger, const int *comm_override,
                       DistanceType<distance_pc>, TimeProfile &profile)
+#ifdef SIGNATURE_ONLY
+    ;
+#else
     {
       constexpr int nDim = 4;
       auto halo = ColorSpinorField::create_comms_batch(in);
@@ -87,6 +89,7 @@ namespace quda
 
       dslash::DslashPolicyTune<decltype(wilson)> policy(wilson, in, halo, profile);
     }
+#endif
   };
 
 } // namespace quda
diff --git a/lib/dslash_wilson_clover.in.cu b/lib/dslash_wilson_clover.in.cu
new file mode 100644
index 0000000000..899f926bff
--- /dev/null
+++ b/lib/dslash_wilson_clover.in.cu
@@ -0,0 +1,27 @@
+#include <dslash_wilson_clover.hpp>
+
+/**
+   This is the Wilson-clover linear operator
+*/
+
+namespace quda
+{
+
+
+  constexpr QudaPrecision precision = QUDA_@QUDA_DSLASH_PREC@_PRECISION;
+  constexpr int nColor = @QUDA_DSLASH_NCOLOR@;
+  constexpr int reconI = @QUDA_DSLASH_RECONI@;
+  constexpr bool distance_pc = @QUDA_DSLASH_DISTANCE@;
+
+  typedef @QUDA_DSLASH_DDARG@ DDArg;
+  typedef precision_type_mapper<precision>::type Float;
+
+  template struct WilsonCloverApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>;
+
+  template WilsonCloverApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>::WilsonCloverApply<distance_pc>(
+			  cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+			  cvector_ref<const ColorSpinorField> &x, const GaugeField &U, const CloverField &A, double a,
+			  double alpha0, int t0, int parity, bool dagger, const int *comm_override,
+			  DistanceType<distance_pc>, TimeProfile &profile);
+
+} // namespace quda

From 514b492c325ef46195f8f4aa4320a99f8b4359b8 Mon Sep 17 00:00:00 2001
From: Ferenc Pittler <pittler@hiskp.uni-bonn.de>
Date: Mon, 14 Oct 2024 11:21:20 +0200
Subject: [PATCH 087/108] separate compilation units for staggered dslash

---
 lib/CMakeLists.txt                  |   6 +
 lib/dslash_improved_staggered.cu    | 169 +-------------------------
 lib/dslash_improved_staggered.hpp   | 179 ++++++++++++++++++++++++++++
 lib/dslash_improved_staggered.in.cu |  22 ++++
 lib/dslash_staggered.cu             |  81 +------------
 lib/dslash_staggered.hpp            |  94 +++++++++++++++
 lib/dslash_staggered.in.cu          |  21 ++++
 7 files changed, 332 insertions(+), 240 deletions(-)
 create mode 100644 lib/dslash_improved_staggered.hpp
 create mode 100644 lib/dslash_improved_staggered.in.cu
 create mode 100644 lib/dslash_staggered.hpp
 create mode 100644 lib/dslash_staggered.in.cu

diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt
index 287c19ff99..8e0d419ea2 100644
--- a/lib/CMakeLists.txt
+++ b/lib/CMakeLists.txt
@@ -254,6 +254,12 @@ foreach(QUDA_DSLASH_PREC ${QUDA_DSLASH_PREC_LIST})
     foreach(QUDA_DSLASH_DDARG ${QUDA_DSLASH_DDARG_LIST})
       foreach(QUDA_DSLASH_RECONI ${QUDA_DSLASH_RECONI_LIST})
 	foreach(QUDA_DSLASH_DISTANCE ${QUDA_DSLASH_DISTANCE_LIST})
+	  set(TMP_FILENAME "dslash_staggered_${QUDA_DSLASH_PREC}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
+	  configure_file(dslash_staggered.in.cu ${TMP_FILENAME} @ONLY)
+          list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
+          set(TMP_FILENAME "dslash_improved_staggered_${QUDA_DSLASH_PREC}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
+          configure_file(dslash_improved_staggered.in.cu ${TMP_FILENAME} @ONLY)
+          list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
 	  set(TMP_FILENAME "dslash_wilson_${QUDA_DSLASH_PREC}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
           configure_file(dslash_wilson.in.cu ${TMP_FILENAME} @ONLY)
           list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
diff --git a/lib/dslash_improved_staggered.cu b/lib/dslash_improved_staggered.cu
index 99bd2d9080..6f1d735e92 100644
--- a/lib/dslash_improved_staggered.cu
+++ b/lib/dslash_improved_staggered.cu
@@ -1,15 +1,6 @@
-#include <dslash.h>
-#include <worker.h>
-#include <dslash_helper.cuh>
-#include <color_spinor_field_order.h>
-#include <gauge_field_order.h>
-#include <color_spinor.h>
-#include <dslash_helper.cuh>
-#include <index_helper.cuh>
-#include <gauge_field.h>
-
-#include <dslash_policy.hpp>
-#include <kernels/dslash_staggered.cuh>
+#define SIGNATURE_ONLY
+#include <dslash_improved_staggered.hpp>
+#undef SIGNATURE_ONLY
 
 /**
    This is a staggered Dirac operator
@@ -18,156 +9,6 @@
 namespace quda
 {
 
-  template <typename Arg> class Staggered : public Dslash<staggered, Arg>
-  {
-    using Dslash = Dslash<staggered, Arg>;
-    using Dslash::arg;
-    using Dslash::halo;
-    using Dslash::in;
-    const GaugeField &L;
-
-  public:
-    Staggered(Arg &arg, cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
-              const ColorSpinorField &halo, const GaugeField &L) :
-      Dslash(arg, out, in, halo), L(L)
-    {
-    }
-
-    void apply(const qudaStream_t &stream)
-    {
-      TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
-      Dslash::setParam(tp);
-      // operator is anti-Hermitian so do not instantiate dagger
-      if (arg.nParity == 1) {
-        if (arg.xpay)
-          Dslash::template instantiate<packStaggeredShmem, 1, false, true>(tp, stream);
-        else
-          Dslash::template instantiate<packStaggeredShmem, 1, false, false>(tp, stream);
-      } else if (arg.nParity == 2) {
-        if (arg.xpay)
-          Dslash::template instantiate<packStaggeredShmem, 2, false, true>(tp, stream);
-        else
-          Dslash::template instantiate<packStaggeredShmem, 2, false, false>(tp, stream);
-      }
-    }
-
-    /*
-      per direction / dimension flops
-      SU(3) matrix-vector flops = (8 Nc - 2) * Nc
-      xpay = 2 * 2 * Nc * Ns
-
-      So for the full dslash we have
-      flops = (2 * 2 * Nd * (8*Nc-2) * Nc)  +  ((2 * 2 * Nd - 1) * 2 * Nc * Ns)
-      flops_xpay = flops + 2 * 2 * Nc * Ns
-
-      For Asqtad this should give 1146 for Nc=3,Ns=2 and 1158 for the axpy equivalent
-    */
-    long long flops() const
-    {
-      int mv_flops = (8 * in.Ncolor() - 2) * in.Ncolor(); // SU(3) matrix-vector flops
-      int ghost_flops = (3 + 1) * (mv_flops + 2 * in.Ncolor() * in.Nspin());
-      int xpay_flops = 2 * 2 * in.Ncolor() * in.Nspin(); // multiply and add per real component
-      int num_dir = 2 * 4;                               // hard code factor of 4 in direction since fields may be 5-d
-
-      long long flops_ = 0;
-
-      switch (arg.kernel_type) {
-      case EXTERIOR_KERNEL_X:
-      case EXTERIOR_KERNEL_Y:
-      case EXTERIOR_KERNEL_Z:
-      case EXTERIOR_KERNEL_T: flops_ = ghost_flops * 2 * halo.GhostFace()[arg.kernel_type]; break;
-      case EXTERIOR_KERNEL_ALL: {
-        long long ghost_sites
-          = 2 * (halo.GhostFace()[0] + halo.GhostFace()[1] + halo.GhostFace()[2] + halo.GhostFace()[3]);
-        flops_ = ghost_flops * ghost_sites;
-        break;
-      }
-      case INTERIOR_KERNEL:
-      case UBER_KERNEL:
-      case KERNEL_POLICY: {
-        long long sites = halo.Volume();
-        flops_ = (2 * num_dir * mv_flops + // SU(3) matrix-vector multiplies
-                  (2 * num_dir - 1) * 2 * in.Ncolor() * in.Nspin())
-          * sites;                                  // accumulation
-        if (arg.xpay) flops_ += xpay_flops * sites; // axpy is always on interior
-
-        if (arg.kernel_type == KERNEL_POLICY) break;
-        // now correct for flops done by exterior kernel
-        long long ghost_sites = 0;
-        for (int d = 0; d < 4; d++)
-          if (arg.commDim[d]) ghost_sites += 2 * halo.GhostFace()[d];
-        flops_ -= ghost_flops * ghost_sites;
-
-        break;
-      }
-      }
-      return flops_;
-    }
-
-    long long bytes() const
-    {
-      int gauge_bytes_fat = QUDA_RECONSTRUCT_NO * in.Precision();
-      int gauge_bytes_long = L.Reconstruct() * in.Precision();
-      int spinor_bytes = 2 * in.Ncolor() * in.Nspin() * in.Precision() + (isFixed<typename Arg::Float>::value ? sizeof(float) : 0);
-      int ghost_bytes = 3 * (spinor_bytes + gauge_bytes_long) + (spinor_bytes + gauge_bytes_fat)
-        + 3 * 2 * spinor_bytes; // last term is the accumulator load/store through the face
-      int num_dir = 2 * 4;      // set to 4-d since we take care of 5-d fermions in derived classes where necessary
-
-      long long bytes_ = 0;
-
-      switch (arg.kernel_type) {
-      case EXTERIOR_KERNEL_X:
-      case EXTERIOR_KERNEL_Y:
-      case EXTERIOR_KERNEL_Z:
-      case EXTERIOR_KERNEL_T: bytes_ = ghost_bytes * 2 * halo.GhostFace()[arg.kernel_type]; break;
-      case EXTERIOR_KERNEL_ALL: {
-        long long ghost_sites
-          = 2 * (halo.GhostFace()[0] + halo.GhostFace()[1] + halo.GhostFace()[2] + halo.GhostFace()[3]);
-        bytes_ = ghost_bytes * ghost_sites;
-        break;
-      }
-      case INTERIOR_KERNEL:
-      case UBER_KERNEL:
-      case KERNEL_POLICY: {
-        long long sites = halo.Volume();
-        bytes_ = (num_dir * (gauge_bytes_fat + gauge_bytes_long) + // gauge reads
-                  num_dir * 2 * spinor_bytes +                     // spinor reads
-                  spinor_bytes)
-          * sites; // spinor write
-        if (arg.xpay) bytes_ += spinor_bytes;
-
-        if (arg.kernel_type == KERNEL_POLICY) break;
-        // now correct for bytes done by exterior kernel
-        long long ghost_sites = 0;
-        for (int d = 0; d < 4; d++)
-          if (arg.commDim[d]) ghost_sites += 2 * halo.GhostFace()[d];
-        bytes_ -= ghost_bytes * ghost_sites;
-
-        break;
-      }
-      }
-      return bytes_;
-    }
-
-  };
-
-  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon_l> struct ImprovedStaggeredApply {
-
-    ImprovedStaggeredApply(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
-                           cvector_ref<const ColorSpinorField> &x, const GaugeField &L, const GaugeField &U, double a,
-                           int parity, bool dagger, const int *comm_override, TimeProfile &profile)
-    {
-      constexpr int nDim = 4;
-      constexpr bool improved = true;
-      constexpr QudaReconstructType recon_u = QUDA_RECONSTRUCT_NO;
-      auto halo = ColorSpinorField::create_comms_batch(in);
-      StaggeredArg<Float, nColor, nDim, DDArg, recon_u, recon_l, improved> arg(out, in, halo, U, L, a, x, parity,
-                                                                               dagger, comm_override);
-      Staggered<decltype(arg)> staggered(arg, out, in, halo, L);
-      dslash::DslashPolicyTune<decltype(staggered)> policy(staggered, in, halo, profile);
-    }
-  };
-
   void ApplyImprovedStaggered(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
                               const GaugeField &U, const GaugeField &L, double a, cvector_ref<const ColorSpinorField> &x,
                               int parity, bool dagger, const int *comm_override, TimeProfile &profile)
@@ -178,9 +19,9 @@ namespace quda
           errorQuda("partitioned dimension with local size less than 6 is not supported in improved staggered dslash");
         }
       }
-
+      auto dummy = DistanceType<false>();
       // L must be first gauge field argument since we template on long reconstruct
-      instantiate<ImprovedStaggeredApply, ReconstructStaggered>(out, in, x, L, U, a, parity, dagger, comm_override,
+      instantiate<ImprovedStaggeredApply, ReconstructStaggered>(out, in, x, L, U, a, parity, dagger, comm_override, dummy, 
                                                                 profile);
     } else {
       errorQuda("Improved staggered operator has not been built");
diff --git a/lib/dslash_improved_staggered.hpp b/lib/dslash_improved_staggered.hpp
new file mode 100644
index 0000000000..abbea494a5
--- /dev/null
+++ b/lib/dslash_improved_staggered.hpp
@@ -0,0 +1,179 @@
+#include <dslash.h>
+#include <worker.h>
+#include <dslash_helper.cuh>
+#include <color_spinor_field_order.h>
+#include <gauge_field_order.h>
+#include <color_spinor.h>
+#include <dslash_helper.cuh>
+#include <index_helper.cuh>
+#include <gauge_field.h>
+
+#include <dslash_policy.hpp>
+#include <kernels/dslash_staggered.cuh>
+
+/**
+   This is a staggered Dirac operator
+*/
+
+namespace quda
+{
+
+  template <typename Arg> class Staggered : public Dslash<staggered, Arg>
+  {
+    using Dslash = Dslash<staggered, Arg>;
+    using Dslash::arg;
+    using Dslash::halo;
+    using Dslash::in;
+    const GaugeField &L;
+
+  public:
+    Staggered(Arg &arg, cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+              const ColorSpinorField &halo, const GaugeField &L) :
+      Dslash(arg, out, in, halo), L(L)
+    {
+    }
+
+    void apply(const qudaStream_t &stream)
+    {
+      TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
+      Dslash::setParam(tp);
+      // operator is anti-Hermitian so do not instantiate dagger
+      if (arg.nParity == 1) {
+        if (arg.xpay)
+          Dslash::template instantiate<packStaggeredShmem, 1, false, true>(tp, stream);
+        else
+          Dslash::template instantiate<packStaggeredShmem, 1, false, false>(tp, stream);
+      } else if (arg.nParity == 2) {
+        if (arg.xpay)
+          Dslash::template instantiate<packStaggeredShmem, 2, false, true>(tp, stream);
+        else
+          Dslash::template instantiate<packStaggeredShmem, 2, false, false>(tp, stream);
+      }
+    }
+
+    /*
+      per direction / dimension flops
+      SU(3) matrix-vector flops = (8 Nc - 2) * Nc
+      xpay = 2 * 2 * Nc * Ns
+
+      So for the full dslash we have
+      flops = (2 * 2 * Nd * (8*Nc-2) * Nc)  +  ((2 * 2 * Nd - 1) * 2 * Nc * Ns)
+      flops_xpay = flops + 2 * 2 * Nc * Ns
+
+      For Asqtad this should give 1146 for Nc=3,Ns=2 and 1158 for the axpy equivalent
+    */
+    long long flops() const
+    {
+      int mv_flops = (8 * in.Ncolor() - 2) * in.Ncolor(); // SU(3) matrix-vector flops
+      int ghost_flops = (3 + 1) * (mv_flops + 2 * in.Ncolor() * in.Nspin());
+      int xpay_flops = 2 * 2 * in.Ncolor() * in.Nspin(); // multiply and add per real component
+      int num_dir = 2 * 4;                               // hard code factor of 4 in direction since fields may be 5-d
+
+      long long flops_ = 0;
+
+      switch (arg.kernel_type) {
+      case EXTERIOR_KERNEL_X:
+      case EXTERIOR_KERNEL_Y:
+      case EXTERIOR_KERNEL_Z:
+      case EXTERIOR_KERNEL_T: flops_ = ghost_flops * 2 * halo.GhostFace()[arg.kernel_type]; break;
+      case EXTERIOR_KERNEL_ALL: {
+        long long ghost_sites
+          = 2 * (halo.GhostFace()[0] + halo.GhostFace()[1] + halo.GhostFace()[2] + halo.GhostFace()[3]);
+        flops_ = ghost_flops * ghost_sites;
+        break;
+      }
+      case INTERIOR_KERNEL:
+      case UBER_KERNEL:
+      case KERNEL_POLICY: {
+        long long sites = halo.Volume();
+        flops_ = (2 * num_dir * mv_flops + // SU(3) matrix-vector multiplies
+                  (2 * num_dir - 1) * 2 * in.Ncolor() * in.Nspin())
+          * sites;                                  // accumulation
+        if (arg.xpay) flops_ += xpay_flops * sites; // axpy is always on interior
+
+        if (arg.kernel_type == KERNEL_POLICY) break;
+        // now correct for flops done by exterior kernel
+        long long ghost_sites = 0;
+        for (int d = 0; d < 4; d++)
+          if (arg.commDim[d]) ghost_sites += 2 * halo.GhostFace()[d];
+        flops_ -= ghost_flops * ghost_sites;
+
+        break;
+      }
+      }
+      return flops_;
+    }
+
+    long long bytes() const
+    {
+      int gauge_bytes_fat = QUDA_RECONSTRUCT_NO * in.Precision();
+      int gauge_bytes_long = L.Reconstruct() * in.Precision();
+      int spinor_bytes = 2 * in.Ncolor() * in.Nspin() * in.Precision() + (isFixed<typename Arg::Float>::value ? sizeof(float) : 0);
+      int ghost_bytes = 3 * (spinor_bytes + gauge_bytes_long) + (spinor_bytes + gauge_bytes_fat)
+        + 3 * 2 * spinor_bytes; // last term is the accumulator load/store through the face
+      int num_dir = 2 * 4;      // set to 4-d since we take care of 5-d fermions in derived classes where necessary
+
+      long long bytes_ = 0;
+
+      switch (arg.kernel_type) {
+      case EXTERIOR_KERNEL_X:
+      case EXTERIOR_KERNEL_Y:
+      case EXTERIOR_KERNEL_Z:
+      case EXTERIOR_KERNEL_T: bytes_ = ghost_bytes * 2 * halo.GhostFace()[arg.kernel_type]; break;
+      case EXTERIOR_KERNEL_ALL: {
+        long long ghost_sites
+          = 2 * (halo.GhostFace()[0] + halo.GhostFace()[1] + halo.GhostFace()[2] + halo.GhostFace()[3]);
+        bytes_ = ghost_bytes * ghost_sites;
+        break;
+      }
+      case INTERIOR_KERNEL:
+      case UBER_KERNEL:
+      case KERNEL_POLICY: {
+        long long sites = halo.Volume();
+        bytes_ = (num_dir * (gauge_bytes_fat + gauge_bytes_long) + // gauge reads
+                  num_dir * 2 * spinor_bytes +                     // spinor reads
+                  spinor_bytes)
+          * sites; // spinor write
+        if (arg.xpay) bytes_ += spinor_bytes;
+
+        if (arg.kernel_type == KERNEL_POLICY) break;
+        // now correct for bytes done by exterior kernel
+        long long ghost_sites = 0;
+        for (int d = 0; d < 4; d++)
+          if (arg.commDim[d]) ghost_sites += 2 * halo.GhostFace()[d];
+        bytes_ -= ghost_bytes * ghost_sites;
+
+        break;
+      }
+      }
+      return bytes_;
+    }
+
+  };
+
+  template <bool distance_pc> struct DistanceType {
+  };
+
+
+  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon_l> struct ImprovedStaggeredApply {
+    template <bool distance_pc>
+    ImprovedStaggeredApply(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+                           cvector_ref<const ColorSpinorField> &x, const GaugeField &L, const GaugeField &U, double a,
+                           int parity, bool dagger, const int *comm_override,DistanceType<distance_pc>, TimeProfile &profile)
+#ifdef SIGNATURE_ONLY
+    ;
+#else
+    {
+      constexpr int nDim = 4;
+      constexpr bool improved = true;
+      constexpr QudaReconstructType recon_u = QUDA_RECONSTRUCT_NO;
+      auto halo = ColorSpinorField::create_comms_batch(in);
+      StaggeredArg<Float, nColor, nDim, DDArg, recon_u, recon_l, improved> arg(out, in, halo, U, L, a, x, parity,
+                                                                               dagger, comm_override);
+      Staggered<decltype(arg)> staggered(arg, out, in, halo, L);
+      dslash::DslashPolicyTune<decltype(staggered)> policy(staggered, in, halo, profile);
+    }
+#endif
+  };
+
+} // namespace quda
diff --git a/lib/dslash_improved_staggered.in.cu b/lib/dslash_improved_staggered.in.cu
new file mode 100644
index 0000000000..5b7494f232
--- /dev/null
+++ b/lib/dslash_improved_staggered.in.cu
@@ -0,0 +1,22 @@
+#include <dslash_improved_staggered.hpp>
+  
+namespace quda
+{
+  constexpr QudaPrecision precision = QUDA_@QUDA_DSLASH_PREC@_PRECISION;
+  constexpr int nColor = @QUDA_DSLASH_NCOLOR@;
+  constexpr int reconI = @QUDA_DSLASH_RECONI@;
+  constexpr bool distance_pc = @QUDA_DSLASH_DISTANCE@;
+
+  typedef @QUDA_DSLASH_DDARG@ DDArg;
+  typedef precision_type_mapper<precision>::type Float;
+
+
+
+  template struct ImprovedStaggeredApply<Float, nColor, DDArg, ReconstructStaggered::recon[reconI]>;
+ 
+  template ImprovedStaggeredApply<Float, nColor, DDArg, ReconstructStaggered::recon[reconI]>::ImprovedStaggeredApply<distance_pc>(
+		  cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+                  cvector_ref<const ColorSpinorField> &x, const GaugeField &L, const GaugeField &U, double a,
+                  int parity, bool dagger, const int *comm_override,DistanceType<distance_pc>, TimeProfile &profile);
+
+} // namespace quda
diff --git a/lib/dslash_staggered.cu b/lib/dslash_staggered.cu
index 401605409a..33ed8e18fb 100644
--- a/lib/dslash_staggered.cu
+++ b/lib/dslash_staggered.cu
@@ -1,14 +1,7 @@
-#include <dslash.h>
-#include <worker.h>
-#include <dslash_helper.cuh>
-#include <color_spinor_field_order.h>
-#include <gauge_field_order.h>
-#include <color_spinor.h>
-#include <index_helper.cuh>
-#include <gauge_field.h>
+#define SIGNATURE_ONLY
+#include <dslash_staggered.hpp>
+#undef SIGNATURE_ONLY
 
-#include <dslash_policy.hpp>
-#include <kernels/dslash_staggered.cuh>
 
 /**
    This is a staggered Dirac operator
@@ -17,77 +10,13 @@
 namespace quda
 {
 
-  template <typename Arg> class Staggered : public Dslash<staggered, Arg>
-  {
-    using Dslash = Dslash<staggered, Arg>;
-    using Dslash::arg;
-
-  public:
-    Staggered(Arg &arg, cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
-              const ColorSpinorField &halo) :
-      Dslash(arg, out, in, halo)
-    {
-    }
-
-    void apply(const qudaStream_t &stream)
-    {
-      TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
-      Dslash::setParam(tp);
-      // operator is anti-Hermitian so do not instantiate dagger
-      if (arg.nParity == 1) {
-        if (arg.xpay)
-          Dslash::template instantiate<packStaggeredShmem, 1, false, true>(tp, stream);
-        else
-          Dslash::template instantiate<packStaggeredShmem, 1, false, false>(tp, stream);
-      } else if (arg.nParity == 2) {
-        if (arg.xpay)
-          Dslash::template instantiate<packStaggeredShmem, 2, false, true>(tp, stream);
-        else
-          Dslash::template instantiate<packStaggeredShmem, 2, false, false>(tp, stream);
-      }
-    }
-  };
-
-  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon_u> struct StaggeredApply {
-    StaggeredApply(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
-                   cvector_ref<const ColorSpinorField> &x, const GaugeField &U, double a, int parity, bool dagger,
-                   const int *comm_override, TimeProfile &profile)
-    {
-      constexpr int nDim = 4;
-      constexpr bool improved = false;
-      auto halo = ColorSpinorField::create_comms_batch(in);
-
-      if (U.StaggeredPhase() == QUDA_STAGGERED_PHASE_MILC
-          || (U.LinkType() == QUDA_GENERAL_LINKS && U.Reconstruct() == QUDA_RECONSTRUCT_NO)) {
-        if constexpr (is_enabled<QUDA_MILC_GAUGE_ORDER>()) {
-          StaggeredArg<Float, nColor, nDim, DDArg, recon_u, QUDA_RECONSTRUCT_NO, improved, QUDA_STAGGERED_PHASE_MILC> arg(
-            out, in, halo, U, U, a, x, parity, dagger, comm_override);
-          Staggered<decltype(arg)> staggered(arg, out, in, halo);
-
-          dslash::DslashPolicyTune<decltype(staggered)> policy(staggered, in, halo, profile);
-        } else {
-          errorQuda("MILC interface has not been built so MILC phase staggered fermions not enabled");
-        }
-      } else if (U.StaggeredPhase() == QUDA_STAGGERED_PHASE_TIFR) {
-        if constexpr (is_enabled<QUDA_TIFR_GAUGE_ORDER>()) {
-          StaggeredArg<Float, nColor, nDim, DDArg, recon_u, QUDA_RECONSTRUCT_NO, improved, QUDA_STAGGERED_PHASE_TIFR> arg(
-            out, in, halo, U, U, a, x, parity, dagger, comm_override);
-          Staggered<decltype(arg)> staggered(arg, out, in, halo);
-
-          dslash::DslashPolicyTune<decltype(staggered)> policy(staggered, in, halo, profile);
-        } else {
-          errorQuda("TIFR interface has not been built so TIFR phase taggered fermions not enabled");
-        }
-      }
-    }
-  };
-
   void ApplyStaggered(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in, const GaugeField &U,
                       double a, cvector_ref<const ColorSpinorField> &x, int parity, bool dagger,
                       const int *comm_override, TimeProfile &profile)
   {
     if constexpr (is_enabled<QUDA_STAGGERED_DSLASH>()) {
-      instantiate<StaggeredApply, ReconstructStaggered>(out, in, x, U, a, parity, dagger, comm_override, profile);
+      auto dummy = DistanceType<false>();
+      instantiate<StaggeredApply, ReconstructStaggered>(out, in, x, U, a, parity, dagger, comm_override, dummy, profile);
     } else {
       errorQuda("Staggered operator has not been built");
     }
diff --git a/lib/dslash_staggered.hpp b/lib/dslash_staggered.hpp
new file mode 100644
index 0000000000..5386ec7dc4
--- /dev/null
+++ b/lib/dslash_staggered.hpp
@@ -0,0 +1,94 @@
+#include <dslash.h>
+#include <worker.h>
+#include <dslash_helper.cuh>
+#include <color_spinor_field_order.h>
+#include <gauge_field_order.h>
+#include <color_spinor.h>
+#include <index_helper.cuh>
+#include <gauge_field.h>
+
+#include <dslash_policy.hpp>
+#include <kernels/dslash_staggered.cuh>
+
+/**
+   This is a staggered Dirac operator
+*/
+
+namespace quda
+{
+
+  template <typename Arg> class Staggered : public Dslash<staggered, Arg>
+  {
+    using Dslash = Dslash<staggered, Arg>;
+    using Dslash::arg;
+
+  public:
+    Staggered(Arg &arg, cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+              const ColorSpinorField &halo) :
+      Dslash(arg, out, in, halo)
+    {
+    }
+
+    void apply(const qudaStream_t &stream)
+    {
+      TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
+      Dslash::setParam(tp);
+      // operator is anti-Hermitian so do not instantiate dagger
+      if (arg.nParity == 1) {
+        if (arg.xpay)
+          Dslash::template instantiate<packStaggeredShmem, 1, false, true>(tp, stream);
+        else
+          Dslash::template instantiate<packStaggeredShmem, 1, false, false>(tp, stream);
+      } else if (arg.nParity == 2) {
+        if (arg.xpay)
+          Dslash::template instantiate<packStaggeredShmem, 2, false, true>(tp, stream);
+        else
+          Dslash::template instantiate<packStaggeredShmem, 2, false, false>(tp, stream);
+      }
+    }
+  };
+
+  template <bool distance_pc> struct DistanceType {
+  };
+
+
+  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon_u> struct StaggeredApply {
+    template <bool distance_pc>
+    StaggeredApply(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+                   cvector_ref<const ColorSpinorField> &x, const GaugeField &U, double a, int parity, bool dagger,
+                   const int *comm_override, DistanceType<distance_pc>, TimeProfile &profile)
+#ifdef SIGNATURE_ONLY
+    ;
+#else
+    {
+      constexpr int nDim = 4;
+      constexpr bool improved = false;
+      auto halo = ColorSpinorField::create_comms_batch(in);
+
+      if (U.StaggeredPhase() == QUDA_STAGGERED_PHASE_MILC
+          || (U.LinkType() == QUDA_GENERAL_LINKS && U.Reconstruct() == QUDA_RECONSTRUCT_NO)) {
+        if constexpr (is_enabled<QUDA_MILC_GAUGE_ORDER>()) {
+          StaggeredArg<Float, nColor, nDim, DDArg, recon_u, QUDA_RECONSTRUCT_NO, improved, QUDA_STAGGERED_PHASE_MILC> arg(
+            out, in, halo, U, U, a, x, parity, dagger, comm_override);
+          Staggered<decltype(arg)> staggered(arg, out, in, halo);
+
+          dslash::DslashPolicyTune<decltype(staggered)> policy(staggered, in, halo, profile);
+        } else {
+          errorQuda("MILC interface has not been built so MILC phase staggered fermions not enabled");
+        }
+      } else if (U.StaggeredPhase() == QUDA_STAGGERED_PHASE_TIFR) {
+        if constexpr (is_enabled<QUDA_TIFR_GAUGE_ORDER>()) {
+          StaggeredArg<Float, nColor, nDim, DDArg, recon_u, QUDA_RECONSTRUCT_NO, improved, QUDA_STAGGERED_PHASE_TIFR> arg(
+            out, in, halo, U, U, a, x, parity, dagger, comm_override);
+          Staggered<decltype(arg)> staggered(arg, out, in, halo);
+
+          dslash::DslashPolicyTune<decltype(staggered)> policy(staggered, in, halo, profile);
+        } else {
+          errorQuda("TIFR interface has not been built so TIFR phase taggered fermions not enabled");
+        }
+      }
+    }
+#endif
+  };
+
+} // namespace quda
diff --git a/lib/dslash_staggered.in.cu b/lib/dslash_staggered.in.cu
new file mode 100644
index 0000000000..9a85f6d378
--- /dev/null
+++ b/lib/dslash_staggered.in.cu
@@ -0,0 +1,21 @@
+#include <dslash_staggered.hpp>
+
+namespace quda
+{
+  constexpr QudaPrecision precision = QUDA_@QUDA_DSLASH_PREC@_PRECISION;
+  constexpr int nColor = @QUDA_DSLASH_NCOLOR@;
+  constexpr int reconI = @QUDA_DSLASH_RECONI@;
+  constexpr bool distance_pc = @QUDA_DSLASH_DISTANCE@;
+
+  typedef @QUDA_DSLASH_DDARG@ DDArg;
+  typedef precision_type_mapper<precision>::type Float;
+
+
+  template struct StaggeredApply<Float, nColor, DDArg, ReconstructStaggered::recon[reconI]>;
+    
+  template StaggeredApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>::StaggeredApply<distance_pc>(
+		  cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+		  cvector_ref<const ColorSpinorField> &x, const GaugeField &U, double a, int parity, bool dagger,
+                  const int *comm_override, DistanceType<distance_pc>,TimeProfile &profile);
+
+} // namespace quda

From b1b8d4b5b236d04b8b0f27914870726954bf904b Mon Sep 17 00:00:00 2001
From: Ferenc Pittler <pittler@hiskp.uni-bonn.de>
Date: Tue, 22 Oct 2024 12:30:21 +0200
Subject: [PATCH 088/108] lower the string for the precision and including the
 4d domain wall separate compilation

---
 lib/CMakeLists.txt              | 26 ++++++++------
 lib/dslash_domain_wall_4d.cu    | 50 +++-----------------------
 lib/dslash_domain_wall_4d.hpp   | 63 +++++++++++++++++++++++++++++++++
 lib/dslash_domain_wall_4d.in.cu | 22 ++++++++++++
 4 files changed, 105 insertions(+), 56 deletions(-)
 create mode 100644 lib/dslash_domain_wall_4d.hpp
 create mode 100644 lib/dslash_domain_wall_4d.in.cu

diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt
index 8e0d419ea2..a0e57535ae 100644
--- a/lib/CMakeLists.txt
+++ b/lib/CMakeLists.txt
@@ -250,41 +250,45 @@ message(STATUS "QUDA_DSLASH_RECONI_LIST=${QUDA_DSLASH_RECONI_LIST}")
 message(STATUS "QUDA_DSLASH_DISTANCE_LIST=${QUDA_DSLASH_DISTANCE_LIST}")
 
 foreach(QUDA_DSLASH_PREC ${QUDA_DSLASH_PREC_LIST})
+  string(TOLOWER "${QUDA_DSLASH_PREC}" QUDA_DSLASH_PREC_LOWER)
   foreach(QUDA_DSLASH_NCOLOR ${QUDA_DSLASH_NCOLOR_LIST})
     foreach(QUDA_DSLASH_DDARG ${QUDA_DSLASH_DDARG_LIST})
       foreach(QUDA_DSLASH_RECONI ${QUDA_DSLASH_RECONI_LIST})
 	foreach(QUDA_DSLASH_DISTANCE ${QUDA_DSLASH_DISTANCE_LIST})
-	  set(TMP_FILENAME "dslash_staggered_${QUDA_DSLASH_PREC}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
+          set(TMP_FILENAME "dslash_domain_wall_4d_${QUDA_DSLASH_PREC_LOWER}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
+          configure_file(dslash_domain_wall_4d.in.cu ${TMP_FILENAME} @ONLY)
+          list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
+	  set(TMP_FILENAME "dslash_staggered_${QUDA_DSLASH_PREC_LOWER}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
 	  configure_file(dslash_staggered.in.cu ${TMP_FILENAME} @ONLY)
           list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
-          set(TMP_FILENAME "dslash_improved_staggered_${QUDA_DSLASH_PREC}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
+          set(TMP_FILENAME "dslash_improved_staggered_${QUDA_DSLASH_PREC_LOWER}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
           configure_file(dslash_improved_staggered.in.cu ${TMP_FILENAME} @ONLY)
           list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
-	  set(TMP_FILENAME "dslash_wilson_${QUDA_DSLASH_PREC}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
+	  set(TMP_FILENAME "dslash_wilson_${QUDA_DSLASH_PREC_LOWER}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
           configure_file(dslash_wilson.in.cu ${TMP_FILENAME} @ONLY)
           list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
-	  set(TMP_FILENAME "dslash_twisted_clover_${QUDA_DSLASH_PREC}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
+	  set(TMP_FILENAME "dslash_twisted_clover_${QUDA_DSLASH_PREC_LOWER}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
           configure_file(dslash_twisted_clover.in.cu ${TMP_FILENAME} @ONLY)
           list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
-	  set(TMP_FILENAME "dslash_wilson_clover_${QUDA_DSLASH_PREC}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
+	  set(TMP_FILENAME "dslash_wilson_clover_${QUDA_DSLASH_PREC_LOWER}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
           configure_file(dslash_wilson_clover.in.cu ${TMP_FILENAME} @ONLY)
           list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
-          set(TMP_FILENAME "dslash_twisted_mass_${QUDA_DSLASH_PREC}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
+          set(TMP_FILENAME "dslash_twisted_mass_${QUDA_DSLASH_PREC_LOWER}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
           configure_file(dslash_twisted_mass.in.cu ${TMP_FILENAME} @ONLY)
           list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
-          set(TMP_FILENAME "dslash_ndeg_twisted_clover_${QUDA_DSLASH_PREC}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
+          set(TMP_FILENAME "dslash_ndeg_twisted_clover_${QUDA_DSLASH_PREC_LOWER}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
           configure_file(dslash_ndeg_twisted_clover.in.cu ${TMP_FILENAME} @ONLY)
           list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
-          set(TMP_FILENAME "dslash_ndeg_twisted_clover_preconditioned.${QUDA_DSLASH_PREC}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
+          set(TMP_FILENAME "dslash_ndeg_twisted_clover_preconditioned.${QUDA_DSLASH_PREC_LOWER}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
           configure_file(dslash_ndeg_twisted_clover_preconditioned.in.cu ${TMP_FILENAME} @ONLY)
           list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
-	  set(TMP_FILENAME "dslash_ndeg_twisted_mass_preconditioned.${QUDA_DSLASH_PREC}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
+	  set(TMP_FILENAME "dslash_ndeg_twisted_mass_preconditioned.${QUDA_DSLASH_PREC_LOWER}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
           configure_file(dslash_ndeg_twisted_mass_preconditioned.in.cu ${TMP_FILENAME} @ONLY)
           list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
-          set(TMP_FILENAME "dslash_ndeg_twisted_mass_${QUDA_DSLASH_PREC}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
+          set(TMP_FILENAME "dslash_ndeg_twisted_mass_${QUDA_DSLASH_PREC_LOWER}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
           configure_file(dslash_ndeg_twisted_mass.in.cu ${TMP_FILENAME} @ONLY)
           list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
-          set(TMP_FILENAME "dslash_twisted_clover_preconditioned.${QUDA_DSLASH_PREC}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
+          set(TMP_FILENAME "dslash_twisted_clover_preconditioned.${QUDA_DSLASH_PREC_LOWER}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
           configure_file(dslash_twisted_clover_preconditioned.in.cu ${TMP_FILENAME} @ONLY)
           list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
 	endforeach()
diff --git a/lib/dslash_domain_wall_4d.cu b/lib/dslash_domain_wall_4d.cu
index 86b2e5fe63..4089091df3 100644
--- a/lib/dslash_domain_wall_4d.cu
+++ b/lib/dslash_domain_wall_4d.cu
@@ -1,11 +1,6 @@
-#include <gauge_field.h>
-#include <color_spinor_field.h>
-#include <dslash.h>
-#include <worker.h>
-
-#include <dslash_policy.hpp>
-#include <kernels/dslash_domain_wall_4d.cuh>
-
+#define SIGNATURE_ONLY
+#include <dslash_domain_wall_4d.hpp>
+#undef SIGNATURE_ONLY
 /**
    This is the gauged domain-wall 4-d preconditioned operator.
 
@@ -16,42 +11,6 @@
 namespace quda
 {
 
-  template <typename Arg> class DomainWall4D : public Dslash<domainWall4D, Arg>
-  {
-    using Dslash = Dslash<domainWall4D, Arg>;
-    using Dslash::arg;
-    using Dslash::in;
-
-  public:
-    DomainWall4D(Arg &arg, cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
-                 const ColorSpinorField &halo) :
-      Dslash(arg, out, in, halo)
-    {
-    }
-
-    void apply(const qudaStream_t &stream)
-    {
-      TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
-      Dslash::setParam(tp);
-      Dslash::template instantiate<packShmem>(tp, stream);
-    }
-  };
-
-  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon> struct DomainWall4DApply {
-
-    DomainWall4DApply(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
-                      cvector_ref<const ColorSpinorField> &x, const GaugeField &U, double a, double m_5,
-                      const Complex *b_5, const Complex *c_5, int parity, bool dagger, const int *comm_override,
-                      TimeProfile &profile)
-    {
-      constexpr int nDim = 4;
-      auto halo = ColorSpinorField::create_comms_batch(in);
-      DomainWall4DArg<Float, nColor, nDim, DDArg, recon> arg(out, in, halo, U, a, m_5, b_5, c_5, a != 0.0, x, parity,
-                                                             dagger, comm_override);
-      DomainWall4D<decltype(arg)> dwf(arg, out, in, halo);
-      dslash::DslashPolicyTune<decltype(dwf)> policy(dwf, in, halo, profile);
-    }
-  };
 
   // Apply the 4-d preconditioned domain-wall Dslash operator
   // out(x) = M*in = in(x) + a*\sum_mu U_{-\mu}(x)in(x+mu) + U^\dagger_mu(x-mu)in(x-mu)
@@ -61,7 +20,8 @@ namespace quda
                          TimeProfile &profile)
   {
     if constexpr (is_enabled<QUDA_DOMAIN_WALL_4D_DSLASH>() || is_enabled<QUDA_TWISTED_CLOVER_DSLASH>()) {
-      instantiate<DomainWall4DApply>(out, in, x, U, a, m_5, b_5, c_5, parity, dagger, comm_override, profile);
+      auto dummy = DistanceType<false>();
+      instantiate<DomainWall4DApply>(out, in, x, U, a, m_5, b_5, c_5, parity, dagger, comm_override, dummy, profile);
     } else {
       errorQuda("Domain-wall dslash has not been built");
     }
diff --git a/lib/dslash_domain_wall_4d.hpp b/lib/dslash_domain_wall_4d.hpp
new file mode 100644
index 0000000000..a2aef1bd39
--- /dev/null
+++ b/lib/dslash_domain_wall_4d.hpp
@@ -0,0 +1,63 @@
+#include <gauge_field.h>
+#include <color_spinor_field.h>
+#include <dslash.h>
+#include <worker.h>
+
+#include <dslash_policy.hpp>
+#include <kernels/dslash_domain_wall_4d.cuh>
+
+/**
+   This is the gauged domain-wall 4-d preconditioned operator.
+
+   Note, for now, this just applies a batched 4-d dslash across the fifth
+   dimension.
+*/
+
+namespace quda
+{
+
+  template <typename Arg> class DomainWall4D : public Dslash<domainWall4D, Arg>
+  {
+    using Dslash = Dslash<domainWall4D, Arg>;
+    using Dslash::arg;
+    using Dslash::in;
+
+  public:
+    DomainWall4D(Arg &arg, cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+                 const ColorSpinorField &halo) :
+      Dslash(arg, out, in, halo)
+    {
+    }
+
+    void apply(const qudaStream_t &stream)
+    {
+      TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
+      Dslash::setParam(tp);
+      Dslash::template instantiate<packShmem>(tp, stream);
+    }
+  };
+
+  template <bool distance_pc> struct DistanceType {
+  };
+
+  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon> struct DomainWall4DApply {
+    template <bool distance_pc>
+    DomainWall4DApply(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+                      cvector_ref<const ColorSpinorField> &x, const GaugeField &U, double a, double m_5,
+                      const Complex *b_5, const Complex *c_5, int parity, bool dagger, const int *comm_override,DistanceType<distance_pc>,
+                      TimeProfile &profile)
+
+#ifdef SIGNATURE_ONLY
+	  ;
+#else
+    {
+      constexpr int nDim = 4;
+      auto halo = ColorSpinorField::create_comms_batch(in);
+      DomainWall4DArg<Float, nColor, nDim, DDArg, recon> arg(out, in, halo, U, a, m_5, b_5, c_5, a != 0.0, x, parity,
+                                                             dagger, comm_override);
+      DomainWall4D<decltype(arg)> dwf(arg, out, in, halo);
+      dslash::DslashPolicyTune<decltype(dwf)> policy(dwf, in, halo, profile);
+    }
+#endif
+  };
+} // namespace quda
diff --git a/lib/dslash_domain_wall_4d.in.cu b/lib/dslash_domain_wall_4d.in.cu
new file mode 100644
index 0000000000..dc7d205ac0
--- /dev/null
+++ b/lib/dslash_domain_wall_4d.in.cu
@@ -0,0 +1,22 @@
+#include <dslash_domain_wall_4d.hpp>
+
+
+namespace quda
+{
+
+  constexpr QudaPrecision precision = QUDA_@QUDA_DSLASH_PREC@_PRECISION;
+  constexpr int nColor = @QUDA_DSLASH_NCOLOR@;
+  constexpr int reconI = @QUDA_DSLASH_RECONI@;
+  constexpr bool distance_pc = @QUDA_DSLASH_DISTANCE@;
+
+  typedef @QUDA_DSLASH_DDARG@ DDArg;
+  typedef precision_type_mapper<precision>::type Float;
+
+  template struct DomainWall4DApply<Float, nColor, DDArg, ReconstructStaggered::recon[reconI]>;
+   
+  template DomainWall4DApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>:: 
+    DomainWall4DApply<distance_pc>(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+                      cvector_ref<const ColorSpinorField> &x, const GaugeField &U, double a, double m_5,
+                      const Complex *b_5, const Complex *c_5, int parity, bool dagger, const int *comm_override,DistanceType<distance_pc>,
+                      TimeProfile &profile);
+} // namespace quda

From 9dbd6057e7acfe821721f7c8c724285bf498ef3a Mon Sep 17 00:00:00 2001
From: Ferenc Pittler <pittler@hiskp.uni-bonn.de>
Date: Tue, 22 Oct 2024 13:25:08 +0200
Subject: [PATCH 089/108] changing ncol to nc and correcting a bug

---
 lib/CMakeLists.txt         | 24 ++++++++++++------------
 lib/dslash_staggered.in.cu |  1 +
 2 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt
index a0e57535ae..1f5c6922fb 100644
--- a/lib/CMakeLists.txt
+++ b/lib/CMakeLists.txt
@@ -255,40 +255,40 @@ foreach(QUDA_DSLASH_PREC ${QUDA_DSLASH_PREC_LIST})
     foreach(QUDA_DSLASH_DDARG ${QUDA_DSLASH_DDARG_LIST})
       foreach(QUDA_DSLASH_RECONI ${QUDA_DSLASH_RECONI_LIST})
 	foreach(QUDA_DSLASH_DISTANCE ${QUDA_DSLASH_DISTANCE_LIST})
-          set(TMP_FILENAME "dslash_domain_wall_4d_${QUDA_DSLASH_PREC_LOWER}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
+          set(TMP_FILENAME "dslash_domain_wall_4d_${QUDA_DSLASH_PREC_LOWER}_nc_${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
           configure_file(dslash_domain_wall_4d.in.cu ${TMP_FILENAME} @ONLY)
           list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
-	  set(TMP_FILENAME "dslash_staggered_${QUDA_DSLASH_PREC_LOWER}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
+	  set(TMP_FILENAME "dslash_staggered_${QUDA_DSLASH_PREC_LOWER}_nc_${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
 	  configure_file(dslash_staggered.in.cu ${TMP_FILENAME} @ONLY)
           list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
-          set(TMP_FILENAME "dslash_improved_staggered_${QUDA_DSLASH_PREC_LOWER}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
+          set(TMP_FILENAME "dslash_improved_staggered_${QUDA_DSLASH_PREC_LOWER}_nc_${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
           configure_file(dslash_improved_staggered.in.cu ${TMP_FILENAME} @ONLY)
           list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
-	  set(TMP_FILENAME "dslash_wilson_${QUDA_DSLASH_PREC_LOWER}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
+	  set(TMP_FILENAME "dslash_wilson_${QUDA_DSLASH_PREC_LOWER}_nc_${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
           configure_file(dslash_wilson.in.cu ${TMP_FILENAME} @ONLY)
           list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
-	  set(TMP_FILENAME "dslash_twisted_clover_${QUDA_DSLASH_PREC_LOWER}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
+	  set(TMP_FILENAME "dslash_twisted_clover_${QUDA_DSLASH_PREC_LOWER}_nc_${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
           configure_file(dslash_twisted_clover.in.cu ${TMP_FILENAME} @ONLY)
           list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
-	  set(TMP_FILENAME "dslash_wilson_clover_${QUDA_DSLASH_PREC_LOWER}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
+	  set(TMP_FILENAME "dslash_wilson_clover_${QUDA_DSLASH_PREC_LOWER}_nc_${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
           configure_file(dslash_wilson_clover.in.cu ${TMP_FILENAME} @ONLY)
           list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
-          set(TMP_FILENAME "dslash_twisted_mass_${QUDA_DSLASH_PREC_LOWER}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
+          set(TMP_FILENAME "dslash_twisted_mass_${QUDA_DSLASH_PREC_LOWER}_nc_${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
           configure_file(dslash_twisted_mass.in.cu ${TMP_FILENAME} @ONLY)
           list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
-          set(TMP_FILENAME "dslash_ndeg_twisted_clover_${QUDA_DSLASH_PREC_LOWER}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
+          set(TMP_FILENAME "dslash_ndeg_twisted_clover_${QUDA_DSLASH_PREC_LOWER}_nc_${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
           configure_file(dslash_ndeg_twisted_clover.in.cu ${TMP_FILENAME} @ONLY)
           list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
-          set(TMP_FILENAME "dslash_ndeg_twisted_clover_preconditioned.${QUDA_DSLASH_PREC_LOWER}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
+          set(TMP_FILENAME "dslash_ndeg_twisted_clover_preconditioned.${QUDA_DSLASH_PREC_LOWER}_nc_${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
           configure_file(dslash_ndeg_twisted_clover_preconditioned.in.cu ${TMP_FILENAME} @ONLY)
           list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
-	  set(TMP_FILENAME "dslash_ndeg_twisted_mass_preconditioned.${QUDA_DSLASH_PREC_LOWER}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
+	  set(TMP_FILENAME "dslash_ndeg_twisted_mass_preconditioned.${QUDA_DSLASH_PREC_LOWER}_nc_${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
           configure_file(dslash_ndeg_twisted_mass_preconditioned.in.cu ${TMP_FILENAME} @ONLY)
           list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
-          set(TMP_FILENAME "dslash_ndeg_twisted_mass_${QUDA_DSLASH_PREC_LOWER}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
+          set(TMP_FILENAME "dslash_ndeg_twisted_mass_${QUDA_DSLASH_PREC_LOWER}_nc_${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
           configure_file(dslash_ndeg_twisted_mass.in.cu ${TMP_FILENAME} @ONLY)
           list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
-          set(TMP_FILENAME "dslash_twisted_clover_preconditioned.${QUDA_DSLASH_PREC_LOWER}_col${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
+          set(TMP_FILENAME "dslash_twisted_clover_preconditioned.${QUDA_DSLASH_PREC_LOWER}_nc_${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
           configure_file(dslash_twisted_clover_preconditioned.in.cu ${TMP_FILENAME} @ONLY)
           list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
 	endforeach()
diff --git a/lib/dslash_staggered.in.cu b/lib/dslash_staggered.in.cu
index 9a85f6d378..2c757f0ce0 100644
--- a/lib/dslash_staggered.in.cu
+++ b/lib/dslash_staggered.in.cu
@@ -9,6 +9,7 @@ namespace quda
 
   typedef @QUDA_DSLASH_DDARG@ DDArg;
   typedef precision_type_mapper<precision>::type Float;
+  constexpr bool distance_pc = @QUDA_DSLASH_DISTANCE@;
 
 
   template struct StaggeredApply<Float, nColor, DDArg, ReconstructStaggered::recon[reconI]>;

From a1ea3bdd60fe396d78ab0c256bf498c150782284 Mon Sep 17 00:00:00 2001
From: Ferenc Pittler <pittler@hiskp.uni-bonn.de>
Date: Tue, 22 Oct 2024 14:05:50 +0200
Subject: [PATCH 090/108] typo corrected

---
 lib/dslash_domain_wall_4d.in.cu | 2 +-
 lib/dslash_staggered.in.cu      | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/lib/dslash_domain_wall_4d.in.cu b/lib/dslash_domain_wall_4d.in.cu
index dc7d205ac0..22ffecfda3 100644
--- a/lib/dslash_domain_wall_4d.in.cu
+++ b/lib/dslash_domain_wall_4d.in.cu
@@ -12,7 +12,7 @@ namespace quda
   typedef @QUDA_DSLASH_DDARG@ DDArg;
   typedef precision_type_mapper<precision>::type Float;
 
-  template struct DomainWall4DApply<Float, nColor, DDArg, ReconstructStaggered::recon[reconI]>;
+  template struct DomainWall4DApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>;
    
   template DomainWall4DApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>:: 
     DomainWall4DApply<distance_pc>(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
diff --git a/lib/dslash_staggered.in.cu b/lib/dslash_staggered.in.cu
index 2c757f0ce0..f17f6168a7 100644
--- a/lib/dslash_staggered.in.cu
+++ b/lib/dslash_staggered.in.cu
@@ -5,7 +5,6 @@ namespace quda
   constexpr QudaPrecision precision = QUDA_@QUDA_DSLASH_PREC@_PRECISION;
   constexpr int nColor = @QUDA_DSLASH_NCOLOR@;
   constexpr int reconI = @QUDA_DSLASH_RECONI@;
-  constexpr bool distance_pc = @QUDA_DSLASH_DISTANCE@;
 
   typedef @QUDA_DSLASH_DDARG@ DDArg;
   typedef precision_type_mapper<precision>::type Float;

From c28858397b4ea9af6de81f2c3eea4565d18e809c Mon Sep 17 00:00:00 2001
From: Ferenc Pittler <pittler@hiskp.uni-bonn.de>
Date: Tue, 22 Oct 2024 14:21:41 +0200
Subject: [PATCH 091/108] another typo corrected

---
 lib/dslash_staggered.in.cu | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/dslash_staggered.in.cu b/lib/dslash_staggered.in.cu
index f17f6168a7..c868557545 100644
--- a/lib/dslash_staggered.in.cu
+++ b/lib/dslash_staggered.in.cu
@@ -5,15 +5,15 @@ namespace quda
   constexpr QudaPrecision precision = QUDA_@QUDA_DSLASH_PREC@_PRECISION;
   constexpr int nColor = @QUDA_DSLASH_NCOLOR@;
   constexpr int reconI = @QUDA_DSLASH_RECONI@;
+  constexpr bool distance_pc = @QUDA_DSLASH_DISTANCE@;
 
   typedef @QUDA_DSLASH_DDARG@ DDArg;
   typedef precision_type_mapper<precision>::type Float;
-  constexpr bool distance_pc = @QUDA_DSLASH_DISTANCE@;
 
 
   template struct StaggeredApply<Float, nColor, DDArg, ReconstructStaggered::recon[reconI]>;
     
-  template StaggeredApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>::StaggeredApply<distance_pc>(
+  template StaggeredApply<Float, nColor, DDArg, ReconstructStaggered::recon[reconI]>::StaggeredApply<distance_pc>(
 		  cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
 		  cvector_ref<const ColorSpinorField> &x, const GaugeField &U, double a, int parity, bool dagger,
                   const int *comm_override, DistanceType<distance_pc>,TimeProfile &profile);

From d4e8b174f5dfd5728422045b95c9291753078cdd Mon Sep 17 00:00:00 2001
From: Ferenc Pittler <pittler@hiskp.uni-bonn.de>
Date: Sat, 26 Oct 2024 06:11:07 +0200
Subject: [PATCH 092/108] finalizing merge

---
 lib/inv_mr_quda.cpp | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/lib/inv_mr_quda.cpp b/lib/inv_mr_quda.cpp
index 2777ecec36..9987d011e6 100644
--- a/lib/inv_mr_quda.cpp
+++ b/lib/inv_mr_quda.cpp
@@ -113,20 +113,24 @@ namespace quda
 
         if (param.do_block_schwarz()) {
           if (param.schwarz_type == QUDA_MULTIPLICATIVE_SCHWARZ) {
+            for (auto i = 0u; i < b.size(); i++) {
             // Red or black active
-            Ar.DD(DD::reset, DD::red_black_type, step % 2 == 0 ? DD::red_active : DD::black_active);
-            r_sloppy.DD(DD::red_black_type, step % 2 == 0 ? DD::red_active : DD::black_active);
+              (Ar[i]).DD(DD::reset, DD::red_black_type, step % 2 == 0 ? DD::red_active : DD::black_active);
+              (r_sloppy[i]).DD(DD::red_black_type, step % 2 == 0 ? DD::red_active : DD::black_active);
+	    }
           } else {
             // Both red and black active but no hopping
-            Ar.DD(DD::reset, DD::red_black_type, DD::red_active, DD::black_active, DD::no_block_hopping);
-            r_sloppy.DD(DD::reset, DD::red_black_type, DD::red_active, DD::black_active, DD::no_block_hopping);
+            for (auto i = 0u; i < b.size(); i++) {
+              (Ar[i]).DD(DD::reset, DD::red_black_type, DD::red_active, DD::black_active, DD::no_block_hopping);
+              (r_sloppy[i]).DD(DD::reset, DD::red_black_type, DD::red_active, DD::black_active, DD::no_block_hopping);
+	    }
           }
         }
 
         blas::zero(x_sloppy); // can get rid of this for a special first update kernel
-        auto c2 = param.global_reduction == QUDA_BOOLEAN_TRUE ? r2 : blas::norm2(r); // c2 holds the initial r2
+        auto c2 = param.global_reduction == QUDA_BOOLEAN_TRUE ? r2 : blas::norm2(r); // c2 holds the initial r2  
         for (auto i = 0u; i < b.size(); i++) {
-          scale[i] = c2[i] > 0.0 ? sqrt(c2[i]) : 1.0;
+	  scale[i] = c2[i] > 0.0 ? sqrt(c2[i]) : 1.0;
           scale_inv[i] = 1.0 / scale[i];
           // domain-wise normalization of the initial residual to prevent underflow
           if (c2[i] > 0.0) r2[i] = 1.0; // by definition by this is now true
@@ -167,8 +171,10 @@ namespace quda
 
         if (param.do_block_schwarz()) {
           // Disable domain decomposition
-          Ar.DD(DD::reset);
-          r_sloppy.DD(DD::reset);
+          for (auto i = 0u; i < Ar.size(); i++) {
+            (Ar[i]).DD(DD::reset);
+            (r_sloppy[i]).DD(DD::reset);
+	  }
         }
 
         commGlobalReductionPop(); // renable global reductions for outer solver

From ad74642a258352f714b5edbbaef64d7ebf366239 Mon Sep 17 00:00:00 2001
From: Ferenc Pittler <pittler@hiskp.ubi-bonn.de>
Date: Sat, 26 Oct 2024 15:27:47 +0300
Subject: [PATCH 093/108] compiling with clang

---
 lib/dslash_domain_wall_4d.in.cu                     | 2 +-
 lib/dslash_improved_staggered.in.cu                 | 2 +-
 lib/dslash_ndeg_twisted_clover.in.cu                | 2 +-
 lib/dslash_ndeg_twisted_clover_preconditioned.in.cu | 2 +-
 lib/dslash_ndeg_twisted_mass.in.cu                  | 2 +-
 lib/dslash_ndeg_twisted_mass_preconditioned.in.cu   | 2 +-
 lib/dslash_staggered.in.cu                          | 2 +-
 lib/dslash_twisted_clover.in.cu                     | 2 +-
 lib/dslash_twisted_clover_preconditioned.in.cu      | 2 +-
 lib/dslash_twisted_mass.in.cu                       | 2 +-
 lib/dslash_wilson.in.cu                             | 2 +-
 lib/dslash_wilson_clover.in.cu                      | 2 +-
 12 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/lib/dslash_domain_wall_4d.in.cu b/lib/dslash_domain_wall_4d.in.cu
index 22ffecfda3..f127797e4e 100644
--- a/lib/dslash_domain_wall_4d.in.cu
+++ b/lib/dslash_domain_wall_4d.in.cu
@@ -15,7 +15,7 @@ namespace quda
   template struct DomainWall4DApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>;
    
   template DomainWall4DApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>:: 
-    DomainWall4DApply<distance_pc>(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+    DomainWall4DApply(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
                       cvector_ref<const ColorSpinorField> &x, const GaugeField &U, double a, double m_5,
                       const Complex *b_5, const Complex *c_5, int parity, bool dagger, const int *comm_override,DistanceType<distance_pc>,
                       TimeProfile &profile);
diff --git a/lib/dslash_improved_staggered.in.cu b/lib/dslash_improved_staggered.in.cu
index 5b7494f232..5fe8ed3490 100644
--- a/lib/dslash_improved_staggered.in.cu
+++ b/lib/dslash_improved_staggered.in.cu
@@ -14,7 +14,7 @@ namespace quda
 
   template struct ImprovedStaggeredApply<Float, nColor, DDArg, ReconstructStaggered::recon[reconI]>;
  
-  template ImprovedStaggeredApply<Float, nColor, DDArg, ReconstructStaggered::recon[reconI]>::ImprovedStaggeredApply<distance_pc>(
+  template ImprovedStaggeredApply<Float, nColor, DDArg, ReconstructStaggered::recon[reconI]>::ImprovedStaggeredApply(
 		  cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
                   cvector_ref<const ColorSpinorField> &x, const GaugeField &L, const GaugeField &U, double a,
                   int parity, bool dagger, const int *comm_override,DistanceType<distance_pc>, TimeProfile &profile);
diff --git a/lib/dslash_ndeg_twisted_clover.in.cu b/lib/dslash_ndeg_twisted_clover.in.cu
index 7e428d30c7..f836f3d455 100644
--- a/lib/dslash_ndeg_twisted_clover.in.cu
+++ b/lib/dslash_ndeg_twisted_clover.in.cu
@@ -13,7 +13,7 @@ namespace quda
 
     template struct NdegTwistedCloverApply<Float,nColor, DDArg, ReconstructWilson::recon[reconI]>;
 
-    template NdegTwistedCloverApply<Float,nColor, DDArg, ReconstructWilson::recon[reconI]>::NdegTwistedCloverApply<distance_pc>(
+    template NdegTwistedCloverApply<Float,nColor, DDArg, ReconstructWilson::recon[reconI]>::NdegTwistedCloverApply(
 	    cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
             cvector_ref<const ColorSpinorField> &x, const GaugeField &U, const CloverField &A, double a,
 	    double b, double c, int parity, bool dagger, const int *comm_override, DistanceType<distance_pc>, TimeProfile &profile);
diff --git a/lib/dslash_ndeg_twisted_clover_preconditioned.in.cu b/lib/dslash_ndeg_twisted_clover_preconditioned.in.cu
index 90b6173be0..e216de6138 100644
--- a/lib/dslash_ndeg_twisted_clover_preconditioned.in.cu
+++ b/lib/dslash_ndeg_twisted_clover_preconditioned.in.cu
@@ -14,7 +14,7 @@ namespace quda
     template struct NdegTwistedCloverPreconditionedApply<Float,nColor, DDArg, ReconstructWilson::recon[reconI]>;
 
 
-    template NdegTwistedCloverPreconditionedApply<Float,nColor, DDArg, ReconstructWilson::recon[reconI]>::NdegTwistedCloverPreconditionedApply<distance_pc>(
+    template NdegTwistedCloverPreconditionedApply<Float,nColor, DDArg, ReconstructWilson::recon[reconI]>::NdegTwistedCloverPreconditionedApply(
 		    cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
                     cvector_ref<const ColorSpinorField> &x, const GaugeField &U,
                     const CloverField &A, double a, double b, double c, bool xpay, int parity,
diff --git a/lib/dslash_ndeg_twisted_mass.in.cu b/lib/dslash_ndeg_twisted_mass.in.cu
index 9cb709ad32..783482a945 100644
--- a/lib/dslash_ndeg_twisted_mass.in.cu
+++ b/lib/dslash_ndeg_twisted_mass.in.cu
@@ -13,7 +13,7 @@ namespace quda
 
   template struct NdegTwistedMassApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>;
 
-  template NdegTwistedMassApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>::NdegTwistedMassApply<distance_pc>(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+  template NdegTwistedMassApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>::NdegTwistedMassApply(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
                          cvector_ref<const ColorSpinorField> &x, const GaugeField &U, double a, double b, double c,
                          int parity, bool dagger, const int *comm_override,DistanceType<distance_pc>, TimeProfile &profile);
 
diff --git a/lib/dslash_ndeg_twisted_mass_preconditioned.in.cu b/lib/dslash_ndeg_twisted_mass_preconditioned.in.cu
index 0ae75055b2..c82269d822 100644
--- a/lib/dslash_ndeg_twisted_mass_preconditioned.in.cu
+++ b/lib/dslash_ndeg_twisted_mass_preconditioned.in.cu
@@ -15,7 +15,7 @@ namespace quda
 
     template struct NdegTwistedMassPreconditionedApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>;
     
-    template NdegTwistedMassPreconditionedApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>::NdegTwistedMassPreconditionedApply<distance_pc>(
+    template NdegTwistedMassPreconditionedApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>::NdegTwistedMassPreconditionedApply(
 		    cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
                     cvector_ref<const ColorSpinorField> &x, const GaugeField &U, double a, double b,
                     double c, bool xpay, int parity, bool dagger, bool asymmetric,
diff --git a/lib/dslash_staggered.in.cu b/lib/dslash_staggered.in.cu
index c868557545..b1c29d2930 100644
--- a/lib/dslash_staggered.in.cu
+++ b/lib/dslash_staggered.in.cu
@@ -13,7 +13,7 @@ namespace quda
 
   template struct StaggeredApply<Float, nColor, DDArg, ReconstructStaggered::recon[reconI]>;
     
-  template StaggeredApply<Float, nColor, DDArg, ReconstructStaggered::recon[reconI]>::StaggeredApply<distance_pc>(
+  template StaggeredApply<Float, nColor, DDArg, ReconstructStaggered::recon[reconI]>::StaggeredApply(
 		  cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
 		  cvector_ref<const ColorSpinorField> &x, const GaugeField &U, double a, int parity, bool dagger,
                   const int *comm_override, DistanceType<distance_pc>,TimeProfile &profile);
diff --git a/lib/dslash_twisted_clover.in.cu b/lib/dslash_twisted_clover.in.cu
index 8a597c5038..ee3b34ee12 100644
--- a/lib/dslash_twisted_clover.in.cu
+++ b/lib/dslash_twisted_clover.in.cu
@@ -13,7 +13,7 @@ namespace quda
 
   template struct TwistedCloverApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>;
   
-  template TwistedCloverApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>::TwistedCloverApply<distance_pc>(
+  template TwistedCloverApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>::TwistedCloverApply(
 		  cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in, 
 		  cvector_ref<const ColorSpinorField> &x,const GaugeField &U, 
 		  const CloverField &C, double a, double b, int parity, 
diff --git a/lib/dslash_twisted_clover_preconditioned.in.cu b/lib/dslash_twisted_clover_preconditioned.in.cu
index e59bc56d26..f59075839d 100644
--- a/lib/dslash_twisted_clover_preconditioned.in.cu
+++ b/lib/dslash_twisted_clover_preconditioned.in.cu
@@ -14,7 +14,7 @@ namespace quda
 
   template struct TwistedCloverPreconditionedApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>;
 
-  template TwistedCloverPreconditionedApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>::TwistedCloverPreconditionedApply<distance_pc>(
+  template TwistedCloverPreconditionedApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>::TwistedCloverPreconditionedApply(
                   cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
                   cvector_ref<const ColorSpinorField> &x,const GaugeField &U,
                   const CloverField &C, double a, double b,bool xpay, int parity,
diff --git a/lib/dslash_twisted_mass.in.cu b/lib/dslash_twisted_mass.in.cu
index 656bcf83ed..74df605e02 100644
--- a/lib/dslash_twisted_mass.in.cu
+++ b/lib/dslash_twisted_mass.in.cu
@@ -12,7 +12,7 @@ namespace quda
 
   template struct TwistedMassApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>; 
 
-  template TwistedMassApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>::TwistedMassApply<distance_pc>(
+  template TwistedMassApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>::TwistedMassApply(
 		  cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
                   cvector_ref<const ColorSpinorField> &x, const GaugeField &U, double a, double b, int parity,
                   bool dagger, const int *comm_override, DistanceType<distance_pc>, TimeProfile &profile);
diff --git a/lib/dslash_wilson.in.cu b/lib/dslash_wilson.in.cu
index a912d4cf03..4ac0e08e09 100644
--- a/lib/dslash_wilson.in.cu
+++ b/lib/dslash_wilson.in.cu
@@ -13,7 +13,7 @@ namespace quda
 
   template struct WilsonApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>;
 
-  template WilsonApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>::WilsonApply<distance_pc>(
+  template WilsonApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>::WilsonApply(
     cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in, cvector_ref<const ColorSpinorField> &x,
     const GaugeField &U, double a, double alpha0, int t0, int parity, bool dagger, const int *comm_override,
     DistanceType<distance_pc>, TimeProfile &profile);
diff --git a/lib/dslash_wilson_clover.in.cu b/lib/dslash_wilson_clover.in.cu
index 899f926bff..4732b0e8a0 100644
--- a/lib/dslash_wilson_clover.in.cu
+++ b/lib/dslash_wilson_clover.in.cu
@@ -18,7 +18,7 @@ namespace quda
 
   template struct WilsonCloverApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>;
 
-  template WilsonCloverApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>::WilsonCloverApply<distance_pc>(
+  template WilsonCloverApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>::WilsonCloverApply(
 			  cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
 			  cvector_ref<const ColorSpinorField> &x, const GaugeField &U, const CloverField &A, double a,
 			  double alpha0, int t0, int parity, bool dagger, const int *comm_override,

From a7fe5bd57249d2fc0fc8533daac131af296cdd38 Mon Sep 17 00:00:00 2001
From: maddyscientist <mclark@nvidia.com>
Date: Tue, 5 Nov 2024 17:05:35 -0800
Subject: [PATCH 094/108] Fix non-improved staggered dslash tests - long link
 should not be created

---
 lib/gauge_field.cpp                                 | 2 ++
 tests/host_reference/staggered_dslash_reference.cpp | 3 ++-
 tests/staggered_dslash_test_utils.h                 | 2 +-
 3 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/lib/gauge_field.cpp b/lib/gauge_field.cpp
index 20cf7621ac..ed414239cd 100644
--- a/lib/gauge_field.cpp
+++ b/lib/gauge_field.cpp
@@ -81,6 +81,8 @@ namespace quda {
       errorQuda("Cannot request a 12/8 reconstruct type without SU(3) link type");
     if (param.reconstruct == QUDA_RECONSTRUCT_10 && param.link_type != QUDA_ASQTAD_MOM_LINKS)
       errorQuda("10-reconstruction only supported with momentum links");
+    if (param.nFace > x[0] || param.nFace > x[1] || param.nFace > x[2] || param.nFace > x[3])
+      errorQuda("Halo depth %d is greater than local lattice x = {%d %d %d %d}", param.nFace, x[0], x[1], x[2], x[3]);
 
     nColor = param.nColor;
     nFace = param.nFace;
diff --git a/tests/host_reference/staggered_dslash_reference.cpp b/tests/host_reference/staggered_dslash_reference.cpp
index 212549dc38..c5846d1ca7 100644
--- a/tests/host_reference/staggered_dslash_reference.cpp
+++ b/tests/host_reference/staggered_dslash_reference.cpp
@@ -107,7 +107,7 @@ void staggeredDslashReference(real_t *res, const real_t *const *fatlink, const r
   } // 4-d volume
 }
 
-void stag_dslash(ColorSpinorField &out, const GaugeField &fat_link, const GaugeField &long_link,
+void stag_dslash(ColorSpinorField &out, const GaugeField &fat_link, const GaugeField &long_link_,
                  const ColorSpinorField &in, int oddBit, int daggerBit, QudaDslashType dslash_type)
 {
   // assert sPrecision and gPrecision must be the same
@@ -131,6 +131,7 @@ void stag_dslash(ColorSpinorField &out, const GaugeField &fat_link, const GaugeF
 
   in.exchangeGhost(otherparity, nFace, daggerBit);
 
+  auto &long_link = dslash_type == QUDA_ASQTAD_DSLASH ? long_link_ : fat_link;
   void *qdp_fatlink[] = {fat_link.data(0), fat_link.data(1), fat_link.data(2), fat_link.data(3)};
   void *qdp_longlink[] = {long_link.data(0), long_link.data(1), long_link.data(2), long_link.data(3)};
   void *ghost_fatlink[]
diff --git a/tests/staggered_dslash_test_utils.h b/tests/staggered_dslash_test_utils.h
index d7e98e01d4..5280184613 100644
--- a/tests/staggered_dslash_test_utils.h
+++ b/tests/staggered_dslash_test_utils.h
@@ -275,7 +275,7 @@ struct StaggeredDslashTestWrapper {
     GaugeFieldParam cpuLongParam(gauge_param, qdp_longlink);
     cpuLongParam.order = QUDA_QDP_GAUGE_ORDER;
     cpuLongParam.ghostExchange = QUDA_GHOST_EXCHANGE_PAD;
-    cpuLong = GaugeField(cpuLongParam);
+    if (dslash_type == QUDA_ASQTAD_DSLASH) cpuLong = GaugeField(cpuLongParam);
 
     // Override link reconstruct as appropriate for staggered or asqtad
     if (is_staggered(dslash_type)) {

From 23f889d258e18614c66e74892499212009ef9f29 Mon Sep 17 00:00:00 2001
From: Ferenc Pittler <pittler@hiskp.uni-bonn.de>
Date: Wed, 6 Nov 2024 08:19:48 +0100
Subject: [PATCH 095/108] separate compilation units for domain wall 5f

---
 lib/CMakeLists.txt              |  3 ++
 lib/dslash_domain_wall_5d.cu    | 80 +++--------------------------
 lib/dslash_domain_wall_5d.hpp   | 89 +++++++++++++++++++++++++++++++++
 lib/dslash_domain_wall_5d.in.cu | 20 ++++++++
 4 files changed, 118 insertions(+), 74 deletions(-)
 create mode 100644 lib/dslash_domain_wall_5d.hpp
 create mode 100644 lib/dslash_domain_wall_5d.in.cu

diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt
index e8495a741d..6523600fb2 100644
--- a/lib/CMakeLists.txt
+++ b/lib/CMakeLists.txt
@@ -256,6 +256,9 @@ foreach(QUDA_DSLASH_PREC ${QUDA_DSLASH_PREC_LIST})
     foreach(QUDA_DSLASH_DDARG ${QUDA_DSLASH_DDARG_LIST})
       foreach(QUDA_DSLASH_RECONI ${QUDA_DSLASH_RECONI_LIST})
 	foreach(QUDA_DSLASH_DISTANCE ${QUDA_DSLASH_DISTANCE_LIST})
+          set(TMP_FILENAME "dslash_domain_wall_5d_${QUDA_DSLASH_PREC_LOWER}_nc_${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
+          configure_file(dslash_domain_wall_5d.in.cu ${TMP_FILENAME} @ONLY)
+          list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
           set(TMP_FILENAME "dslash_domain_wall_4d_${QUDA_DSLASH_PREC_LOWER}_nc_${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
           configure_file(dslash_domain_wall_4d.in.cu ${TMP_FILENAME} @ONLY)
           list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
diff --git a/lib/dslash_domain_wall_5d.cu b/lib/dslash_domain_wall_5d.cu
index 9f2946c0bc..3bb5f8f3a8 100644
--- a/lib/dslash_domain_wall_5d.cu
+++ b/lib/dslash_domain_wall_5d.cu
@@ -1,10 +1,6 @@
-#include <gauge_field.h>
-#include <color_spinor_field.h>
-#include <dslash.h>
-#include <worker.h>
-
-#include <dslash_policy.hpp>
-#include <kernels/dslash_domain_wall_5d.cuh>
+#define SIGNATURE_ONLY
+#include <dslash_domain_wall_5d.hpp>
+#undef SIGNATURE_ONLY
 
 /**
    This is the gauged domain-wall 5-d preconditioned operator.
@@ -13,80 +9,16 @@
 namespace quda
 {
 
-  template <typename Arg> class DomainWall5D : public Dslash<domainWall5D, Arg>
-  {
-    using Dslash = Dslash<domainWall5D, Arg>;
-    using Dslash::arg;
-    using Dslash::in;
-
-  public:
-    DomainWall5D(Arg &arg, cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
-                 const ColorSpinorField &halo) :
-      Dslash(arg, out, in, halo)
-    {
-    }
-
-    void apply(const qudaStream_t &stream)
-    {
-      TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
-      Dslash::setParam(tp);
-      Dslash::template instantiate<packShmem>(tp, stream);
-    }
-
-    long long flops() const
-    {
-      long long flops = Dslash::flops();
-      switch (arg.kernel_type) {
-      case INTERIOR_KERNEL:
-      case UBER_KERNEL:
-      case KERNEL_POLICY: {
-        int Ls = in.X(4);
-        long long bulk = (Ls - 2) * (in.Volume() / Ls);
-        long long wall = 2 * (in.Volume() / Ls);
-        flops += in.size() * 96ll * bulk + 120ll * wall;
-      } break;
-      default: break; // 5-d flops are in the interior kernel
-      }
-      return flops;
-    }
-
-    long long bytes() const
-    {
-      int spinor_bytes = 2 * in.Ncolor() * in.Nspin() * in.Precision() + (isFixed<typename Arg::Float>::value ? sizeof(float) : 0);
-      long long bytes = Dslash::bytes();
-      switch (arg.kernel_type) {
-      case INTERIOR_KERNEL:
-      case UBER_KERNEL:
-      case KERNEL_POLICY: bytes += in.size() * 2 * spinor_bytes * in.VolumeCB(); break;
-      default: break;
-      }
-      return bytes;
-    }
-  };
-
-  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon> struct DomainWall5DApply {
-
-    DomainWall5DApply(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
-                      cvector_ref<const ColorSpinorField> &x, const GaugeField &U, double a, double m_f, int parity,
-                      bool dagger, const int *comm_override, TimeProfile &profile)
-    {
-      constexpr int nDim = 5;
-      auto halo = ColorSpinorField::create_comms_batch(in);
-      DomainWall5DArg<Float, nColor, nDim, DDArg, recon> arg(out, in, halo, U, a, m_f, a != 0.0, x, parity, dagger,
-                                                             comm_override);
-      DomainWall5D<decltype(arg)> dwf(arg, out, in, halo);
-      dslash::DslashPolicyTune<decltype(dwf)> policy(dwf, in, halo, profile);
-    }
-  };
 
-  // Apply the 4-d preconditioned domain-wall Dslash operator
+  // Apply the 5-d preconditioned domain-wall Dslash operator
   // out(x) = M*in = in(x) + a*\sum_mu U_{-\mu}(x)in(x+mu) + U^\dagger_mu(x-mu)in(x-mu)
   void ApplyDomainWall5D(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
                          const GaugeField &U, double a, double m_f, cvector_ref<const ColorSpinorField> &x, int parity,
                          bool dagger, const int *comm_override, TimeProfile &profile)
   {
     if constexpr (is_enabled<QUDA_DOMAIN_WALL_DSLASH>()) {
-      instantiate<DomainWall5DApply>(out, in, x, U, a, m_f, parity, dagger, comm_override, profile);
+      auto dummy = DistanceType<false>();
+      instantiate<DomainWall5DApply>(out, in, x, U, a, m_f, parity, dagger, comm_override, dummy, profile);
     } else {
       errorQuda("Domain-wall operator has not been built");
     }
diff --git a/lib/dslash_domain_wall_5d.hpp b/lib/dslash_domain_wall_5d.hpp
new file mode 100644
index 0000000000..bb030223a5
--- /dev/null
+++ b/lib/dslash_domain_wall_5d.hpp
@@ -0,0 +1,89 @@
+#include <gauge_field.h>
+#include <color_spinor_field.h>
+#include <dslash.h>
+#include <worker.h>
+
+#include <dslash_policy.hpp>
+#include <kernels/dslash_domain_wall_5d.cuh>
+
+/**
+   This is the gauged domain-wall 5-d preconditioned operator.
+*/
+
+namespace quda
+{
+
+  template <typename Arg> class DomainWall5D : public Dslash<domainWall5D, Arg>
+  {
+    using Dslash = Dslash<domainWall5D, Arg>;
+    using Dslash::arg;
+    using Dslash::in;
+
+  public:
+    DomainWall5D(Arg &arg, cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+                 const ColorSpinorField &halo) :
+      Dslash(arg, out, in, halo)
+    {
+    }
+
+    void apply(const qudaStream_t &stream)
+    {
+      TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
+      Dslash::setParam(tp);
+      Dslash::template instantiate<packShmem>(tp, stream);
+    }
+
+    long long flops() const
+    {
+      long long flops = Dslash::flops();
+      switch (arg.kernel_type) {
+      case INTERIOR_KERNEL:
+      case UBER_KERNEL:
+      case KERNEL_POLICY: {
+        int Ls = in.X(4);
+        long long bulk = (Ls - 2) * (in.Volume() / Ls);
+        long long wall = 2 * (in.Volume() / Ls);
+        flops += in.size() * 96ll * bulk + 120ll * wall;
+      } break;
+      default: break; // 5-d flops are in the interior kernel
+      }
+      return flops;
+    }
+
+    long long bytes() const
+    {
+      int spinor_bytes = 2 * in.Ncolor() * in.Nspin() * in.Precision() + (isFixed<typename Arg::Float>::value ? sizeof(float) : 0);
+      long long bytes = Dslash::bytes();
+      switch (arg.kernel_type) {
+      case INTERIOR_KERNEL:
+      case UBER_KERNEL:
+      case KERNEL_POLICY: bytes += in.size() * 2 * spinor_bytes * in.VolumeCB(); break;
+      default: break;
+      }
+      return bytes;
+    }
+  };
+
+  template <bool distance_pc> struct DistanceType {
+  };
+
+  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon> struct DomainWall5DApply {
+    template <bool distance_pc>
+    DomainWall5DApply(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+                      cvector_ref<const ColorSpinorField> &x, const GaugeField &U, double a, double m_f, int parity,
+                      bool dagger, const int *comm_override,DistanceType<distance_pc>, TimeProfile &profile)
+    #ifdef SIGNATURE_ONLY
+          ;
+#else
+    {
+      constexpr int nDim = 5;
+      auto halo = ColorSpinorField::create_comms_batch(in);
+      DomainWall5DArg<Float, nColor, nDim, DDArg, recon> arg(out, in, halo, U, a, m_f, a != 0.0, x, parity, dagger,
+                                                             comm_override);
+      DomainWall5D<decltype(arg)> dwf(arg, out, in, halo);
+      dslash::DslashPolicyTune<decltype(dwf)> policy(dwf, in, halo, profile);
+    }
+#endif
+  };
+
+} // namespace quda
diff --git a/lib/dslash_domain_wall_5d.in.cu b/lib/dslash_domain_wall_5d.in.cu
new file mode 100644
index 0000000000..d0aa8b4710
--- /dev/null
+++ b/lib/dslash_domain_wall_5d.in.cu
@@ -0,0 +1,20 @@
+#include <dslash_domain_wall_5d.hpp>
+
+namespace quda
+{
+
+  constexpr QudaPrecision precision = QUDA_@QUDA_DSLASH_PREC@_PRECISION;
+  constexpr int nColor = @QUDA_DSLASH_NCOLOR@;
+  constexpr int reconI = @QUDA_DSLASH_RECONI@;
+
+  typedef @QUDA_DSLASH_DDARG@ DDArg;
+  typedef precision_type_mapper<precision>::type Float;
+  constexpr bool distance_pc = @QUDA_DSLASH_DISTANCE@;
+
+  template struct DomainWall5DApply < Float, nColor, DDArg, ReconstructWilson::recon[reconI]>; 
+
+  template DomainWall5DApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>::DomainWall5DApply
+	  (cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+                      cvector_ref<const ColorSpinorField> &x, const GaugeField &U, double a, double m_f, int parity,
+                      bool dagger, const int *comm_override, DistanceType<distance_pc>, TimeProfile &profile);
+} // namespace quda

From 4041e67d99b0af18f1f4328b9c314c5cba8f0b29 Mon Sep 17 00:00:00 2001
From: Ferenc Pittler <pittler@hiskp.uni-bonn.de>
Date: Wed, 6 Nov 2024 08:53:51 +0100
Subject: [PATCH 096/108] separate compilation for wilson clover preconditioned

---
 lib/CMakeLists.txt                            |  3 ++
 lib/dslash_wilson_clover_preconditioned.cu    |  2 ++
 lib/dslash_wilson_clover_preconditioned.hpp   |  4 +++
 lib/dslash_wilson_clover_preconditioned.in.cu | 28 +++++++++++++++++++
 4 files changed, 37 insertions(+)
 create mode 100644 lib/dslash_wilson_clover_preconditioned.in.cu

diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt
index 6523600fb2..54e068e130 100644
--- a/lib/CMakeLists.txt
+++ b/lib/CMakeLists.txt
@@ -277,6 +277,9 @@ foreach(QUDA_DSLASH_PREC ${QUDA_DSLASH_PREC_LIST})
 	  set(TMP_FILENAME "dslash_wilson_clover_${QUDA_DSLASH_PREC_LOWER}_nc_${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
           configure_file(dslash_wilson_clover.in.cu ${TMP_FILENAME} @ONLY)
           list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
+          set(TMP_FILENAME "dslash_wilson_clover_preconditioned_${QUDA_DSLASH_PREC_LOWER}_nc_${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
+          configure_file(dslash_wilson_clover_preconditioned.in.cu ${TMP_FILENAME} @ONLY)
+          list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
           set(TMP_FILENAME "dslash_twisted_mass_${QUDA_DSLASH_PREC_LOWER}_nc_${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
           configure_file(dslash_twisted_mass.in.cu ${TMP_FILENAME} @ONLY)
           list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
diff --git a/lib/dslash_wilson_clover_preconditioned.cu b/lib/dslash_wilson_clover_preconditioned.cu
index 6102d9c2e2..237e4b07f0 100644
--- a/lib/dslash_wilson_clover_preconditioned.cu
+++ b/lib/dslash_wilson_clover_preconditioned.cu
@@ -1,4 +1,6 @@
+#define SIGNATURE_ONLY
 #include <dslash_wilson_clover_preconditioned.hpp>
+#undef SIGNATURE_ONLY
 
 /**
    This is the Wilson-clover preconditioned linear operator
diff --git a/lib/dslash_wilson_clover_preconditioned.hpp b/lib/dslash_wilson_clover_preconditioned.hpp
index 88f3952053..0ca37f886c 100644
--- a/lib/dslash_wilson_clover_preconditioned.hpp
+++ b/lib/dslash_wilson_clover_preconditioned.hpp
@@ -124,6 +124,9 @@ namespace quda
                                     cvector_ref<const ColorSpinorField> &x, const GaugeField &U, const CloverField &A,
                                     double a, double alpha0, int t0, int parity, bool dagger, const int *comm_override,
                                     DistanceType<distance_pc>, TimeProfile &profile)
+#ifdef SIGNATURE_ONLY
+    ;
+#else
     {
       constexpr int nDim = 4;
       auto halo = ColorSpinorField::create_comms_batch(in);
@@ -133,6 +136,7 @@ namespace quda
 
       dslash::DslashPolicyTune<decltype(wilson)> policy(wilson, in, halo, profile);
     }
+#endif
   };
 
 } // namespace quda
diff --git a/lib/dslash_wilson_clover_preconditioned.in.cu b/lib/dslash_wilson_clover_preconditioned.in.cu
new file mode 100644
index 0000000000..549522f8d4
--- /dev/null
+++ b/lib/dslash_wilson_clover_preconditioned.in.cu
@@ -0,0 +1,28 @@
+#include <dslash_wilson_clover_preconditioned.hpp>
+
+/**
+   This is the Wilson-clover linear operator
+*/
+
+namespace quda
+{
+
+
+  constexpr QudaPrecision precision = QUDA_@QUDA_DSLASH_PREC@_PRECISION;
+  constexpr int nColor = @QUDA_DSLASH_NCOLOR@;
+  constexpr int reconI = @QUDA_DSLASH_RECONI@;
+  constexpr bool distance_pc = @QUDA_DSLASH_DISTANCE@;
+
+  typedef @QUDA_DSLASH_DDARG@ DDArg;
+  typedef precision_type_mapper<precision>::type Float;
+
+  template struct WilsonCloverPreconditionedApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>;
+
+  template WilsonCloverPreconditionedApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>::WilsonCloverPreconditionedApply(
+                          cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+                          cvector_ref<const ColorSpinorField> &x, const GaugeField &U, const CloverField &A, double a,
+                          double alpha0, int t0, int parity, bool dagger, const int *comm_override,
+                          DistanceType<distance_pc>, TimeProfile &profile);
+
+} // namespace quda
+

From 5c0b686edb1650661102b78dae6d3edbd73ecf18 Mon Sep 17 00:00:00 2001
From: maddyscientist <mclark@nvidia.com>
Date: Wed, 6 Nov 2024 12:59:08 -0800
Subject: [PATCH 097/108] Fix redundant compilation with Wilson-clover using
 distain preconditioning

---
 lib/dslash_wilson_clover.in.cu                      | 1 -
 lib/dslash_wilson_clover_distance.cu                | 2 ++
 lib/dslash_wilson_clover_preconditioned_distance.cu | 2 ++
 3 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/lib/dslash_wilson_clover.in.cu b/lib/dslash_wilson_clover.in.cu
index 4732b0e8a0..40cb2ac36b 100644
--- a/lib/dslash_wilson_clover.in.cu
+++ b/lib/dslash_wilson_clover.in.cu
@@ -7,7 +7,6 @@
 namespace quda
 {
 
-
   constexpr QudaPrecision precision = QUDA_@QUDA_DSLASH_PREC@_PRECISION;
   constexpr int nColor = @QUDA_DSLASH_NCOLOR@;
   constexpr int reconI = @QUDA_DSLASH_RECONI@;
diff --git a/lib/dslash_wilson_clover_distance.cu b/lib/dslash_wilson_clover_distance.cu
index 8ff71313da..f578143906 100644
--- a/lib/dslash_wilson_clover_distance.cu
+++ b/lib/dslash_wilson_clover_distance.cu
@@ -1,4 +1,6 @@
+#define SIGNATURE_ONLY
 #include <dslash_wilson_clover.hpp>
+#undef SIGNATURE_ONLY
 
 /**
    This is the Wilson-clover linear operator
diff --git a/lib/dslash_wilson_clover_preconditioned_distance.cu b/lib/dslash_wilson_clover_preconditioned_distance.cu
index eac8ed1ff9..33c65dd9a7 100644
--- a/lib/dslash_wilson_clover_preconditioned_distance.cu
+++ b/lib/dslash_wilson_clover_preconditioned_distance.cu
@@ -1,4 +1,6 @@
+#define SIGNATURE_ONLY
 #include <dslash_wilson_clover_preconditioned.hpp>
+#undef SIGNATURE_ONLY
 
 /**
    This is the Wilson-clover preconditioned linear operator

From be72ab31159bc85ec5cc11de4c2575ec90010c43 Mon Sep 17 00:00:00 2001
From: maddyscientist <mclark@nvidia.com>
Date: Wed, 6 Nov 2024 15:50:20 -0800
Subject: [PATCH 098/108] Fix issues with non-asqtad fermions

---
 tests/staggered_eigensolve_test.cpp | 11 ++++++-----
 tests/staggered_invert_test.cpp     | 11 ++++++-----
 2 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/tests/staggered_eigensolve_test.cpp b/tests/staggered_eigensolve_test.cpp
index 93ae358896..5041ff49b4 100644
--- a/tests/staggered_eigensolve_test.cpp
+++ b/tests/staggered_eigensolve_test.cpp
@@ -114,7 +114,7 @@ void init()
   cpuFatMILC = GaugeField(cpuParam);
 
   cpuParam.link_type = QUDA_ASQTAD_LONG_LINKS;
-  cpuParam.nFace = 3;
+  cpuParam.nFace = dslash_type == QUDA_ASQTAD_DSLASH ? 3 : 1;
   cpuParam.order = QUDA_QDP_GAUGE_ORDER;
   cpuLongQDP = GaugeField(cpuParam);
   cpuParam.order = QUDA_MILC_GAUGE_ORDER;
@@ -127,7 +127,7 @@ void init()
 
   // Reorder gauge fields to MILC order
   cpuFatMILC = cpuFatQDP;
-  cpuLongMILC = cpuLongQDP;
+  if (dslash_type == QUDA_ASQTAD_DSLASH) cpuLongMILC = cpuLongQDP;
 
   // Compute plaquette. Routine is aware that the gauge fields already have the phases on them.
   // This needs to be called before `loadFatLongGaugeQuda` because this routine also loads the
@@ -148,10 +148,11 @@ void init()
 
   // now copy back to QDP aliases, since these are used for the reference dslash
   cpuFatQDP = cpuFatMILC;
-  cpuLongQDP = cpuLongMILC;
-  // ensure QDP alias has exchanged ghosts
   cpuFatQDP.exchangeGhost();
-  cpuLongQDP.exchangeGhost();
+  if (dslash_type == QUDA_ASQTAD_DSLASH) {
+    cpuLongQDP = cpuLongMILC;
+    cpuLongQDP.exchangeGhost();
+  }
 
   // Staggered Gauge construct END
   //-----------------------------------------------------------------------------------
diff --git a/tests/staggered_invert_test.cpp b/tests/staggered_invert_test.cpp
index a098324ef6..1140aceab3 100644
--- a/tests/staggered_invert_test.cpp
+++ b/tests/staggered_invert_test.cpp
@@ -203,7 +203,7 @@ void init()
   cpuFatMILC = GaugeField(cpuParam);
 
   cpuParam.link_type = QUDA_ASQTAD_LONG_LINKS;
-  cpuParam.nFace = 3;
+  cpuParam.nFace = dslash_type == QUDA_ASQTAD_DSLASH ? 3 : 1;
   cpuParam.order = QUDA_QDP_GAUGE_ORDER;
   cpuLongQDP = GaugeField(cpuParam);
   cpuParam.order = QUDA_MILC_GAUGE_ORDER;
@@ -216,7 +216,7 @@ void init()
 
   // Reorder gauge fields to MILC order
   cpuFatMILC = cpuFatQDP;
-  cpuLongMILC = cpuLongQDP;
+  if (dslash_type == QUDA_ASQTAD_DSLASH) cpuLongMILC = cpuLongQDP;
 
   // Compute plaquette. Routine is aware that the gauge fields already have the phases on them.
   // This needs to be called before `loadFatLongGaugeQuda` because this routine also loads the
@@ -237,10 +237,11 @@ void init()
 
   // now copy back to QDP aliases, since these are used for the reference dslash
   cpuFatQDP = cpuFatMILC;
-  cpuLongQDP = cpuLongMILC;
-  // ensure QDP alias has exchanged ghosts
   cpuFatQDP.exchangeGhost();
-  cpuLongQDP.exchangeGhost();
+  if (dslash_type == QUDA_ASQTAD_DSLASH) {
+    cpuLongQDP = cpuLongMILC;
+    cpuLongQDP.exchangeGhost();
+  }
 
   // Staggered Gauge construct END
   //-----------------------------------------------------------------------------------

From cb84d79386d8ad9c8caa910f5dad24c2a16bbc31 Mon Sep 17 00:00:00 2001
From: Ferenc Pittler <pittler@hiskp.uni-bonn.de>
Date: Thu, 7 Nov 2024 07:59:46 +0100
Subject: [PATCH 099/108] Renaming dir to dist

---
 include/domain_decomposition_helper.cuh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/include/domain_decomposition_helper.cuh b/include/domain_decomposition_helper.cuh
index fcd036f502..2b7ab5acea 100644
--- a/include/domain_decomposition_helper.cuh
+++ b/include/domain_decomposition_helper.cuh
@@ -73,10 +73,10 @@ namespace quda
       return block_parity % 2 == 1;
     }
 
-    template <typename Coord> constexpr bool on_border(const Coord &x, int mu, int dir) const
+    template <typename Coord> constexpr bool on_border(const Coord &x, int mu, int dist) const
     {
       if (block_dim[mu] == 0) return false;
-      int x_mu = x.gx[mu] + dir;
+      int x_mu = x.gx[mu] + dist;
       if (x_mu < 0) x_mu += x.gDim[mu];
       if (x_mu >= x.gDim[mu]) x_mu -= x.gDim[mu];
       return x.gx[mu] / block_dim[mu] != x_mu / block_dim[mu];
@@ -92,11 +92,11 @@ namespace quda
       return true;
     }
 
-    template <typename Coord> constexpr bool doHopping(const Coord &x, int mu, int dir) const
+    template <typename Coord> constexpr bool doHopping(const Coord &x, int mu, int dist) const
     {
       bool is_black = block_parity(x);
       bool is_red = !is_black;
-      bool is_border = on_border(x, mu, dir);
+      bool is_border = on_border(x, mu, dist);
 
       if (!is_border) { // Within block
         if (is_red and red_active) return true;

From a1fb649159b44baf74572eae504fadac1e5094f8 Mon Sep 17 00:00:00 2001
From: Ferenc Pittler <pittler@hiskp.uni-bonn.de>
Date: Thu, 7 Nov 2024 08:35:26 +0100
Subject: [PATCH 100/108] separate compilation units for different domain wall

---
 lib/CMakeLists.txt                       | 84 ++++++++++++------------
 lib/dslash_domain_wall_4d_fused_m5.hpp   | 11 +++-
 lib/dslash_domain_wall_4d_fused_m5.in.cu | 28 ++++++++
 lib/dslash_domain_wall_4d_m5inv.cu       |  5 +-
 lib/dslash_domain_wall_4d_m5inv.hpp      | 27 ++++++++
 lib/dslash_domain_wall_4d_m5inv.in.cu    | 27 ++++++++
 lib/dslash_domain_wall_4d_m5inv_m5inv.cu |  5 +-
 lib/dslash_domain_wall_4d_m5inv_m5pre.cu |  5 +-
 lib/dslash_domain_wall_4d_m5mob.cu       |  5 +-
 lib/dslash_domain_wall_4d_m5pre.cu       |  5 +-
 lib/dslash_domain_wall_4d_m5pre_m5inv.cu |  5 +-
 lib/dslash_domain_wall_4d_m5pre_m5mob.cu |  5 +-
 12 files changed, 161 insertions(+), 51 deletions(-)
 create mode 100644 lib/dslash_domain_wall_4d_fused_m5.in.cu
 create mode 100644 lib/dslash_domain_wall_4d_m5inv.hpp
 create mode 100644 lib/dslash_domain_wall_4d_m5inv.in.cu

diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt
index 54e068e130..ee52bc1e56 100644
--- a/lib/CMakeLists.txt
+++ b/lib/CMakeLists.txt
@@ -194,16 +194,43 @@ function(check_bit number bit)
   set(BIT_CHECK "${result}" PARENT_SCOPE)
 endfunction()
 
+set(QUDA_DSLASH_FILENAME_LIST "") 
+list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_wilson")
+list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_wilson_clover")
+list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_wilson_clover_preconditioned")
+list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_twisted_mass")
+list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_twisted_clover")
+list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_twisted_clover_preconditioned")
+list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_ndeg_twisted_mass")
+list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_ndeg_twisted_mass_preconditioned")
+list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_ndeg_twisted_clover")
+list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_ndeg_twisted_clover_preconditioned")
+list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_staggered")
+list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_improved_staggered")
+list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_domain_wall_4d")
+list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_domain_wall_5d")
+
 set(QUDA_DSLASH_PREC_LIST "")
 set(QUDA_DSLASH_NCOLOR_LIST "3")
 set(QUDA_DSLASH_DDARG_LIST "DDNo")
 set(QUDA_DSLASH_RECONI_LIST "")
 set(QUDA_DSLASH_DISTANCE_LIST "false")
+set(QUDA_DSLASH_DWTYPE_LIST "") 
 
 if(QUDA_DIRAC_DISTANCE_PRECONDITIONING)
   list(APPEND QUDA_DSLASH_DISTANCE_LIST "true")
 endif()
 
+if(QUDA_DIRAC_DOMAIN_WALL)
+  list(APPEND QUDA_DSLASH_DWTYPE_LIST "DSLASH5_MOBIUS_PRE")
+  list(APPEND QUDA_DSLASH_DWTYPE_LIST "DSLASH5_MOBIUS")
+  list(APPEND QUDA_DSLASH_DWTYPE_LIST "M5_INV_MOBIUS")
+  list(APPEND QUDA_DSLASH_DWTYPE_LIST "M5_INV_MOBIUS_M5_PRE")
+  list(APPEND QUDA_DSLASH_DWTYPE_LIST "M5_PRE_MOBIUS_M5_INV")
+  list(APPEND QUDA_DSLASH_DWTYPE_LIST "M5_INV_MOBIUS_M5_INV_DAG")
+  list(APPEND QUDA_DSLASH_DWTYPE_LIST "DSLASH5_MOBIUS_PRE_M5_MOB")
+endif()
+
 check_bit(${QUDA_PRECISION} 0)
 if(BIT_CHECK)
   list(APPEND QUDA_DSLASH_PREC_LIST "QUARTER")
@@ -244,11 +271,13 @@ if(BIT_CHECK)
   list(APPEND QUDA_DSLASH_DDARG_LIST "DDRedBlack")
 endif()
 
+message(STATUS "QUDA_DSLASH_FILENAME_LIST=${QUDA_DSLASH_FILENAME_LIST}")
 message(STATUS "QUDA_DSLASH_PREC_LIST=${QUDA_DSLASH_PREC_LIST}")
 message(STATUS "QUDA_DSLASH_NCOLOR_LIST=${QUDA_DSLASH_NCOLOR_LIST}")
 message(STATUS "QUDA_DSLASH_DDARG_LIST=${QUDA_DSLASH_DDARG_LIST}")
 message(STATUS "QUDA_DSLASH_RECONI_LIST=${QUDA_DSLASH_RECONI_LIST}")
 message(STATUS "QUDA_DSLASH_DISTANCE_LIST=${QUDA_DSLASH_DISTANCE_LIST}")
+message(STATUS "QUDA_DSLASH_DWTYPE_LIST=${QUDA_DSLASH_DWTYPE_LIST}")
 
 foreach(QUDA_DSLASH_PREC ${QUDA_DSLASH_PREC_LIST})
   string(TOLOWER "${QUDA_DSLASH_PREC}" QUDA_DSLASH_PREC_LOWER)
@@ -256,48 +285,18 @@ foreach(QUDA_DSLASH_PREC ${QUDA_DSLASH_PREC_LIST})
     foreach(QUDA_DSLASH_DDARG ${QUDA_DSLASH_DDARG_LIST})
       foreach(QUDA_DSLASH_RECONI ${QUDA_DSLASH_RECONI_LIST})
 	foreach(QUDA_DSLASH_DISTANCE ${QUDA_DSLASH_DISTANCE_LIST})
-          set(TMP_FILENAME "dslash_domain_wall_5d_${QUDA_DSLASH_PREC_LOWER}_nc_${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
-          configure_file(dslash_domain_wall_5d.in.cu ${TMP_FILENAME} @ONLY)
-          list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
-          set(TMP_FILENAME "dslash_domain_wall_4d_${QUDA_DSLASH_PREC_LOWER}_nc_${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
-          configure_file(dslash_domain_wall_4d.in.cu ${TMP_FILENAME} @ONLY)
-          list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
-	  set(TMP_FILENAME "dslash_staggered_${QUDA_DSLASH_PREC_LOWER}_nc_${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
-	  configure_file(dslash_staggered.in.cu ${TMP_FILENAME} @ONLY)
-          list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
-          set(TMP_FILENAME "dslash_improved_staggered_${QUDA_DSLASH_PREC_LOWER}_nc_${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
-          configure_file(dslash_improved_staggered.in.cu ${TMP_FILENAME} @ONLY)
-          list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
-	  set(TMP_FILENAME "dslash_wilson_${QUDA_DSLASH_PREC_LOWER}_nc_${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
-          configure_file(dslash_wilson.in.cu ${TMP_FILENAME} @ONLY)
-          list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
-	  set(TMP_FILENAME "dslash_twisted_clover_${QUDA_DSLASH_PREC_LOWER}_nc_${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
-          configure_file(dslash_twisted_clover.in.cu ${TMP_FILENAME} @ONLY)
-          list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
-	  set(TMP_FILENAME "dslash_wilson_clover_${QUDA_DSLASH_PREC_LOWER}_nc_${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
-          configure_file(dslash_wilson_clover.in.cu ${TMP_FILENAME} @ONLY)
-          list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
-          set(TMP_FILENAME "dslash_wilson_clover_preconditioned_${QUDA_DSLASH_PREC_LOWER}_nc_${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
-          configure_file(dslash_wilson_clover_preconditioned.in.cu ${TMP_FILENAME} @ONLY)
-          list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
-          set(TMP_FILENAME "dslash_twisted_mass_${QUDA_DSLASH_PREC_LOWER}_nc_${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
-          configure_file(dslash_twisted_mass.in.cu ${TMP_FILENAME} @ONLY)
-          list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
-          set(TMP_FILENAME "dslash_ndeg_twisted_clover_${QUDA_DSLASH_PREC_LOWER}_nc_${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
-          configure_file(dslash_ndeg_twisted_clover.in.cu ${TMP_FILENAME} @ONLY)
-          list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
-          set(TMP_FILENAME "dslash_ndeg_twisted_clover_preconditioned.${QUDA_DSLASH_PREC_LOWER}_nc_${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
-          configure_file(dslash_ndeg_twisted_clover_preconditioned.in.cu ${TMP_FILENAME} @ONLY)
-          list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
-	  set(TMP_FILENAME "dslash_ndeg_twisted_mass_preconditioned.${QUDA_DSLASH_PREC_LOWER}_nc_${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
-          configure_file(dslash_ndeg_twisted_mass_preconditioned.in.cu ${TMP_FILENAME} @ONLY)
-          list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
-          set(TMP_FILENAME "dslash_ndeg_twisted_mass_${QUDA_DSLASH_PREC_LOWER}_nc_${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
-          configure_file(dslash_ndeg_twisted_mass.in.cu ${TMP_FILENAME} @ONLY)
-          list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
-          set(TMP_FILENAME "dslash_twisted_clover_preconditioned.${QUDA_DSLASH_PREC_LOWER}_nc_${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}.cu")
-          configure_file(dslash_twisted_clover_preconditioned.in.cu ${TMP_FILENAME} @ONLY)
-          list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
+	  set(DSLASH_SPECS "${QUDA_DSLASH_PREC_LOWER}_nc_${QUDA_DSLASH_NCOLOR}_${QUDA_DSLASH_DDARG}_recon${QUDA_DSLASH_RECONI}_dist-${QUDA_DSLASH_DISTANCE}")
+          foreach(QUDA_DSLASH_FILENAME ${QUDA_DSLASH_FILENAME_LIST})
+	    set(TMP_FILENAME "${QUDA_DSLASH_FILENAME}_${DSLASH_SPECS}.cu")
+	    configure_file(${QUDA_DSLASH_FILENAME}.in.cu ${TMP_FILENAME} @ONLY)
+	    list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
+	  endforeach()
+	  foreach(QUDA_DSLASH_DWTYPE ${QUDA_DSLASH_DWTYPE_LIST})
+	    string(TOLOWER "${QUDA_DSLASH_DWTYPE}" QUDA_DSLASH_DWTYPE_LOWER)
+	    set(TMP_FILENAME "dslash_domain_wall_4d_${QUDA_DSLASH_DWTYPE_LOWER}_${DSLASH_SPECS}.cu")
+	    configure_file(dslash_domain_wall_4d_fused_m5.in.cu ${TMP_FILENAME} @ONLY)
+            list(PREPEND QUDA_CU_OBJS ${TMP_FILENAME})
+          endforeach()
 	endforeach()
       endforeach()
     endforeach()
@@ -605,6 +604,7 @@ endif()
 # set which precisions to enable
 target_compile_definitions(quda PUBLIC QUDA_PRECISION=${QUDA_PRECISION})
 target_compile_definitions(quda PUBLIC QUDA_RECONSTRUCT=${QUDA_RECONSTRUCT})
+target_compile_definitions(quda PUBLIC QUDA_DSLASH5TYPE=${QUDA_DSLASH5TYPE})
 
 if(QUDA_ENABLE_MMA)
   target_compile_definitions(quda PUBLIC QUDA_ENABLE_MMA)
diff --git a/lib/dslash_domain_wall_4d_fused_m5.hpp b/lib/dslash_domain_wall_4d_fused_m5.hpp
index 48e1aaae9e..1ec465e318 100644
--- a/lib/dslash_domain_wall_4d_fused_m5.hpp
+++ b/lib/dslash_domain_wall_4d_fused_m5.hpp
@@ -120,14 +120,20 @@ namespace quda
   template <Dslash5Type...> struct Dslash5TypeList {
   };
 
+  template <bool distance_pc> struct DistanceType {
+  };
+
   template <typename Float, int nColor, typename DDArg, QudaReconstructType recon> struct DomainWall4DApplyFusedM5 {
 
-    template <Dslash5Type dslash5_type_impl, Dslash5Type... N>
+    template <bool distance_pc, Dslash5Type dslash5_type_impl, Dslash5Type... N>
     DomainWall4DApplyFusedM5(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
                              cvector_ref<const ColorSpinorField> &x, const GaugeField &U,
                              cvector_ref<ColorSpinorField> &y, const Complex *b_5, const Complex *c_5, double a,
                              double m_5, int parity, bool dagger, const int *comm_override, double m_f,
-                             Dslash5TypeList<dslash5_type_impl, N...>, TimeProfile &profile)
+                             DistanceType<distance_pc>, Dslash5TypeList<dslash5_type_impl, N...>, TimeProfile &profile)
+#ifdef SIGNATURE_ONLY
+      ;
+#else
     {
 #ifdef NVSHMEM_COMMS
       errorQuda("Fused Mobius/DWF-4D kernels do not currently work with NVSHMEM.");
@@ -140,6 +146,7 @@ namespace quda
       dslash::DslashPolicyTune<decltype(dwf)> policy(dwf, in, halo, profile);
 #endif
     }
+#endif
   };
 
   // use custom instantiate to deal with field splitting if needed
diff --git a/lib/dslash_domain_wall_4d_fused_m5.in.cu b/lib/dslash_domain_wall_4d_fused_m5.in.cu
new file mode 100644
index 0000000000..97f131e92e
--- /dev/null
+++ b/lib/dslash_domain_wall_4d_fused_m5.in.cu
@@ -0,0 +1,28 @@
+#include <dslash_wilson_clover_preconditioned.hpp>
+
+/**
+   This is the Wilson-clover linear operator
+*/
+
+namespace quda
+{
+
+  constexpr QudaPrecision precision = QUDA_ @QUDA_DSLASH_PREC @_PRECISION;
+  constexpr int nColor = @QUDA_DSLASH_NCOLOR @;
+  constexpr int reconI = @QUDA_DSLASH_RECONI @;
+  constexpr bool distance_pc = @QUDA_DSLASH_DISTANCE @;
+
+  constexpr Dslash5Type dslash5_type = Dslash5Type::@QUDA_DSLASH_DWTYPE @;
+
+  typedef @QUDA_DSLASH_DDARG @DDArg;
+  typedef precision_type_mapper<precision>::type Float;
+
+  template struct DomainWall4DApplyFusedM5<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>;
+
+  template DomainWall4DApplyFusedM5<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>::DomainWall4DFusedM5Apply(
+    cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in, cvector_ref<const ColorSpinorField> &x,
+    const GaugeField &U, cvector_ref<ColorSpinorField> &y, const Complex *b_5, const Complex *c_5, double a, double m_5,
+    int parity, bool dagger, const int *comm_override, double m_f, Dslash5TypeList<dslash5_type>,
+    DistanceType<distance_pc>, TimeProfile &profile);
+
+} // namespace quda
diff --git a/lib/dslash_domain_wall_4d_m5inv.cu b/lib/dslash_domain_wall_4d_m5inv.cu
index e739dddadb..b7b107c5fd 100644
--- a/lib/dslash_domain_wall_4d_m5inv.cu
+++ b/lib/dslash_domain_wall_4d_m5inv.cu
@@ -1,4 +1,6 @@
+#define SIGNATURE_ONLY
 #include <dslash_domain_wall_4d_fused_m5.hpp>
+#undef SIGNATURE_ONLY
 
 /**
    This is the gauged domain-wall 4-d preconditioned operator, fused with immediately followed fifth dimension operators.
@@ -16,9 +18,10 @@ namespace quda
                               bool dagger, const int *comm_override, double m_f, TimeProfile &profile)
   {
     if constexpr (is_enabled<QUDA_DOMAIN_WALL_4D_DSLASH>()) {
+      auto dummy = DistanceType<false>();
       auto dummy_list = Dslash5TypeList<Dslash5Type::M5_INV_MOBIUS>();
       instantiate<DomainWall4DApplyFusedM5>(out, in, x, y, U, b_5, c_5, a, m_5, parity, dagger, comm_override, m_f,
-                                            dummy_list, profile);
+                                            dummy, dummy_list, profile);
     } else {
       errorQuda("Domain-wall operator has not been built");
     }
diff --git a/lib/dslash_domain_wall_4d_m5inv.hpp b/lib/dslash_domain_wall_4d_m5inv.hpp
new file mode 100644
index 0000000000..e739dddadb
--- /dev/null
+++ b/lib/dslash_domain_wall_4d_m5inv.hpp
@@ -0,0 +1,27 @@
+#include <dslash_domain_wall_4d_fused_m5.hpp>
+
+/**
+   This is the gauged domain-wall 4-d preconditioned operator, fused with immediately followed fifth dimension operators.
+*/
+
+namespace quda
+{
+
+  // Apply the 4-d preconditioned domain-wall Dslash operator
+  //   i.e. out(x) = M*in = in(x) + a*\sum_mu U_{-\mu}(x)in(x+mu) + U^\dagger_mu(x-mu)in(x-mu)
+  // ... and then m5inv
+  void ApplyDomainWall4DM5inv(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+                              const GaugeField &U, double a, double m_5, const Complex *b_5, const Complex *c_5,
+                              cvector_ref<const ColorSpinorField> &x, cvector_ref<ColorSpinorField> &y, int parity,
+                              bool dagger, const int *comm_override, double m_f, TimeProfile &profile)
+  {
+    if constexpr (is_enabled<QUDA_DOMAIN_WALL_4D_DSLASH>()) {
+      auto dummy_list = Dslash5TypeList<Dslash5Type::M5_INV_MOBIUS>();
+      instantiate<DomainWall4DApplyFusedM5>(out, in, x, y, U, b_5, c_5, a, m_5, parity, dagger, comm_override, m_f,
+                                            dummy_list, profile);
+    } else {
+      errorQuda("Domain-wall operator has not been built");
+    }
+  }
+
+} // namespace quda
diff --git a/lib/dslash_domain_wall_4d_m5inv.in.cu b/lib/dslash_domain_wall_4d_m5inv.in.cu
new file mode 100644
index 0000000000..e739dddadb
--- /dev/null
+++ b/lib/dslash_domain_wall_4d_m5inv.in.cu
@@ -0,0 +1,27 @@
+#include <dslash_domain_wall_4d_fused_m5.hpp>
+
+/**
+   This is the gauged domain-wall 4-d preconditioned operator, fused with immediately followed fifth dimension operators.
+*/
+
+namespace quda
+{
+
+  // Apply the 4-d preconditioned domain-wall Dslash operator
+  //   i.e. out(x) = M*in = in(x) + a*\sum_mu U_{-\mu}(x)in(x+mu) + U^\dagger_mu(x-mu)in(x-mu)
+  // ... and then m5inv
+  void ApplyDomainWall4DM5inv(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+                              const GaugeField &U, double a, double m_5, const Complex *b_5, const Complex *c_5,
+                              cvector_ref<const ColorSpinorField> &x, cvector_ref<ColorSpinorField> &y, int parity,
+                              bool dagger, const int *comm_override, double m_f, TimeProfile &profile)
+  {
+    if constexpr (is_enabled<QUDA_DOMAIN_WALL_4D_DSLASH>()) {
+      auto dummy_list = Dslash5TypeList<Dslash5Type::M5_INV_MOBIUS>();
+      instantiate<DomainWall4DApplyFusedM5>(out, in, x, y, U, b_5, c_5, a, m_5, parity, dagger, comm_override, m_f,
+                                            dummy_list, profile);
+    } else {
+      errorQuda("Domain-wall operator has not been built");
+    }
+  }
+
+} // namespace quda
diff --git a/lib/dslash_domain_wall_4d_m5inv_m5inv.cu b/lib/dslash_domain_wall_4d_m5inv_m5inv.cu
index 870c311fb2..cd4b760a72 100644
--- a/lib/dslash_domain_wall_4d_m5inv_m5inv.cu
+++ b/lib/dslash_domain_wall_4d_m5inv_m5inv.cu
@@ -1,4 +1,6 @@
+#define SIGNATURE_ONLY
 #include <dslash_domain_wall_4d_fused_m5.hpp>
+#undef SIGNATURE_ONLY
 
 /**
    This is the gauged domain-wall 4-d preconditioned operator, fused with immediately followed fifth dimension operators.
@@ -16,9 +18,10 @@ namespace quda
                                    bool dagger, const int *comm_override, double m_f, TimeProfile &profile)
   {
     if constexpr (is_enabled<QUDA_DOMAIN_WALL_4D_DSLASH>()) {
+      auto dummy = DistanceType<false>();
       auto dummy_list = Dslash5TypeList<Dslash5Type::M5_INV_MOBIUS_M5_INV_DAG>();
       instantiate<DomainWall4DApplyFusedM5>(out, in, x, y, U, b_5, c_5, a, m_5, parity, dagger, comm_override, m_f,
-                                            dummy_list, profile);
+                                            dummy, dummy_list, profile);
     } else {
       errorQuda("Domain-wall operator has not been built");
     }
diff --git a/lib/dslash_domain_wall_4d_m5inv_m5pre.cu b/lib/dslash_domain_wall_4d_m5inv_m5pre.cu
index fedfd154b9..414d26aebf 100644
--- a/lib/dslash_domain_wall_4d_m5inv_m5pre.cu
+++ b/lib/dslash_domain_wall_4d_m5inv_m5pre.cu
@@ -1,4 +1,6 @@
+#define SIGNATURE_ONLY
 #include <dslash_domain_wall_4d_fused_m5.hpp>
+#undef SIGNATURE_ONLY
 
 /**
    This is the gauged domain-wall 4-d preconditioned operator, fused with immediately followed fifth dimension operators.
@@ -16,9 +18,10 @@ namespace quda
                                    bool dagger, const int *comm_override, double m_f, TimeProfile &profile)
   {
     if constexpr (is_enabled<QUDA_DOMAIN_WALL_4D_DSLASH>()) {
+      auto dummy = DistanceType<false>();
       auto dummy_list = Dslash5TypeList<Dslash5Type::M5_INV_MOBIUS_M5_PRE>();
       instantiate<DomainWall4DApplyFusedM5>(out, in, x, y, U, b_5, c_5, a, m_5, parity, dagger, comm_override, m_f,
-                                            dummy_list, profile);
+                                            dummy, dummy_list, profile);
     } else {
       errorQuda("Domain-wall operator has not been built");
     }
diff --git a/lib/dslash_domain_wall_4d_m5mob.cu b/lib/dslash_domain_wall_4d_m5mob.cu
index 76527b9321..87f5804b0e 100644
--- a/lib/dslash_domain_wall_4d_m5mob.cu
+++ b/lib/dslash_domain_wall_4d_m5mob.cu
@@ -1,4 +1,6 @@
+#define SIGNATURE_ONLY
 #include <dslash_domain_wall_4d_fused_m5.hpp>
+#undef SIGNATURE_ONLY
 
 /**
    This is the gauged domain-wall 4-d preconditioned operator, fused with immediately followed fifth dimension operators.
@@ -16,9 +18,10 @@ namespace quda
                               bool dagger, const int *comm_override, double m_f, TimeProfile &profile)
   {
     if constexpr (is_enabled<QUDA_DOMAIN_WALL_4D_DSLASH>()) {
+      auto dummy = DistanceType<false>();
       auto dummy_list = Dslash5TypeList<Dslash5Type::DSLASH5_MOBIUS>();
       instantiate<DomainWall4DApplyFusedM5>(out, in, x, y, U, b_5, c_5, a, m_5, parity, dagger, comm_override, m_f,
-                                            dummy_list, profile);
+                                            dummy, dummy_list, profile);
     } else {
       errorQuda("Domain-wall operator has not been built");
     }
diff --git a/lib/dslash_domain_wall_4d_m5pre.cu b/lib/dslash_domain_wall_4d_m5pre.cu
index b9ea9dae29..f88e4ea6d4 100644
--- a/lib/dslash_domain_wall_4d_m5pre.cu
+++ b/lib/dslash_domain_wall_4d_m5pre.cu
@@ -1,4 +1,6 @@
+#define SIGNATURE_ONLY
 #include <dslash_domain_wall_4d_fused_m5.hpp>
+#undef SIGNATURE_ONLY
 
 /**
    This is the gauged domain-wall 4-d preconditioned operator, fused with immediately followed fifth dimension operators.
@@ -16,9 +18,10 @@ namespace quda
                               bool dagger, const int *comm_override, double m_f, TimeProfile &profile)
   {
     if constexpr (is_enabled<QUDA_DOMAIN_WALL_4D_DSLASH>()) {
+      auto dummy = DistanceType<false>();
       auto dummy_list = Dslash5TypeList<Dslash5Type::DSLASH5_MOBIUS_PRE>();
       instantiate<DomainWall4DApplyFusedM5>(out, in, x, y, U, b_5, c_5, a, m_5, parity, dagger, comm_override, m_f,
-                                            dummy_list, profile);
+                                            dummy, dummy_list, profile);
     } else {
       errorQuda("Domain-wall operator has not been built");
     }
diff --git a/lib/dslash_domain_wall_4d_m5pre_m5inv.cu b/lib/dslash_domain_wall_4d_m5pre_m5inv.cu
index 33cb13c1b1..c4c708a102 100644
--- a/lib/dslash_domain_wall_4d_m5pre_m5inv.cu
+++ b/lib/dslash_domain_wall_4d_m5pre_m5inv.cu
@@ -1,4 +1,6 @@
+#define SIGNATURE_ONLY
 #include <dslash_domain_wall_4d_fused_m5.hpp>
+#undef SIGNATURE_ONLY
 
 /**
    This is the gauged domain-wall 4-d preconditioned operator, fused with immediately followed fifth dimension operators.
@@ -16,9 +18,10 @@ namespace quda
                                    bool dagger, const int *comm_override, double m_f, TimeProfile &profile)
   {
     if constexpr (is_enabled<QUDA_DOMAIN_WALL_4D_DSLASH>()) {
+      auto dummy = DistanceType<false>();
       auto dummy_list = Dslash5TypeList<Dslash5Type::M5_PRE_MOBIUS_M5_INV>();
       instantiate<DomainWall4DApplyFusedM5>(out, in, x, y, U, b_5, c_5, a, m_5, parity, dagger, comm_override, m_f,
-                                            dummy_list, profile);
+                                            dummy, dummy_list, profile);
     } else {
       errorQuda("Domain-wall operator has not been built");
     }
diff --git a/lib/dslash_domain_wall_4d_m5pre_m5mob.cu b/lib/dslash_domain_wall_4d_m5pre_m5mob.cu
index 7ceabeec6a..017ddae244 100644
--- a/lib/dslash_domain_wall_4d_m5pre_m5mob.cu
+++ b/lib/dslash_domain_wall_4d_m5pre_m5mob.cu
@@ -1,4 +1,6 @@
+#define SIGNATURE_ONLY
 #include <dslash_domain_wall_4d_fused_m5.hpp>
+#undef SIGNATURE_ONLY
 
 /**
    This is the gauged domain-wall 4-d preconditioned operator, fused with immediately followed fifth dimension operators.
@@ -16,9 +18,10 @@ namespace quda
                                    bool dagger, const int *comm_override, double m_f, TimeProfile &profile)
   {
     if constexpr (is_enabled<QUDA_DOMAIN_WALL_4D_DSLASH>()) {
+      auto dummy = DistanceType<false>();
       auto dummy_list = Dslash5TypeList<Dslash5Type::DSLASH5_MOBIUS_PRE_M5_MOB>();
       instantiate<DomainWall4DApplyFusedM5>(out, in, x, y, U, b_5, c_5, a, m_5, parity, dagger, comm_override, m_f,
-                                            dummy_list, profile);
+                                            dummy, dummy_list, profile);
     } else {
       errorQuda("Domain-wall operator has not been built");
     }

From c73991f60cf9ee25c38b0d097810f4e188a7bbf1 Mon Sep 17 00:00:00 2001
From: Ferenc Pittler <pittler@hiskp.uni-bonn.de>
Date: Thu, 7 Nov 2024 10:41:02 +0100
Subject: [PATCH 101/108] including wilson clover hasenbusch twist

---
 lib/CMakeLists.txt                            |   3 +
 lib/dslash_domain_wall_4d_fused_m5.in.cu      |  12 +-
 lib/dslash_wilson_clover_hasenbusch_twist.cu  |  94 +-----
 lib/dslash_wilson_clover_hasenbusch_twist.hpp |  99 ++++++
 ...slash_wilson_clover_hasenbusch_twist.in.cu |  21 ++
 ..._clover_hasenbusch_twist_preconditioned.cu | 272 +----------------
 ...clover_hasenbusch_twist_preconditioned.hpp | 282 ++++++++++++++++++
 ...over_hasenbusch_twist_preconditioned.in.cu |  30 ++
 ...enbusch_twist_preconditioned_ClovInv.in.cu |  21 ++
 ...busch_twist_preconditioned_NoClovInv.in.cu |  24 ++
 lib/laplace.cu                                | 177 +----------
 lib/laplace.hpp                               | 183 ++++++++++++
 lib/laplace.in.cu                             |  21 ++
 13 files changed, 708 insertions(+), 531 deletions(-)
 create mode 100644 lib/dslash_wilson_clover_hasenbusch_twist.hpp
 create mode 100644 lib/dslash_wilson_clover_hasenbusch_twist.in.cu
 create mode 100644 lib/dslash_wilson_clover_hasenbusch_twist_preconditioned.hpp
 create mode 100644 lib/dslash_wilson_clover_hasenbusch_twist_preconditioned.in.cu
 create mode 100644 lib/dslash_wilson_clover_hasenbusch_twist_preconditioned_ClovInv.in.cu
 create mode 100644 lib/dslash_wilson_clover_hasenbusch_twist_preconditioned_NoClovInv.in.cu
 create mode 100644 lib/laplace.hpp
 create mode 100644 lib/laplace.in.cu

diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt
index ee52bc1e56..4198ce185f 100644
--- a/lib/CMakeLists.txt
+++ b/lib/CMakeLists.txt
@@ -198,6 +198,8 @@ set(QUDA_DSLASH_FILENAME_LIST "")
 list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_wilson")
 list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_wilson_clover")
 list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_wilson_clover_preconditioned")
+list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_wilson_clover_hasenbusch_twist")
+list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_wilson_clover_hasenbusch_twist_preconditioned")
 list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_twisted_mass")
 list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_twisted_clover")
 list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_twisted_clover_preconditioned")
@@ -209,6 +211,7 @@ list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_staggered")
 list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_improved_staggered")
 list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_domain_wall_4d")
 list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_domain_wall_5d")
+list(APPEND QUDA_DSLASH_FILENAME_LIST "laplace")
 
 set(QUDA_DSLASH_PREC_LIST "")
 set(QUDA_DSLASH_NCOLOR_LIST "3")
diff --git a/lib/dslash_domain_wall_4d_fused_m5.in.cu b/lib/dslash_domain_wall_4d_fused_m5.in.cu
index 97f131e92e..c376eb0618 100644
--- a/lib/dslash_domain_wall_4d_fused_m5.in.cu
+++ b/lib/dslash_domain_wall_4d_fused_m5.in.cu
@@ -7,14 +7,14 @@
 namespace quda
 {
 
-  constexpr QudaPrecision precision = QUDA_ @QUDA_DSLASH_PREC @_PRECISION;
-  constexpr int nColor = @QUDA_DSLASH_NCOLOR @;
-  constexpr int reconI = @QUDA_DSLASH_RECONI @;
-  constexpr bool distance_pc = @QUDA_DSLASH_DISTANCE @;
+  constexpr QudaPrecision precision = QUDA_ @QUDA_DSLASH_PREC@_PRECISION;
+  constexpr int nColor = @QUDA_DSLASH_NCOLOR@;
+  constexpr int reconI = @QUDA_DSLASH_RECONI@;
+  constexpr bool distance_pc = @QUDA_DSLASH_DISTANCE@;
 
-  constexpr Dslash5Type dslash5_type = Dslash5Type::@QUDA_DSLASH_DWTYPE @;
+  constexpr Dslash5Type dslash5_type = Dslash5Type::@QUDA_DSLASH_DWTYPE@;
 
-  typedef @QUDA_DSLASH_DDARG @DDArg;
+  typedef @QUDA_DSLASH_DDARG@ DDArg;
   typedef precision_type_mapper<precision>::type Float;
 
   template struct DomainWall4DApplyFusedM5<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>;
diff --git a/lib/dslash_wilson_clover_hasenbusch_twist.cu b/lib/dslash_wilson_clover_hasenbusch_twist.cu
index 17b64991fe..f600980588 100644
--- a/lib/dslash_wilson_clover_hasenbusch_twist.cu
+++ b/lib/dslash_wilson_clover_hasenbusch_twist.cu
@@ -1,94 +1,10 @@
-#include <gauge_field.h>
-#include <color_spinor_field.h>
-#include <clover_field.h>
-#include <dslash.h>
-#include <worker.h>
-
-#include <dslash_policy.hpp>
-#include <kernels/dslash_wilson_clover_hasenbusch_twist.cuh>
-
-/**
-   This is the Wilson-clover linear operator
-*/
+#define SIGNATURE_ONLY
+#include <dslash_wilson_clover_hasenbusch_twist.hpp>
+#undef SIGNATURE_ONLY
 
 namespace quda
 {
 
-  template <typename Arg> class WilsonCloverHasenbuschTwist : public Dslash<cloverHasenbusch, Arg>
-  {
-    using Dslash = Dslash<cloverHasenbusch, Arg>;
-    using Dslash::arg;
-    using Dslash::in;
-
-  public:
-    WilsonCloverHasenbuschTwist(Arg &arg, cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
-                                const ColorSpinorField &halo) :
-      Dslash(arg, out, in, halo)
-    {
-    }
-
-    void apply(const qudaStream_t &stream)
-    {
-      TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
-      Dslash::setParam(tp);
-      if (arg.xpay)
-        Dslash::template instantiate<packShmem, true>(tp, stream);
-      else
-        errorQuda("Wilson-clover - Hasenbusch Twist operator only defined for xpay=true");
-    }
-
-    long long flops() const
-    {
-      int clover_flops = in.size() * 504;
-      long long flops = Dslash::flops();
-      switch (arg.kernel_type) {
-      case INTERIOR_KERNEL:
-      case UBER_KERNEL:
-      case KERNEL_POLICY:
-        flops += clover_flops * in.Volume();
-
-        // -mu * (i gamma_5 A) (A x)
-        flops += ((clover_flops + 48) * in.Volume());
-
-        break;
-      default: break; // all clover flops are in the interior kernel
-      }
-      return flops;
-    }
-
-    long long bytes() const
-    {
-      int clover_bytes = in.size() * 72 * in.Precision() + (isFixed<typename Arg::Float>::value ? 2 * sizeof(float) : 0);
-
-      long long bytes = Dslash::bytes();
-      switch (arg.kernel_type) {
-      case INTERIOR_KERNEL:
-      case UBER_KERNEL:
-      case KERNEL_POLICY: bytes += clover_bytes * in.Volume(); break;
-      default: break;
-      }
-
-      return bytes;
-    }
-  };
-
-  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon>
-  struct WilsonCloverHasenbuschTwistApply {
-
-    WilsonCloverHasenbuschTwistApply(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
-                                     cvector_ref<const ColorSpinorField> &x, const GaugeField &U, const CloverField &A,
-                                     double a, double b, int parity, bool dagger, const int *comm_override,
-                                     TimeProfile &profile)
-    {
-      constexpr int nDim = 4;
-      auto halo = ColorSpinorField::create_comms_batch(in);
-      WilsonCloverHasenbuschTwistArg<Float, nColor, nDim, DDArg, recon> arg(out, in, halo, U, A, a, b, x, parity,
-                                                                            dagger, comm_override);
-      WilsonCloverHasenbuschTwist<decltype(arg)> wilson(arg, out, in, halo);
-      dslash::DslashPolicyTune<decltype(wilson)> policy(wilson, in, halo, profile);
-    }
-  };
-
   // Apply the Wilson-clover operator
   // out(x) = M*in = (A(x) + a * \sum_mu U_{-\mu}(x)in(x+mu) + U^\dagger_mu(x-mu)in(x-mu))
   // Uses the kappa normalization for the Wilson operator.
@@ -98,7 +14,9 @@ namespace quda
                                         const int *comm_override, TimeProfile &profile)
   {
     if constexpr (is_enabled<QUDA_CLOVER_HASENBUSCH_TWIST_DSLASH>()) {
-      instantiate<WilsonCloverHasenbuschTwistApply>(out, in, x, U, A, a, b, parity, dagger, comm_override, profile);
+      auto dummy = DistanceType<false>();
+      instantiate<WilsonCloverHasenbuschTwistApply>(out, in, x, U, A, a, b, parity, dagger, comm_override, dummy,
+                                                    profile);
     } else {
       errorQuda("Clover Hasensbuch Twist operator has not been built");
     }
diff --git a/lib/dslash_wilson_clover_hasenbusch_twist.hpp b/lib/dslash_wilson_clover_hasenbusch_twist.hpp
new file mode 100644
index 0000000000..6991048398
--- /dev/null
+++ b/lib/dslash_wilson_clover_hasenbusch_twist.hpp
@@ -0,0 +1,99 @@
+#include <gauge_field.h>
+#include <color_spinor_field.h>
+#include <clover_field.h>
+#include <dslash.h>
+#include <worker.h>
+
+#include <dslash_policy.hpp>
+#include <kernels/dslash_wilson_clover_hasenbusch_twist.cuh>
+
+/**
+   This is the Wilson-clover linear operator
+*/
+
+namespace quda
+{
+
+  template <typename Arg> class WilsonCloverHasenbuschTwist : public Dslash<cloverHasenbusch, Arg>
+  {
+    using Dslash = Dslash<cloverHasenbusch, Arg>;
+    using Dslash::arg;
+    using Dslash::in;
+
+  public:
+    WilsonCloverHasenbuschTwist(Arg &arg, cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+                                const ColorSpinorField &halo) :
+      Dslash(arg, out, in, halo)
+    {
+    }
+
+    void apply(const qudaStream_t &stream)
+    {
+      TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
+      Dslash::setParam(tp);
+      if (arg.xpay)
+        Dslash::template instantiate<packShmem, true>(tp, stream);
+      else
+        errorQuda("Wilson-clover - Hasenbusch Twist operator only defined for xpay=true");
+    }
+
+    long long flops() const
+    {
+      int clover_flops = in.size() * 504;
+      long long flops = Dslash::flops();
+      switch (arg.kernel_type) {
+      case INTERIOR_KERNEL:
+      case UBER_KERNEL:
+      case KERNEL_POLICY:
+        flops += clover_flops * in.Volume();
+
+        // -mu * (i gamma_5 A) (A x)
+        flops += ((clover_flops + 48) * in.Volume());
+
+        break;
+      default: break; // all clover flops are in the interior kernel
+      }
+      return flops;
+    }
+
+    long long bytes() const
+    {
+      int clover_bytes = in.size() * 72 * in.Precision() + (isFixed<typename Arg::Float>::value ? 2 * sizeof(float) : 0);
+
+      long long bytes = Dslash::bytes();
+      switch (arg.kernel_type) {
+      case INTERIOR_KERNEL:
+      case UBER_KERNEL:
+      case KERNEL_POLICY: bytes += clover_bytes * in.Volume(); break;
+      default: break;
+      }
+
+      return bytes;
+    }
+  };
+
+  template <bool distance_pc> struct DistanceType {
+  };
+
+  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon>
+  struct WilsonCloverHasenbuschTwistApply {
+    template <bool distance_pc>
+    WilsonCloverHasenbuschTwistApply(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+                                     cvector_ref<const ColorSpinorField> &x, const GaugeField &U, const CloverField &A,
+                                     double a, double b, int parity, bool dagger, const int *comm_override,DistanceType<distance_pc>,
+                                     TimeProfile &profile)
+#ifdef SIGNATURE_ONLY // Used to hide from the compiler the implementation of the function
+      ;
+#else
+    {
+      constexpr int nDim = 4;
+      auto halo = ColorSpinorField::create_comms_batch(in);
+      WilsonCloverHasenbuschTwistArg<Float, nColor, nDim, DDArg, recon> arg(out, in, halo, U, A, a, b, x, parity,
+                                                                            dagger, comm_override);
+      WilsonCloverHasenbuschTwist<decltype(arg)> wilson(arg, out, in, halo);
+      dslash::DslashPolicyTune<decltype(wilson)> policy(wilson, in, halo, profile);
+    }
+#endif
+  };
+
+} // namespace quda
diff --git a/lib/dslash_wilson_clover_hasenbusch_twist.in.cu b/lib/dslash_wilson_clover_hasenbusch_twist.in.cu
new file mode 100644
index 0000000000..08840c0e82
--- /dev/null
+++ b/lib/dslash_wilson_clover_hasenbusch_twist.in.cu
@@ -0,0 +1,21 @@
+#include <dslash_wilson_clover_hasenbusch_twist.hpp>
+
+namespace quda
+{
+
+  constexpr QudaPrecision precision = QUDA_@QUDA_DSLASH_PREC@_PRECISION;
+  constexpr int nColor = @QUDA_DSLASH_NCOLOR@;
+  constexpr int reconI = @QUDA_DSLASH_RECONI@;
+  constexpr bool distance_pc = @QUDA_DSLASH_DISTANCE@;
+
+  typedef @QUDA_DSLASH_DDARG@ DDArg;
+  typedef precision_type_mapper<precision>::type Float;
+
+  template struct WilsonCloverHasenbuschTwistApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>;
+
+  template WilsonCloverHasenbuschTwistApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>::WilsonCloverHasenbuschTwistApply(
+    cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in, cvector_ref<const ColorSpinorField> &x,
+    const GaugeField &U, const CloverField &A, double a, double b, int parity, bool dagger, const int *comm_override,
+    DistanceType<distance_pc>, TimeProfile &profile);
+
+} // namespace quda
diff --git a/lib/dslash_wilson_clover_hasenbusch_twist_preconditioned.cu b/lib/dslash_wilson_clover_hasenbusch_twist_preconditioned.cu
index 7beda2f745..67dab8b2a8 100644
--- a/lib/dslash_wilson_clover_hasenbusch_twist_preconditioned.cu
+++ b/lib/dslash_wilson_clover_hasenbusch_twist_preconditioned.cu
@@ -1,141 +1,10 @@
-#include <gauge_field.h>
-#include <color_spinor_field.h>
-#include <clover_field.h>
-#include <dslash.h>
-#include <worker.h>
-
-#include <dslash_policy.hpp>
-#include <kernels/dslash_wilson_clover_hasenbusch_twist_preconditioned.cuh>
+#define SIGNATURE_ONLY
+#include <dslash_wilson_clover_hasenbusch_twist_preconditioned.hpp>
+#undef SIGNATURE_ONLY
 
 namespace quda
 {
 
-  /* ***************************
-   * No Clov Inv:  1 - k^2 D - i mu gamma_5 A
-   * **************************/
-  template <typename Arg>
-  class WilsonCloverHasenbuschTwistPCNoClovInv : public Dslash<cloverHasenbuschPreconditioned, Arg>
-  {
-    using Dslash = Dslash<cloverHasenbuschPreconditioned, Arg>;
-    using Dslash::arg;
-    using Dslash::halo;
-    using Dslash::in;
-
-  public:
-    WilsonCloverHasenbuschTwistPCNoClovInv(Arg &arg, cvector_ref<ColorSpinorField> &out,
-                                           cvector_ref<const ColorSpinorField> &in, const ColorSpinorField &halo) :
-      Dslash(arg, out, in, halo)
-    {
-    }
-
-    void apply(const qudaStream_t &stream)
-    {
-      TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
-      Dslash::setParam(tp);
-
-      // specialize here to constrain the template instantiation
-      if (arg.nParity == 1) {
-        if (arg.xpay)
-          Dslash::template instantiate<packShmem, 1, true>(tp, stream);
-        else
-          errorQuda("Operator only defined for xpay=true");
-      } else {
-        errorQuda("Operator not defined nParity=%d", arg.nParity);
-      }
-    }
-
-    long long flops() const
-    {
-      int clover_flops = 504;
-      long long flops = Dslash::flops();
-      switch (arg.kernel_type) {
-      case EXTERIOR_KERNEL_X:
-      case EXTERIOR_KERNEL_Y:
-      case EXTERIOR_KERNEL_Z:
-      case EXTERIOR_KERNEL_T:
-        // 2 from fwd / back face * 1 clover terms:
-        // there is no A^{-1}D only D
-        // there is one clover_term and 48 is the - mu (igamma_5) A
-        flops += 2 * (clover_flops + 48) * halo.GhostFace()[arg.kernel_type];
-        break;
-      case EXTERIOR_KERNEL_ALL:
-        flops += 2 * (clover_flops + 48)
-          * (halo.GhostFace()[0] + halo.GhostFace()[1] + halo.GhostFace()[2] + halo.GhostFace()[3]);
-        break;
-      case INTERIOR_KERNEL:
-      case UBER_KERNEL:
-      case KERNEL_POLICY:
-        flops += (clover_flops + 48) * halo.Volume();
-
-        if (arg.kernel_type == KERNEL_POLICY) break;
-        // now correct for flops done by exterior kernel
-        long long ghost_sites = 0;
-        for (int d = 0; d < 4; d++)
-          if (arg.commDim[d]) ghost_sites += 2 * halo.GhostFace()[d];
-        flops -= (clover_flops + 48) * ghost_sites;
-
-        break;
-      }
-      return flops;
-    }
-
-    long long bytes() const
-    {
-      int clover_bytes = 72 * in.Precision() + (isFixed<typename Arg::Float>::value ? 2 * sizeof(float) : 0);
-
-      long long bytes = Dslash::bytes();
-      switch (arg.kernel_type) {
-      case EXTERIOR_KERNEL_X:
-      case EXTERIOR_KERNEL_Y:
-      case EXTERIOR_KERNEL_Z:
-      case EXTERIOR_KERNEL_T:
-        // Factor of 2 is from the fwd/back faces.
-        bytes += clover_bytes * 2 * halo.GhostFace()[arg.kernel_type];
-        break;
-      case EXTERIOR_KERNEL_ALL:
-        // Factor of 2 is from the fwd/back faces
-        bytes
-          += clover_bytes * 2 * (halo.GhostFace()[0] + halo.GhostFace()[1] + halo.GhostFace()[2] + halo.GhostFace()[3]);
-        break;
-      case INTERIOR_KERNEL:
-      case UBER_KERNEL:
-      case KERNEL_POLICY:
-
-        bytes += clover_bytes * halo.Volume();
-
-        if (arg.kernel_type == KERNEL_POLICY) break;
-        // now correct for bytes done by exterior kernel
-        long long ghost_sites = 0;
-        for (int d = 0; d < 4; d++)
-          if (arg.commDim[d]) ghost_sites += 2 * halo.GhostFace()[d];
-        bytes -= clover_bytes * ghost_sites;
-
-        break;
-      }
-
-      return bytes;
-    }
-  };
-
-  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon>
-  struct WilsonCloverHasenbuschTwistPCNoClovInvApply {
-
-    WilsonCloverHasenbuschTwistPCNoClovInvApply(cvector_ref<ColorSpinorField> &out,
-                                                cvector_ref<const ColorSpinorField> &in,
-                                                cvector_ref<const ColorSpinorField> &x, const GaugeField &U,
-                                                const CloverField &A, double a, double b, int parity, bool dagger,
-                                                const int *comm_override, TimeProfile &profile)
-    {
-      constexpr int nDim = 4;
-      auto halo = ColorSpinorField::create_comms_batch(in);
-      using ArgType = WilsonCloverHasenbuschTwistPCArg<Float, nColor, nDim, DDArg, recon, false>;
-      ArgType arg(out, in, halo, U, A, a, b, x, parity, dagger, comm_override);
-      WilsonCloverHasenbuschTwistPCNoClovInv<ArgType> wilson(arg, out, in, halo);
-
-      dslash::DslashPolicyTune<decltype(wilson)> policy(wilson, in, halo, profile);
-    }
-  };
-
   // Apply the Wilson-clover operator
   // out(x) = M*in = (A(x) + kappa * \sum_mu U_{-\mu}(x)in(x+mu) + U^\dagger_mu(x-mu)in(x-mu))
   // Uses the kappa normalization for the Wilson operator.
@@ -146,142 +15,14 @@ namespace quda
                                                    const int *comm_override, TimeProfile &profile)
   {
     if constexpr (is_enabled<QUDA_CLOVER_HASENBUSCH_TWIST_DSLASH>()) {
-      instantiate<WilsonCloverHasenbuschTwistPCNoClovInvApply>(out, in, x, U, A, a, b, parity, dagger, comm_override,
+      auto dummy = DistanceType<false>();
+      instantiate<WilsonCloverHasenbuschTwistPCNoClovInvApply>(out, in, x, U, A, a, b, parity, dagger, comm_override, dummy,
                                                                profile);
     } else {
       errorQuda("Clover Hasenbusch Twist operator has not been built");
     }
   }
 
-  /* ***************************
-   * Clov Inv
-   *
-   * M = psi_p - k^2 A^{-1} D_p\not{p} - i mu gamma_5 A_{pp} psi_{p}
-   * **************************/
-  template <typename Arg>
-  class WilsonCloverHasenbuschTwistPCClovInv : public Dslash<cloverHasenbuschPreconditioned, Arg>
-  {
-    using Dslash = Dslash<cloverHasenbuschPreconditioned, Arg>;
-    using Dslash::arg;
-    using Dslash::halo;
-    using Dslash::in;
-
-  public:
-    WilsonCloverHasenbuschTwistPCClovInv(Arg &arg, cvector_ref<ColorSpinorField> &out,
-                                         cvector_ref<const ColorSpinorField> &in, const ColorSpinorField &halo) :
-      Dslash(arg, out, in, halo)
-    {
-    }
-
-    void apply(const qudaStream_t &stream)
-    {
-      TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
-      Dslash::setParam(tp);
-
-      // specialize here to constrain the template instantiation
-      if (arg.nParity == 1) {
-        if (arg.xpay)
-          Dslash::template instantiate<packShmem, 1, true>(tp, stream);
-        else
-          errorQuda("Operator only defined for xpay=true");
-      } else {
-        errorQuda("Operator not defined nParity=%d", arg.nParity);
-      }
-    }
-
-    long long flops() const
-    {
-      int clover_flops = 504;
-      long long flops = Dslash::flops();
-      switch (arg.kernel_type) {
-      case EXTERIOR_KERNEL_X:
-      case EXTERIOR_KERNEL_Y:
-      case EXTERIOR_KERNEL_Z:
-      case EXTERIOR_KERNEL_T:
-        // 2 from fwd / back face * 2 clover terms:
-        // one clover_term from the A^{-1}D
-        // second clover_term and 48 is the - mu (igamma_5) A
-        flops += 2 * (2 * clover_flops + 48) * halo.GhostFace()[arg.kernel_type];
-        break;
-      case EXTERIOR_KERNEL_ALL:
-        flops += 2 * (2 * clover_flops + 48)
-          * (halo.GhostFace()[0] + halo.GhostFace()[1] + halo.GhostFace()[2] + halo.GhostFace()[3]);
-        break;
-      case INTERIOR_KERNEL:
-      case UBER_KERNEL:
-      case KERNEL_POLICY:
-        flops += (2 * clover_flops + 48) * halo.Volume();
-
-        if (arg.kernel_type == KERNEL_POLICY) break;
-        // now correct for flops done by exterior kernel
-        long long ghost_sites = 0;
-        for (int d = 0; d < 4; d++)
-          if (arg.commDim[d]) ghost_sites += 2 * halo.GhostFace()[d];
-        flops -= (2 * clover_flops + 48) * ghost_sites;
-
-        break;
-      }
-      return flops;
-    }
-
-    long long bytes() const
-    {
-      int clover_bytes = 72 * in.Precision() + (isFixed<typename Arg::Float>::value ? 2 * sizeof(float) : 0);
-
-      // if we use dynamic clover we read only A (even for A^{-1}
-      // otherwise we read both A and A^{-1}
-      int dyn_factor = arg.dynamic_clover ? 1 : 2;
-
-      long long bytes = Dslash::bytes();
-      switch (arg.kernel_type) {
-      case EXTERIOR_KERNEL_X:
-      case EXTERIOR_KERNEL_Y:
-      case EXTERIOR_KERNEL_Z:
-      case EXTERIOR_KERNEL_T:
-        // Factor of 2 is from the fwd/back faces.
-        bytes += dyn_factor * clover_bytes * 2 * halo.GhostFace()[arg.kernel_type];
-        break;
-      case EXTERIOR_KERNEL_ALL:
-        // Factor of 2 is from the fwd/back faces
-        bytes += dyn_factor * clover_bytes * 2
-          * (halo.GhostFace()[0] + halo.GhostFace()[1] + halo.GhostFace()[2] + halo.GhostFace()[3]);
-        break;
-      case INTERIOR_KERNEL:
-      case UBER_KERNEL:
-      case KERNEL_POLICY:
-
-        bytes += dyn_factor * clover_bytes * halo.Volume();
-
-        if (arg.kernel_type == KERNEL_POLICY) break;
-        // now correct for bytes done by exterior kernel
-        long long ghost_sites = 0;
-        for (int d = 0; d < 4; d++)
-          if (arg.commDim[d]) ghost_sites += 2 * halo.GhostFace()[d];
-        bytes -= dyn_factor * clover_bytes * ghost_sites;
-
-        break;
-      }
-
-      return bytes;
-    }
-  };
-
-  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon>
-  struct WilsonCloverHasenbuschTwistPCClovInvApply {
-
-    WilsonCloverHasenbuschTwistPCClovInvApply(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
-                                              cvector_ref<const ColorSpinorField> &x, const GaugeField &U,
-                                              const CloverField &A, double kappa, double mu, int parity, bool dagger,
-                                              const int *comm_override, TimeProfile &profile)
-    {
-      constexpr int nDim = 4;
-      auto halo = ColorSpinorField::create_comms_batch(in);
-      using ArgType = WilsonCloverHasenbuschTwistPCArg<Float, nColor, nDim, DDArg, recon, true>;
-      ArgType arg(out, in, halo, U, A, kappa, mu, x, parity, dagger, comm_override);
-      WilsonCloverHasenbuschTwistPCClovInv<ArgType> wilson(arg, out, in, halo);
-      dslash::DslashPolicyTune<decltype(wilson)> policy(wilson, in, halo, profile);
-    }
-  };
 
   // Apply the Wilson-clover operator
   // out(x) = M*in = (A(x) + kappa * \sum_mu U_{-\mu}(x)in(x+mu) + U^\dagger_mu(x-mu)in(x-mu))
@@ -293,7 +34,8 @@ namespace quda
                                                  const int *comm_override, TimeProfile &profile)
   {
     if constexpr (is_enabled<QUDA_CLOVER_HASENBUSCH_TWIST_DSLASH>()) {
-      instantiate<WilsonCloverHasenbuschTwistPCClovInvApply>(out, in, x, U, A, a, b, parity, dagger, comm_override,
+      auto dummy = DistanceType<false>();
+      instantiate<WilsonCloverHasenbuschTwistPCClovInvApply>(out, in, x, U, A, a, b, parity, dagger, comm_override, dummy,
                                                              profile);
     } else {
       errorQuda("Clover Hasenbusch Twist operator has not been built");
diff --git a/lib/dslash_wilson_clover_hasenbusch_twist_preconditioned.hpp b/lib/dslash_wilson_clover_hasenbusch_twist_preconditioned.hpp
new file mode 100644
index 0000000000..b6e1fcc64d
--- /dev/null
+++ b/lib/dslash_wilson_clover_hasenbusch_twist_preconditioned.hpp
@@ -0,0 +1,282 @@
+#include <gauge_field.h>
+#include <color_spinor_field.h>
+#include <clover_field.h>
+#include <dslash.h>
+#include <worker.h>
+
+#include <dslash_policy.hpp>
+#include <kernels/dslash_wilson_clover_hasenbusch_twist_preconditioned.cuh>
+
+namespace quda
+{
+
+  /* ***************************
+   * No Clov Inv:  1 - k^2 D - i mu gamma_5 A
+   * **************************/
+  template <typename Arg>
+  class WilsonCloverHasenbuschTwistPCNoClovInv : public Dslash<cloverHasenbuschPreconditioned, Arg>
+  {
+    using Dslash = Dslash<cloverHasenbuschPreconditioned, Arg>;
+    using Dslash::arg;
+    using Dslash::halo;
+    using Dslash::in;
+
+  public:
+    WilsonCloverHasenbuschTwistPCNoClovInv(Arg &arg, cvector_ref<ColorSpinorField> &out,
+                                           cvector_ref<const ColorSpinorField> &in, const ColorSpinorField &halo) :
+      Dslash(arg, out, in, halo)
+    {
+    }
+
+    void apply(const qudaStream_t &stream)
+    {
+      TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
+      Dslash::setParam(tp);
+
+      // specialize here to constrain the template instantiation
+      if (arg.nParity == 1) {
+        if (arg.xpay)
+          Dslash::template instantiate<packShmem, 1, true>(tp, stream);
+        else
+          errorQuda("Operator only defined for xpay=true");
+      } else {
+        errorQuda("Operator not defined nParity=%d", arg.nParity);
+      }
+    }
+
+    long long flops() const
+    {
+      int clover_flops = 504;
+      long long flops = Dslash::flops();
+      switch (arg.kernel_type) {
+      case EXTERIOR_KERNEL_X:
+      case EXTERIOR_KERNEL_Y:
+      case EXTERIOR_KERNEL_Z:
+      case EXTERIOR_KERNEL_T:
+        // 2 from fwd / back face * 1 clover terms:
+        // there is no A^{-1}D only D
+        // there is one clover_term and 48 is the - mu (igamma_5) A
+        flops += 2 * (clover_flops + 48) * halo.GhostFace()[arg.kernel_type];
+        break;
+      case EXTERIOR_KERNEL_ALL:
+        flops += 2 * (clover_flops + 48)
+          * (halo.GhostFace()[0] + halo.GhostFace()[1] + halo.GhostFace()[2] + halo.GhostFace()[3]);
+        break;
+      case INTERIOR_KERNEL:
+      case UBER_KERNEL:
+      case KERNEL_POLICY:
+        flops += (clover_flops + 48) * halo.Volume();
+
+        if (arg.kernel_type == KERNEL_POLICY) break;
+        // now correct for flops done by exterior kernel
+        long long ghost_sites = 0;
+        for (int d = 0; d < 4; d++)
+          if (arg.commDim[d]) ghost_sites += 2 * halo.GhostFace()[d];
+        flops -= (clover_flops + 48) * ghost_sites;
+
+        break;
+      }
+      return flops;
+    }
+
+    long long bytes() const
+    {
+      int clover_bytes = 72 * in.Precision() + (isFixed<typename Arg::Float>::value ? 2 * sizeof(float) : 0);
+
+      long long bytes = Dslash::bytes();
+      switch (arg.kernel_type) {
+      case EXTERIOR_KERNEL_X:
+      case EXTERIOR_KERNEL_Y:
+      case EXTERIOR_KERNEL_Z:
+      case EXTERIOR_KERNEL_T:
+        // Factor of 2 is from the fwd/back faces.
+        bytes += clover_bytes * 2 * halo.GhostFace()[arg.kernel_type];
+        break;
+      case EXTERIOR_KERNEL_ALL:
+        // Factor of 2 is from the fwd/back faces
+        bytes
+          += clover_bytes * 2 * (halo.GhostFace()[0] + halo.GhostFace()[1] + halo.GhostFace()[2] + halo.GhostFace()[3]);
+        break;
+      case INTERIOR_KERNEL:
+      case UBER_KERNEL:
+      case KERNEL_POLICY:
+
+        bytes += clover_bytes * halo.Volume();
+
+        if (arg.kernel_type == KERNEL_POLICY) break;
+        // now correct for bytes done by exterior kernel
+        long long ghost_sites = 0;
+        for (int d = 0; d < 4; d++)
+          if (arg.commDim[d]) ghost_sites += 2 * halo.GhostFace()[d];
+        bytes -= clover_bytes * ghost_sites;
+
+        break;
+      }
+
+      return bytes;
+    }
+  };
+
+  template <bool distance_pc> struct DistanceType {
+  };
+
+  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon>
+  struct WilsonCloverHasenbuschTwistPCNoClovInvApply {
+    template <bool distance_pc>
+    WilsonCloverHasenbuschTwistPCNoClovInvApply(cvector_ref<ColorSpinorField> &out,
+                                                cvector_ref<const ColorSpinorField> &in,
+                                                cvector_ref<const ColorSpinorField> &x, const GaugeField &U,
+                                                const CloverField &A, double a, double b, int parity, bool dagger,
+                                                const int *comm_override, DistanceType<distance_pc>, TimeProfile &profile)
+#ifdef SIGNATURE_ONLY
+    ;
+#else
+    {
+      constexpr int nDim = 4;
+      auto halo = ColorSpinorField::create_comms_batch(in);
+      using ArgType = WilsonCloverHasenbuschTwistPCArg<Float, nColor, nDim, DDArg, recon, false>;
+      ArgType arg(out, in, halo, U, A, a, b, x, parity, dagger, comm_override);
+      WilsonCloverHasenbuschTwistPCNoClovInv<ArgType> wilson(arg, out, in, halo);
+
+      dslash::DslashPolicyTune<decltype(wilson)> policy(wilson, in, halo, profile);
+    }
+#endif
+  };
+
+
+  /* ***************************
+   * Clov Inv
+   *
+   * M = psi_p - k^2 A^{-1} D_p\not{p} - i mu gamma_5 A_{pp} psi_{p}
+   * **************************/
+  template <typename Arg>
+  class WilsonCloverHasenbuschTwistPCClovInv : public Dslash<cloverHasenbuschPreconditioned, Arg>
+  {
+    using Dslash = Dslash<cloverHasenbuschPreconditioned, Arg>;
+    using Dslash::arg;
+    using Dslash::halo;
+    using Dslash::in;
+
+  public:
+    WilsonCloverHasenbuschTwistPCClovInv(Arg &arg, cvector_ref<ColorSpinorField> &out,
+                                         cvector_ref<const ColorSpinorField> &in, const ColorSpinorField &halo) :
+      Dslash(arg, out, in, halo)
+    {
+    }
+
+    void apply(const qudaStream_t &stream)
+    {
+      TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
+      Dslash::setParam(tp);
+
+      // specialize here to constrain the template instantiation
+      if (arg.nParity == 1) {
+        if (arg.xpay)
+          Dslash::template instantiate<packShmem, 1, true>(tp, stream);
+        else
+          errorQuda("Operator only defined for xpay=true");
+      } else {
+        errorQuda("Operator not defined nParity=%d", arg.nParity);
+      }
+    }
+
+    long long flops() const
+    {
+      int clover_flops = 504;
+      long long flops = Dslash::flops();
+      switch (arg.kernel_type) {
+      case EXTERIOR_KERNEL_X:
+      case EXTERIOR_KERNEL_Y:
+      case EXTERIOR_KERNEL_Z:
+      case EXTERIOR_KERNEL_T:
+        // 2 from fwd / back face * 2 clover terms:
+        // one clover_term from the A^{-1}D
+        // second clover_term and 48 is the - mu (igamma_5) A
+        flops += 2 * (2 * clover_flops + 48) * halo.GhostFace()[arg.kernel_type];
+        break;
+      case EXTERIOR_KERNEL_ALL:
+        flops += 2 * (2 * clover_flops + 48)
+          * (halo.GhostFace()[0] + halo.GhostFace()[1] + halo.GhostFace()[2] + halo.GhostFace()[3]);
+        break;
+      case INTERIOR_KERNEL:
+      case UBER_KERNEL:
+      case KERNEL_POLICY:
+        flops += (2 * clover_flops + 48) * halo.Volume();
+
+        if (arg.kernel_type == KERNEL_POLICY) break;
+        // now correct for flops done by exterior kernel
+        long long ghost_sites = 0;
+        for (int d = 0; d < 4; d++)
+          if (arg.commDim[d]) ghost_sites += 2 * halo.GhostFace()[d];
+        flops -= (2 * clover_flops + 48) * ghost_sites;
+
+        break;
+      }
+      return flops;
+    }
+
+    long long bytes() const
+    {
+      int clover_bytes = 72 * in.Precision() + (isFixed<typename Arg::Float>::value ? 2 * sizeof(float) : 0);
+
+      // if we use dynamic clover we read only A (even for A^{-1}
+      // otherwise we read both A and A^{-1}
+      int dyn_factor = arg.dynamic_clover ? 1 : 2;
+
+      long long bytes = Dslash::bytes();
+      switch (arg.kernel_type) {
+      case EXTERIOR_KERNEL_X:
+      case EXTERIOR_KERNEL_Y:
+      case EXTERIOR_KERNEL_Z:
+      case EXTERIOR_KERNEL_T:
+        // Factor of 2 is from the fwd/back faces.
+        bytes += dyn_factor * clover_bytes * 2 * halo.GhostFace()[arg.kernel_type];
+        break;
+      case EXTERIOR_KERNEL_ALL:
+        // Factor of 2 is from the fwd/back faces
+        bytes += dyn_factor * clover_bytes * 2
+          * (halo.GhostFace()[0] + halo.GhostFace()[1] + halo.GhostFace()[2] + halo.GhostFace()[3]);
+        break;
+      case INTERIOR_KERNEL:
+      case UBER_KERNEL:
+      case KERNEL_POLICY:
+
+        bytes += dyn_factor * clover_bytes * halo.Volume();
+
+        if (arg.kernel_type == KERNEL_POLICY) break;
+        // now correct for bytes done by exterior kernel
+        long long ghost_sites = 0;
+        for (int d = 0; d < 4; d++)
+          if (arg.commDim[d]) ghost_sites += 2 * halo.GhostFace()[d];
+        bytes -= dyn_factor * clover_bytes * ghost_sites;
+
+        break;
+      }
+
+      return bytes;
+    }
+  };
+
+
+  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon>
+  struct WilsonCloverHasenbuschTwistPCClovInvApply {
+    template <bool distance_pc>
+    WilsonCloverHasenbuschTwistPCClovInvApply(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+                                              cvector_ref<const ColorSpinorField> &x, const GaugeField &U,
+                                              const CloverField &A, double kappa, double mu, int parity, bool dagger,
+                                              const int *comm_override,DistanceType<distance_pc>, TimeProfile &profile)
+#ifdef SIGNATURE_ONLY
+    ;
+#else
+    {
+      constexpr int nDim = 4;
+      auto halo = ColorSpinorField::create_comms_batch(in);
+      using ArgType = WilsonCloverHasenbuschTwistPCArg<Float, nColor, nDim, DDArg, recon, true>;
+      ArgType arg(out, in, halo, U, A, kappa, mu, x, parity, dagger, comm_override);
+      WilsonCloverHasenbuschTwistPCClovInv<ArgType> wilson(arg, out, in, halo);
+      dslash::DslashPolicyTune<decltype(wilson)> policy(wilson, in, halo, profile);
+    }
+#endif
+  };
+
+} // namespace quda
diff --git a/lib/dslash_wilson_clover_hasenbusch_twist_preconditioned.in.cu b/lib/dslash_wilson_clover_hasenbusch_twist_preconditioned.in.cu
new file mode 100644
index 0000000000..37230d6a66
--- /dev/null
+++ b/lib/dslash_wilson_clover_hasenbusch_twist_preconditioned.in.cu
@@ -0,0 +1,30 @@
+#include <dslash_wilson_clover_hasenbusch_twist_preconditioned.hpp>
+
+namespace quda
+{
+
+  constexpr QudaPrecision precision = QUDA_@QUDA_DSLASH_PREC@_PRECISION;
+  constexpr int nColor = @QUDA_DSLASH_NCOLOR@;
+  constexpr int reconI = @QUDA_DSLASH_RECONI@;
+  constexpr bool distance_pc = @QUDA_DSLASH_DISTANCE@;
+
+  typedef @QUDA_DSLASH_DDARG@ DDArg;
+  typedef precision_type_mapper<precision>::type Float;
+
+
+
+  template struct WilsonCloverHasenbuschTwistPCNoClovInvApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>;
+  
+  template WilsonCloverHasenbuschTwistPCNoClovInvApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>::WilsonCloverHasenbuschTwistPCNoClovInvApply(cvector_ref<ColorSpinorField> &out,
+                                                cvector_ref<const ColorSpinorField> &in,
+                                                cvector_ref<const ColorSpinorField> &x, const GaugeField &U,
+                                                const CloverField &A, double a, double b, int parity, bool dagger,
+                                                const int *comm_override, DistanceType<distance_pc>, TimeProfile &profile);
+
+  template struct WilsonCloverHasenbuschTwistPCClovInvApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>;
+
+  template WilsonCloverHasenbuschTwistPCClovInvApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>::WilsonCloverHasenbuschTwistPCClovInvApply(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+                                              cvector_ref<const ColorSpinorField> &x, const GaugeField &U,
+                                              const CloverField &A, double kappa, double mu, int parity, bool dagger,
+                                              const int *comm_override, DistanceType<distance_pc>, TimeProfile &profile);
+} // namespace quda
diff --git a/lib/dslash_wilson_clover_hasenbusch_twist_preconditioned_ClovInv.in.cu b/lib/dslash_wilson_clover_hasenbusch_twist_preconditioned_ClovInv.in.cu
new file mode 100644
index 0000000000..91e927859f
--- /dev/null
+++ b/lib/dslash_wilson_clover_hasenbusch_twist_preconditioned_ClovInv.in.cu
@@ -0,0 +1,21 @@
+#include <dslash_wilson_clover_hasenbusch_twist_preconditioned.hpp>
+
+namespace quda
+{
+
+  constexpr QudaPrecision precision = QUDA_@QUDA_DSLASH_PREC@_PRECISION;
+  constexpr int nColor = @QUDA_DSLASH_NCOLOR@;
+  constexpr int reconI = @QUDA_DSLASH_RECONI@;
+  constexpr bool distance_pc = @QUDA_DSLASH_DISTANCE@;
+
+  typedef @QUDA_DSLASH_DDARG@ DDArg;
+  typedef precision_type_mapper<precision>::type Float;
+
+
+  template struct WilsonCloverHasenbuschTwistPCClovInvApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>;
+
+  template WilsonCloverHasenbuschTwistPCClovInvApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>::WilsonCloverHasenbuschTwistPCClovInvApply(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+                                              cvector_ref<const ColorSpinorField> &x, const GaugeField &U,
+                                              const CloverField &A, double kappa, double mu, int parity, bool dagger,
+                                              const int *comm_override, DistanceType<distance_pc>, TimeProfile &profile);
+} // namespace quda
diff --git a/lib/dslash_wilson_clover_hasenbusch_twist_preconditioned_NoClovInv.in.cu b/lib/dslash_wilson_clover_hasenbusch_twist_preconditioned_NoClovInv.in.cu
new file mode 100644
index 0000000000..0079450b3f
--- /dev/null
+++ b/lib/dslash_wilson_clover_hasenbusch_twist_preconditioned_NoClovInv.in.cu
@@ -0,0 +1,24 @@
+#include <dslash_wilson_clover_hasenbusch_twist_preconditioned.hpp>
+
+namespace quda
+{
+
+  constexpr QudaPrecision precision = QUDA_@QUDA_DSLASH_PREC@_PRECISION;
+  constexpr int nColor = @QUDA_DSLASH_NCOLOR@;
+  constexpr int reconI = @QUDA_DSLASH_RECONI@;
+  constexpr bool distance_pc = @QUDA_DSLASH_DISTANCE@;
+
+  typedef @QUDA_DSLASH_DDARG@ DDArg;
+  typedef precision_type_mapper<precision>::type Float;
+
+
+
+  template struct WilsonCloverHasenbuschTwistPCNoClovInvApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>;
+  
+  template WilsonCloverHasenbuschTwistPCNoClovInvApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>::WilsonCloverHasenbuschTwistPCNoClovInvApply(cvector_ref<ColorSpinorField> &out,
+                                                cvector_ref<const ColorSpinorField> &in,
+                                                cvector_ref<const ColorSpinorField> &x, const GaugeField &U,
+                                                const CloverField &A, double a, double b, int parity, bool dagger,
+                                                const int *comm_override, DistanceType<distance_pc>, TimeProfile &profile);
+
+} // namespace quda
diff --git a/lib/laplace.cu b/lib/laplace.cu
index 870371b5cb..c7cc21a872 100644
--- a/lib/laplace.cu
+++ b/lib/laplace.cu
@@ -1,178 +1,10 @@
-#include <dslash.h>
-#include <worker.h>
-#include <dslash_helper.cuh>
-#include <color_spinor_field_order.h>
-#include <gauge_field_order.h>
-#include <color_spinor.h>
-#include <dslash_helper.cuh>
-#include <index_helper.cuh>
-#include <gauge_field.h>
-#include <uint_to_char.h>
-
-#include <dslash_policy.hpp>
-#include <kernels/laplace.cuh>
-
-/**
-   This is the laplacian derivative based on the basic gauged differential operator
-*/
+#define SIGNATURE_ONLY
+#include <laplace.hpp>
+#undef SIGNATURE_ONLY
 
 namespace quda
 {
 
-  template <typename Arg> class Laplace : public Dslash<laplace, Arg>
-  {
-    using Dslash = Dslash<laplace, Arg>;
-    using Dslash::arg;
-    using Dslash::halo;
-    using Dslash::in;
-
-  public:
-    Laplace(Arg &arg, cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
-            const ColorSpinorField &halo) :
-      Dslash(arg, out, in, halo)
-    {
-    }
-
-    void apply(const qudaStream_t &stream) override
-    {
-      TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
-      Dslash::setParam(tp);
-
-      // operator is Hermitian so do not instantiate dagger
-      if (arg.nParity == 1) {
-        if (arg.xpay)
-          Dslash::template instantiate<packStaggeredShmem, 1, false, true>(tp, stream);
-        else
-          Dslash::template instantiate<packStaggeredShmem, 1, false, false>(tp, stream);
-      } else if (arg.nParity == 2) {
-        if (arg.xpay)
-          Dslash::template instantiate<packStaggeredShmem, 2, false, true>(tp, stream);
-        else
-          Dslash::template instantiate<packStaggeredShmem, 2, false, false>(tp, stream);
-      }
-    }
-
-    long long flops() const override
-    {
-      int mv_flops = (8 * in.Ncolor() - 2) * in.Ncolor(); // SU(3) matrix-vector flops
-      int ghost_flops = (in.Nspin() * mv_flops + 2 * in.Ncolor() * in.Nspin());
-      int xpay_flops = 2 * 2 * in.Ncolor() * in.Nspin(); // multiply and add per real component
-      int num_dir = (arg.dir == 4 ? 2 * 4 : 2 * 3);      // 3D or 4D operator
-
-      long long flops_ = 0;
-
-      switch (arg.kernel_type) {
-      case EXTERIOR_KERNEL_X:
-      case EXTERIOR_KERNEL_Y:
-      case EXTERIOR_KERNEL_Z:
-      case EXTERIOR_KERNEL_T:
-        flops_ = (ghost_flops + (arg.xpay ? xpay_flops : xpay_flops / 2)) * 2 * halo.GhostFace()[arg.kernel_type];
-        break;
-      case EXTERIOR_KERNEL_ALL: {
-        long long ghost_sites
-          = 2 * (halo.GhostFace()[0] + halo.GhostFace()[1] + halo.GhostFace()[2] + halo.GhostFace()[3]);
-        flops_ = (ghost_flops + (arg.xpay ? xpay_flops : xpay_flops / 2)) * ghost_sites;
-        break;
-      }
-      case INTERIOR_KERNEL:
-      case UBER_KERNEL:
-      case KERNEL_POLICY: {
-        long long sites = halo.Volume();
-        flops_ = (num_dir * in.Nspin() * mv_flops +                  // SU(3) matrix-vector multiplies
-                  ((num_dir - 1) * 2 * in.Ncolor() * in.Nspin()))
-          * sites; // accumulation
-        if (arg.xpay) flops_ += xpay_flops * sites;
-
-        if (arg.kernel_type == KERNEL_POLICY) break;
-        // now correct for flops done by exterior kernel
-        long long ghost_sites = 0;
-        for (int d = 0; d < 4; d++)
-          if (arg.commDim[d]) ghost_sites += 2 * halo.GhostFace()[d];
-        flops_ -= ghost_flops * ghost_sites;
-
-        break;
-      }
-      }
-
-      return flops_;
-    }
-
-    virtual long long bytes() const override
-    {
-      int gauge_bytes = arg.reconstruct * in.Precision();
-      int spinor_bytes = 2 * in.Ncolor() * in.Nspin() * in.Precision() + (isFixed<typename Arg::Float>::value ? sizeof(float) : 0);
-      int proj_spinor_bytes = in.Nspin() == 4 ? spinor_bytes / 2 : spinor_bytes;
-      int ghost_bytes = (proj_spinor_bytes + gauge_bytes) + 2 * spinor_bytes; // 2 since we have to load the partial
-      int num_dir = (arg.dir == 4 ? 2 * 4 : 2 * 3);                           // 3D or 4D operator
-
-      long long bytes_ = 0;
-
-      switch (arg.kernel_type) {
-      case EXTERIOR_KERNEL_X:
-      case EXTERIOR_KERNEL_Y:
-      case EXTERIOR_KERNEL_Z:
-      case EXTERIOR_KERNEL_T: bytes_ = ghost_bytes * 2 * halo.GhostFace()[arg.kernel_type]; break;
-      case EXTERIOR_KERNEL_ALL: {
-        long long ghost_sites
-          = 2 * (halo.GhostFace()[0] + halo.GhostFace()[1] + halo.GhostFace()[2] + halo.GhostFace()[3]);
-        bytes_ = ghost_bytes * ghost_sites;
-        break;
-      }
-      case INTERIOR_KERNEL:
-      case UBER_KERNEL:
-      case KERNEL_POLICY: {
-        long long sites = halo.Volume();
-        bytes_ = (num_dir * gauge_bytes + ((num_dir - 2) * spinor_bytes + 2 * proj_spinor_bytes) + spinor_bytes) * sites;
-        if (arg.xpay) bytes_ += spinor_bytes;
-	
-        if (arg.kernel_type == KERNEL_POLICY) break;
-        // now correct for bytes done by exterior kernel
-        long long ghost_sites = 0;
-        for (int d = 0; d < 4; d++)
-          if (arg.commDim[d]) ghost_sites += 2 * halo.GhostFace()[d];
-        bytes_ -= ghost_bytes * ghost_sites;
-	
-        break;
-      }
-      }
-      return bytes_;
-    }
-    
-    TuneKey tuneKey() const override
-    { // add laplace transverse dir to the key
-      auto key = Dslash::tuneKey();
-      strcat(key.aux, ",laplace=");
-      u32toa(key.aux + strlen(key.aux), arg.dir);
-      return key;
-    }
-  };
-
-  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon> struct LaplaceApply {
-
-    LaplaceApply(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
-                 cvector_ref<const ColorSpinorField> &x, const GaugeField &U, int dir, double a, double b, int parity,
-                 bool dagger, const int *comm_override, TimeProfile &profile)
-    {
-      constexpr int nDim = 4;
-      auto halo = ColorSpinorField::create_comms_batch(in);
-      if (in.Nspin() == 1) {
-        constexpr int nSpin = 1;
-        LaplaceArg<Float, nSpin, nColor, nDim, DDArg, recon> arg(out, in, halo, U, dir, a, b, x, parity, dagger,
-                                                                 comm_override);
-        Laplace<decltype(arg)> laplace(arg, out, in, halo);
-        dslash::DslashPolicyTune<decltype(laplace)> policy(laplace, in, halo, profile);
-      } else if (in.Nspin() == 4) {
-        constexpr int nSpin = 4;
-        LaplaceArg<Float, nSpin, nColor, nDim, DDArg, recon> arg(out, in, halo, U, dir, a, b, x, parity, dagger,
-                                                                 comm_override);
-        Laplace<decltype(arg)> laplace(arg, out, in, halo);
-        dslash::DslashPolicyTune<decltype(laplace)> policy(laplace, in, halo, profile);
-      } else {
-        errorQuda("Unsupported nSpin= %d", in.Nspin());
-      }
-    }
-  };
-
   // Apply the Laplace operator
   // out(x) = M*in = - a*\sum_mu U_{-\mu}(x)in(x+mu) + U^\dagger_mu(x-mu)in(x-mu) + b*in(x)
   // Omits direction 'dir' from the operator.
@@ -181,7 +13,8 @@ namespace quda
                     const int *comm_override, TimeProfile &profile)
   {
     if constexpr (is_enabled<QUDA_LAPLACE_DSLASH>()) {
-      instantiate<LaplaceApply>(out, in, x, U, dir, a, b, parity, dagger, comm_override, profile);
+      auto dummy = DistanceType<false>();
+      instantiate<LaplaceApply>(out, in, x, U, dir, a, b, parity, dagger, comm_override, dummy, profile);
     } else {
       errorQuda("Laplace operator has not been enabled");
     }
diff --git a/lib/laplace.hpp b/lib/laplace.hpp
new file mode 100644
index 0000000000..3f3e58b9e9
--- /dev/null
+++ b/lib/laplace.hpp
@@ -0,0 +1,183 @@
+#include <dslash.h>
+#include <worker.h>
+#include <dslash_helper.cuh>
+#include <color_spinor_field_order.h>
+#include <gauge_field_order.h>
+#include <color_spinor.h>
+#include <dslash_helper.cuh>
+#include <index_helper.cuh>
+#include <gauge_field.h>
+#include <uint_to_char.h>
+
+#include <dslash_policy.hpp>
+#include <kernels/laplace.cuh>
+
+/**
+   This is the laplacian derivative based on the basic gauged differential operator
+*/
+
+namespace quda
+{
+
+  template <typename Arg> class Laplace : public Dslash<laplace, Arg>
+  {
+    using Dslash = Dslash<laplace, Arg>;
+    using Dslash::arg;
+    using Dslash::halo;
+    using Dslash::in;
+
+  public:
+    Laplace(Arg &arg, cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+            const ColorSpinorField &halo) :
+      Dslash(arg, out, in, halo)
+    {
+    }
+
+    void apply(const qudaStream_t &stream) override
+    {
+      TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
+      Dslash::setParam(tp);
+
+      // operator is Hermitian so do not instantiate dagger
+      if (arg.nParity == 1) {
+        if (arg.xpay)
+          Dslash::template instantiate<packStaggeredShmem, 1, false, true>(tp, stream);
+        else
+          Dslash::template instantiate<packStaggeredShmem, 1, false, false>(tp, stream);
+      } else if (arg.nParity == 2) {
+        if (arg.xpay)
+          Dslash::template instantiate<packStaggeredShmem, 2, false, true>(tp, stream);
+        else
+          Dslash::template instantiate<packStaggeredShmem, 2, false, false>(tp, stream);
+      }
+    }
+
+    long long flops() const override
+    {
+      int mv_flops = (8 * in.Ncolor() - 2) * in.Ncolor(); // SU(3) matrix-vector flops
+      int ghost_flops = (in.Nspin() * mv_flops + 2 * in.Ncolor() * in.Nspin());
+      int xpay_flops = 2 * 2 * in.Ncolor() * in.Nspin(); // multiply and add per real component
+      int num_dir = (arg.dir == 4 ? 2 * 4 : 2 * 3);      // 3D or 4D operator
+
+      long long flops_ = 0;
+
+      switch (arg.kernel_type) {
+      case EXTERIOR_KERNEL_X:
+      case EXTERIOR_KERNEL_Y:
+      case EXTERIOR_KERNEL_Z:
+      case EXTERIOR_KERNEL_T:
+        flops_ = (ghost_flops + (arg.xpay ? xpay_flops : xpay_flops / 2)) * 2 * halo.GhostFace()[arg.kernel_type];
+        break;
+      case EXTERIOR_KERNEL_ALL: {
+        long long ghost_sites
+          = 2 * (halo.GhostFace()[0] + halo.GhostFace()[1] + halo.GhostFace()[2] + halo.GhostFace()[3]);
+        flops_ = (ghost_flops + (arg.xpay ? xpay_flops : xpay_flops / 2)) * ghost_sites;
+        break;
+      }
+      case INTERIOR_KERNEL:
+      case UBER_KERNEL:
+      case KERNEL_POLICY: {
+        long long sites = halo.Volume();
+        flops_ = (num_dir * in.Nspin() * mv_flops + // SU(3) matrix-vector multiplies
+                  ((num_dir - 1) * 2 * in.Ncolor() * in.Nspin()))
+          * sites; // accumulation
+        if (arg.xpay) flops_ += xpay_flops * sites;
+
+        if (arg.kernel_type == KERNEL_POLICY) break;
+        // now correct for flops done by exterior kernel
+        long long ghost_sites = 0;
+        for (int d = 0; d < 4; d++)
+          if (arg.commDim[d]) ghost_sites += 2 * halo.GhostFace()[d];
+        flops_ -= ghost_flops * ghost_sites;
+
+        break;
+      }
+      }
+
+      return flops_;
+    }
+
+    virtual long long bytes() const override
+    {
+      int gauge_bytes = arg.reconstruct * in.Precision();
+      int spinor_bytes
+        = 2 * in.Ncolor() * in.Nspin() * in.Precision() + (isFixed<typename Arg::Float>::value ? sizeof(float) : 0);
+      int proj_spinor_bytes = in.Nspin() == 4 ? spinor_bytes / 2 : spinor_bytes;
+      int ghost_bytes = (proj_spinor_bytes + gauge_bytes) + 2 * spinor_bytes; // 2 since we have to load the partial
+      int num_dir = (arg.dir == 4 ? 2 * 4 : 2 * 3);                           // 3D or 4D operator
+
+      long long bytes_ = 0;
+
+      switch (arg.kernel_type) {
+      case EXTERIOR_KERNEL_X:
+      case EXTERIOR_KERNEL_Y:
+      case EXTERIOR_KERNEL_Z:
+      case EXTERIOR_KERNEL_T: bytes_ = ghost_bytes * 2 * halo.GhostFace()[arg.kernel_type]; break;
+      case EXTERIOR_KERNEL_ALL: {
+        long long ghost_sites
+          = 2 * (halo.GhostFace()[0] + halo.GhostFace()[1] + halo.GhostFace()[2] + halo.GhostFace()[3]);
+        bytes_ = ghost_bytes * ghost_sites;
+        break;
+      }
+      case INTERIOR_KERNEL:
+      case UBER_KERNEL:
+      case KERNEL_POLICY: {
+        long long sites = halo.Volume();
+        bytes_ = (num_dir * gauge_bytes + ((num_dir - 2) * spinor_bytes + 2 * proj_spinor_bytes) + spinor_bytes) * sites;
+        if (arg.xpay) bytes_ += spinor_bytes;
+
+        if (arg.kernel_type == KERNEL_POLICY) break;
+        // now correct for bytes done by exterior kernel
+        long long ghost_sites = 0;
+        for (int d = 0; d < 4; d++)
+          if (arg.commDim[d]) ghost_sites += 2 * halo.GhostFace()[d];
+        bytes_ -= ghost_bytes * ghost_sites;
+
+        break;
+      }
+      }
+      return bytes_;
+    }
+
+    TuneKey tuneKey() const override
+    { // add laplace transverse dir to the key
+      auto key = Dslash::tuneKey();
+      strcat(key.aux, ",laplace=");
+      u32toa(key.aux + strlen(key.aux), arg.dir);
+      return key;
+    }
+  };
+
+  template <bool distance_pc> struct DistanceType {
+  };
+
+  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon> struct LaplaceApply {
+    template <bool distance_pc>
+    LaplaceApply(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+                 cvector_ref<const ColorSpinorField> &x, const GaugeField &U, int dir, double a, double b, int parity,
+                 bool dagger, const int *comm_override, DistanceType<distance_pc>, TimeProfile &profile)
+#ifdef SIGNATURE_ONLY // Used to hide from the compiler the implementation of the function
+      ;
+#else
+    {
+      constexpr int nDim = 4;
+      auto halo = ColorSpinorField::create_comms_batch(in);
+      if (in.Nspin() == 1) {
+        constexpr int nSpin = 1;
+        LaplaceArg<Float, nSpin, nColor, nDim, DDArg, recon> arg(out, in, halo, U, dir, a, b, x, parity, dagger,
+                                                                 comm_override);
+        Laplace<decltype(arg)> laplace(arg, out, in, halo);
+        dslash::DslashPolicyTune<decltype(laplace)> policy(laplace, in, halo, profile);
+      } else if (in.Nspin() == 4) {
+        constexpr int nSpin = 4;
+        LaplaceArg<Float, nSpin, nColor, nDim, DDArg, recon> arg(out, in, halo, U, dir, a, b, x, parity, dagger,
+                                                                 comm_override);
+        Laplace<decltype(arg)> laplace(arg, out, in, halo);
+        dslash::DslashPolicyTune<decltype(laplace)> policy(laplace, in, halo, profile);
+      } else {
+        errorQuda("Unsupported nSpin= %d", in.Nspin());
+      }
+    }
+#endif
+  };
+} // namespace quda
diff --git a/lib/laplace.in.cu b/lib/laplace.in.cu
new file mode 100644
index 0000000000..bb58818ab5
--- /dev/null
+++ b/lib/laplace.in.cu
@@ -0,0 +1,21 @@
+#include <laplace.hpp>
+
+namespace quda
+{
+
+  constexpr QudaPrecision precision = QUDA_@QUDA_DSLASH_PREC@_PRECISION;
+  constexpr int nColor = @QUDA_DSLASH_NCOLOR@;
+  constexpr int reconI = @QUDA_DSLASH_RECONI@;
+  constexpr bool distance_pc = @QUDA_DSLASH_DISTANCE@;
+
+  typedef @QUDA_DSLASH_DDARG@ DDArg;
+  typedef precision_type_mapper<precision>::type Float;
+
+  template struct LaplaceApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>;
+
+  template LaplaceApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>::LaplaceApply(
+    cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in, cvector_ref<const ColorSpinorField> &x,
+    const GaugeField &U, int dir, double a, double b, int parity, bool dagger, const int *comm_override,
+    DistanceType<distance_pc>, TimeProfile &profile);
+
+} // namespace quda

From 21d81e6a5901e4dd0e949e23232f82c2ad4b182d Mon Sep 17 00:00:00 2001
From: Ferenc Pittler <pittler@hiskp.uni-bonn.de>
Date: Thu, 7 Nov 2024 10:41:44 +0100
Subject: [PATCH 102/108] remove in file no longer needed

---
 ...over_hasenbusch_twist_preconditioned.in.cu | 30 -------------------
 1 file changed, 30 deletions(-)
 delete mode 100644 lib/dslash_wilson_clover_hasenbusch_twist_preconditioned.in.cu

diff --git a/lib/dslash_wilson_clover_hasenbusch_twist_preconditioned.in.cu b/lib/dslash_wilson_clover_hasenbusch_twist_preconditioned.in.cu
deleted file mode 100644
index 37230d6a66..0000000000
--- a/lib/dslash_wilson_clover_hasenbusch_twist_preconditioned.in.cu
+++ /dev/null
@@ -1,30 +0,0 @@
-#include <dslash_wilson_clover_hasenbusch_twist_preconditioned.hpp>
-
-namespace quda
-{
-
-  constexpr QudaPrecision precision = QUDA_@QUDA_DSLASH_PREC@_PRECISION;
-  constexpr int nColor = @QUDA_DSLASH_NCOLOR@;
-  constexpr int reconI = @QUDA_DSLASH_RECONI@;
-  constexpr bool distance_pc = @QUDA_DSLASH_DISTANCE@;
-
-  typedef @QUDA_DSLASH_DDARG@ DDArg;
-  typedef precision_type_mapper<precision>::type Float;
-
-
-
-  template struct WilsonCloverHasenbuschTwistPCNoClovInvApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>;
-  
-  template WilsonCloverHasenbuschTwistPCNoClovInvApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>::WilsonCloverHasenbuschTwistPCNoClovInvApply(cvector_ref<ColorSpinorField> &out,
-                                                cvector_ref<const ColorSpinorField> &in,
-                                                cvector_ref<const ColorSpinorField> &x, const GaugeField &U,
-                                                const CloverField &A, double a, double b, int parity, bool dagger,
-                                                const int *comm_override, DistanceType<distance_pc>, TimeProfile &profile);
-
-  template struct WilsonCloverHasenbuschTwistPCClovInvApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>;
-
-  template WilsonCloverHasenbuschTwistPCClovInvApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>::WilsonCloverHasenbuschTwistPCClovInvApply(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
-                                              cvector_ref<const ColorSpinorField> &x, const GaugeField &U,
-                                              const CloverField &A, double kappa, double mu, int parity, bool dagger,
-                                              const int *comm_override, DistanceType<distance_pc>, TimeProfile &profile);
-} // namespace quda

From 907ee8fe3928a7be59e66cc114fec2b554813d59 Mon Sep 17 00:00:00 2001
From: Ferenc Pittler <pittler@hiskp.uni-bonn.de>
Date: Thu, 7 Nov 2024 11:05:54 +0100
Subject: [PATCH 103/108] correcting preconditioning

---
 lib/CMakeLists.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt
index 4198ce185f..812a7200fe 100644
--- a/lib/CMakeLists.txt
+++ b/lib/CMakeLists.txt
@@ -199,7 +199,8 @@ list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_wilson")
 list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_wilson_clover")
 list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_wilson_clover_preconditioned")
 list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_wilson_clover_hasenbusch_twist")
-list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_wilson_clover_hasenbusch_twist_preconditioned")
+list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_wilson_clover_hasenbusch_twist_preconditioned_NoClovInv")
+list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_wilson_clover_hasenbusch_twist_preconditioned_ClovInv")
 list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_twisted_mass")
 list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_twisted_clover")
 list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_twisted_clover_preconditioned")

From 8539aa3ab7f45838b773334f02e26f23a5336c01 Mon Sep 17 00:00:00 2001
From: Ferenc Pittler <pittler@hiskp.uni-bonn.de>
Date: Thu, 7 Nov 2024 11:12:38 +0100
Subject: [PATCH 104/108] correcting typo

---
 lib/dslash_domain_wall_4d_fused_m5.in.cu | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/dslash_domain_wall_4d_fused_m5.in.cu b/lib/dslash_domain_wall_4d_fused_m5.in.cu
index c376eb0618..5517e6dac8 100644
--- a/lib/dslash_domain_wall_4d_fused_m5.in.cu
+++ b/lib/dslash_domain_wall_4d_fused_m5.in.cu
@@ -7,7 +7,7 @@
 namespace quda
 {
 
-  constexpr QudaPrecision precision = QUDA_ @QUDA_DSLASH_PREC@_PRECISION;
+  constexpr QudaPrecision precision = QUDA_@QUDA_DSLASH_PREC@_PRECISION;
   constexpr int nColor = @QUDA_DSLASH_NCOLOR@;
   constexpr int reconI = @QUDA_DSLASH_RECONI@;
   constexpr bool distance_pc = @QUDA_DSLASH_DISTANCE@;

From a73e6a1756a8a5e319804a77af44565eba29831b Mon Sep 17 00:00:00 2001
From: Ferenc Pittler <pittler@hiskp.uni-bonn.de>
Date: Thu, 7 Nov 2024 13:40:54 +0100
Subject: [PATCH 105/108] correcting typo

---
 lib/dslash_domain_wall_4d_fused_m5.in.cu | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/lib/dslash_domain_wall_4d_fused_m5.in.cu b/lib/dslash_domain_wall_4d_fused_m5.in.cu
index 5517e6dac8..91fa23e122 100644
--- a/lib/dslash_domain_wall_4d_fused_m5.in.cu
+++ b/lib/dslash_domain_wall_4d_fused_m5.in.cu
@@ -1,8 +1,4 @@
-#include <dslash_wilson_clover_preconditioned.hpp>
-
-/**
-   This is the Wilson-clover linear operator
-*/
+#include <dslash_domain_wall_4d_fused_m5.hpp>
 
 namespace quda
 {

From a0123fc5ef9002ad41938a7aa6c90b75547082ef Mon Sep 17 00:00:00 2001
From: Ferenc Pittler <pittler@hiskp.uni-bonn.de>
Date: Thu, 7 Nov 2024 13:55:14 +0100
Subject: [PATCH 106/108] correcting typo

---
 lib/dslash_domain_wall_4d_fused_m5.in.cu | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/dslash_domain_wall_4d_fused_m5.in.cu b/lib/dslash_domain_wall_4d_fused_m5.in.cu
index 91fa23e122..d91951b216 100644
--- a/lib/dslash_domain_wall_4d_fused_m5.in.cu
+++ b/lib/dslash_domain_wall_4d_fused_m5.in.cu
@@ -15,7 +15,7 @@ namespace quda
 
   template struct DomainWall4DApplyFusedM5<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>;
 
-  template DomainWall4DApplyFusedM5<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>::DomainWall4DFusedM5Apply(
+  template DomainWall4DApplyFusedM5<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>::DomainWall4DApplyFusedM5(
     cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in, cvector_ref<const ColorSpinorField> &x,
     const GaugeField &U, cvector_ref<ColorSpinorField> &y, const Complex *b_5, const Complex *c_5, double a, double m_5,
     int parity, bool dagger, const int *comm_override, double m_f, Dslash5TypeList<dslash5_type>,

From 8ffb905794030f008bfeec0af2d80e78208f74b5 Mon Sep 17 00:00:00 2001
From: Ferenc Pittler <pittler@hiskp.uni-bonn.de>
Date: Thu, 7 Nov 2024 14:17:07 +0100
Subject: [PATCH 107/108] typo corrected

---
 lib/dslash_domain_wall_4d_fused_m5.in.cu | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/dslash_domain_wall_4d_fused_m5.in.cu b/lib/dslash_domain_wall_4d_fused_m5.in.cu
index d91951b216..df99e8f863 100644
--- a/lib/dslash_domain_wall_4d_fused_m5.in.cu
+++ b/lib/dslash_domain_wall_4d_fused_m5.in.cu
@@ -18,7 +18,7 @@ namespace quda
   template DomainWall4DApplyFusedM5<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>::DomainWall4DApplyFusedM5(
     cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in, cvector_ref<const ColorSpinorField> &x,
     const GaugeField &U, cvector_ref<ColorSpinorField> &y, const Complex *b_5, const Complex *c_5, double a, double m_5,
-    int parity, bool dagger, const int *comm_override, double m_f, Dslash5TypeList<dslash5_type>,
-    DistanceType<distance_pc>, TimeProfile &profile);
+    int parity, bool dagger, const int *comm_override, double m_f, DistanceType<distance_pc>, Dslash5TypeList<dslash5_type>,
+    TimeProfile &profile);
 
 } // namespace quda

From 9bac7f7c66c304a5dded1a1139fc62b4d39059d2 Mon Sep 17 00:00:00 2001
From: Ferenc Pittler <pittler@hiskp.uni-bonn.de>
Date: Fri, 8 Nov 2024 08:23:07 +0100
Subject: [PATCH 108/108] separate compilation for twisted mass preconditioned

---
 lib/CMakeLists.txt                           |   1 +
 lib/dslash_twisted_mass_preconditioned.cu    |  96 +----------------
 lib/dslash_twisted_mass_preconditioned.hpp   | 106 +++++++++++++++++++
 lib/dslash_twisted_mass_preconditioned.in.cu |  21 ++++
 4 files changed, 133 insertions(+), 91 deletions(-)
 create mode 100644 lib/dslash_twisted_mass_preconditioned.hpp
 create mode 100644 lib/dslash_twisted_mass_preconditioned.in.cu

diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt
index 812a7200fe..5905978e33 100644
--- a/lib/CMakeLists.txt
+++ b/lib/CMakeLists.txt
@@ -202,6 +202,7 @@ list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_wilson_clover_hasenbusch_twist")
 list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_wilson_clover_hasenbusch_twist_preconditioned_NoClovInv")
 list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_wilson_clover_hasenbusch_twist_preconditioned_ClovInv")
 list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_twisted_mass")
+list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_twisted_mass_preconditioned")
 list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_twisted_clover")
 list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_twisted_clover_preconditioned")
 list(APPEND QUDA_DSLASH_FILENAME_LIST "dslash_ndeg_twisted_mass")
diff --git a/lib/dslash_twisted_mass_preconditioned.cu b/lib/dslash_twisted_mass_preconditioned.cu
index fe0f0c7a27..00ee3aed24 100644
--- a/lib/dslash_twisted_mass_preconditioned.cu
+++ b/lib/dslash_twisted_mass_preconditioned.cu
@@ -1,10 +1,6 @@
-#include <gauge_field.h>
-#include <color_spinor_field.h>
-#include <dslash.h>
-#include <worker.h>
-
-#include <dslash_policy.hpp>
-#include <kernels/dslash_twisted_mass_preconditioned.cuh>
+#define SIGNATURE_ONLY
+#include <dslash_twisted_mass_preconditioned.hpp>
+#undef SIGNATURE_ONLY
 
 /**
    This is the preconditioned gauged twisted-mass operator
@@ -13,89 +9,6 @@
 namespace quda
 {
 
-  // trait to ensure we don't instantiate asymmetric & xpay
-  template <bool symmetric> constexpr bool xpay_() { return true; }
-  template <> constexpr bool xpay_<true>() { return false; }
-
-  // trait to ensure we don't instantiate asymmetric & !dagger
-  template <bool symmetric> constexpr bool not_dagger_() { return false; }
-  template <> constexpr bool not_dagger_<true>() { return true; }
-
-  template <typename Arg> class TwistedMassPreconditioned : public Dslash<twistedMassPreconditioned, Arg>
-  {
-    using Dslash = Dslash<twistedMassPreconditioned, Arg>;
-    using Dslash::arg;
-    using Dslash::in;
-
-  public:
-    TwistedMassPreconditioned(Arg &arg, cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
-                              const ColorSpinorField &halo) :
-      Dslash(arg, out, in, halo)
-    {
-    }
-
-    void apply(const qudaStream_t &stream)
-    {
-      TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
-      Dslash::setParam(tp);
-      if (arg.asymmetric && !arg.dagger) errorQuda("asymmetric operator only defined for dagger");
-      if (arg.asymmetric && arg.xpay) errorQuda("asymmetric operator not defined for xpay");
-      if (arg.nParity != 1) errorQuda("Preconditioned twisted-mass operator not defined nParity=%d", arg.nParity);
-
-      if (arg.dagger) {
-        if (arg.xpay)
-          Dslash::template instantiate<packShmem, 1, true, xpay_<Arg::asymmetric>()>(tp, stream);
-        else
-          Dslash::template instantiate<packShmem, 1, true, false>(tp, stream);
-      } else {
-        if (arg.xpay)
-          Dslash::template instantiate<packShmem, 1, not_dagger_<Arg::asymmetric>(), xpay_<Arg::asymmetric>()>(tp, stream);
-        else
-          Dslash::template instantiate<packShmem, 1, not_dagger_<Arg::asymmetric>(), false>(tp, stream);
-      }
-    }
-
-    long long flops() const
-    {
-      long long flops = Dslash::flops();
-      switch (arg.kernel_type) {
-      case INTERIOR_KERNEL:
-      case UBER_KERNEL:
-      case KERNEL_POLICY:
-        flops += in.size() * 2 * in.Ncolor() * 4 * 2 * in.Volume(); // complex * Nc * Ns * fma * vol
-        break;
-      default: break;
-      }
-      return flops;
-    }
-  };
-
-  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon>
-  struct TwistedMassPreconditionedApply {
-
-    TwistedMassPreconditionedApply(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
-                                   cvector_ref<const ColorSpinorField> &x, const GaugeField &U, double a, double b,
-                                   bool xpay, int parity, bool dagger, bool asymmetric, const int *comm_override,
-                                   TimeProfile &profile)
-    {
-      constexpr int nDim = 4;
-      auto halo = ColorSpinorField::create_comms_batch(in);
-      if (asymmetric) {
-        TwistedMassArg<Float, nColor, nDim, DDArg, recon, true> arg(out, in, halo, U, a, b, xpay, x, parity, dagger,
-                                                                    comm_override);
-        TwistedMassPreconditioned<decltype(arg)> twisted(arg, out, in, halo);
-
-        dslash::DslashPolicyTune<decltype(twisted)> policy(twisted, in, halo, profile);
-      } else {
-        TwistedMassArg<Float, nColor, nDim, DDArg, recon, false> arg(out, in, halo, U, a, b, xpay, x, parity, dagger,
-                                                                     comm_override);
-        TwistedMassPreconditioned<decltype(arg)> twisted(arg, out, in, halo);
-
-        dslash::DslashPolicyTune<decltype(twisted)> policy(twisted, in, halo, profile);
-      }
-    }
-  };
-
   /*
     Apply the preconditioned twisted-mass Dslash operator
 
@@ -106,8 +19,9 @@ namespace quda
                                       cvector_ref<const ColorSpinorField> &x, int parity, bool dagger, bool asymmetric,
                                       const int *comm_override, TimeProfile &profile)
   {
+    auto dummy = DistanceType<false>();
     if constexpr (is_enabled<QUDA_TWISTED_MASS_DSLASH>()) {
-      instantiate<TwistedMassPreconditionedApply>(out, in, x, U, a, b, xpay, parity, dagger, asymmetric, comm_override,
+      instantiate<TwistedMassPreconditionedApply>(out, in, x, U, a, b, xpay, parity, dagger, asymmetric, comm_override, dummy,
                                                   profile);
     } else {
       errorQuda("Twisted-mass operator has not been built");
diff --git a/lib/dslash_twisted_mass_preconditioned.hpp b/lib/dslash_twisted_mass_preconditioned.hpp
new file mode 100644
index 0000000000..a354c5113c
--- /dev/null
+++ b/lib/dslash_twisted_mass_preconditioned.hpp
@@ -0,0 +1,106 @@
+#include <gauge_field.h>
+#include <color_spinor_field.h>
+#include <dslash.h>
+#include <worker.h>
+
+#include <dslash_policy.hpp>
+#include <kernels/dslash_twisted_mass_preconditioned.cuh>
+
+/**
+   This is the preconditioned gauged twisted-mass operator
+*/
+
+namespace quda
+{
+
+  // trait to ensure we don't instantiate asymmetric & xpay
+  template <bool symmetric> constexpr bool xpay_() { return true; }
+  template <> constexpr bool xpay_<true>() { return false; }
+
+  // trait to ensure we don't instantiate asymmetric & !dagger
+  template <bool symmetric> constexpr bool not_dagger_() { return false; }
+  template <> constexpr bool not_dagger_<true>() { return true; }
+
+  template <typename Arg> class TwistedMassPreconditioned : public Dslash<twistedMassPreconditioned, Arg>
+  {
+    using Dslash = Dslash<twistedMassPreconditioned, Arg>;
+    using Dslash::arg;
+    using Dslash::in;
+
+  public:
+    TwistedMassPreconditioned(Arg &arg, cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+                              const ColorSpinorField &halo) :
+      Dslash(arg, out, in, halo)
+    {
+    }
+
+    void apply(const qudaStream_t &stream)
+    {
+      TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
+      Dslash::setParam(tp);
+      if (arg.asymmetric && !arg.dagger) errorQuda("asymmetric operator only defined for dagger");
+      if (arg.asymmetric && arg.xpay) errorQuda("asymmetric operator not defined for xpay");
+      if (arg.nParity != 1) errorQuda("Preconditioned twisted-mass operator not defined nParity=%d", arg.nParity);
+
+      if (arg.dagger) {
+        if (arg.xpay)
+          Dslash::template instantiate<packShmem, 1, true, xpay_<Arg::asymmetric>()>(tp, stream);
+        else
+          Dslash::template instantiate<packShmem, 1, true, false>(tp, stream);
+      } else {
+        if (arg.xpay)
+          Dslash::template instantiate<packShmem, 1, not_dagger_<Arg::asymmetric>(), xpay_<Arg::asymmetric>()>(tp, stream);
+        else
+          Dslash::template instantiate<packShmem, 1, not_dagger_<Arg::asymmetric>(), false>(tp, stream);
+      }
+    }
+
+    long long flops() const
+    {
+      long long flops = Dslash::flops();
+      switch (arg.kernel_type) {
+      case INTERIOR_KERNEL:
+      case UBER_KERNEL:
+      case KERNEL_POLICY:
+        flops += in.size() * 2 * in.Ncolor() * 4 * 2 * in.Volume(); // complex * Nc * Ns * fma * vol
+        break;
+      default: break;
+      }
+      return flops;
+    }
+  };
+
+  template <bool distance_pc> struct DistanceType {
+  };
+
+  template <typename Float, int nColor, typename DDArg, QudaReconstructType recon>
+  struct TwistedMassPreconditionedApply {
+    template <bool distance_pc>
+    TwistedMassPreconditionedApply(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+                                   cvector_ref<const ColorSpinorField> &x, const GaugeField &U, double a, double b,
+                                   bool xpay, int parity, bool dagger, bool asymmetric, const int *comm_override,
+                                   DistanceType<distance_pc>, TimeProfile &profile)
+#ifdef SIGNATURE_ONLY
+      ;
+#else
+    {
+      constexpr int nDim = 4;
+      auto halo = ColorSpinorField::create_comms_batch(in);
+      if (asymmetric) {
+        TwistedMassArg<Float, nColor, nDim, DDArg, recon, true> arg(out, in, halo, U, a, b, xpay, x, parity, dagger,
+                                                                    comm_override);
+        TwistedMassPreconditioned<decltype(arg)> twisted(arg, out, in, halo);
+
+        dslash::DslashPolicyTune<decltype(twisted)> policy(twisted, in, halo, profile);
+      } else {
+        TwistedMassArg<Float, nColor, nDim, DDArg, recon, false> arg(out, in, halo, U, a, b, xpay, x, parity, dagger,
+                                                                     comm_override);
+        TwistedMassPreconditioned<decltype(arg)> twisted(arg, out, in, halo);
+
+        dslash::DslashPolicyTune<decltype(twisted)> policy(twisted, in, halo, profile);
+      }
+    }
+#endif
+  };
+
+} // namespace quda
diff --git a/lib/dslash_twisted_mass_preconditioned.in.cu b/lib/dslash_twisted_mass_preconditioned.in.cu
new file mode 100644
index 0000000000..cbcec90d05
--- /dev/null
+++ b/lib/dslash_twisted_mass_preconditioned.in.cu
@@ -0,0 +1,21 @@
+#include <dslash_twisted_mass_preconditioned.hpp>
+
+namespace quda
+{
+
+  constexpr QudaPrecision precision = QUDA_@QUDA_DSLASH_PREC@_PRECISION;
+  constexpr int nColor = @QUDA_DSLASH_NCOLOR@;
+  constexpr int reconI = @QUDA_DSLASH_RECONI@;
+  constexpr bool distance_pc = @QUDA_DSLASH_DISTANCE@;
+
+  typedef @QUDA_DSLASH_DDARG@ DDArg;
+  typedef precision_type_mapper<precision>::type Float;
+	
+
+  template struct TwistedMassPreconditionedApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>;
+
+  template TwistedMassPreconditionedApply<Float, nColor, DDArg, ReconstructWilson::recon[reconI]>::TwistedMassPreconditionedApply(cvector_ref<ColorSpinorField> &out, cvector_ref<const ColorSpinorField> &in,
+                                   cvector_ref<const ColorSpinorField> &x, const GaugeField &U, double a, double b,
+                                   bool xpay, int parity, bool dagger, bool asymmetric, const int *comm_override,
+                                   DistanceType<distance_pc>,TimeProfile &profile);
+} // namespace quda