Check bench dfem

- Add target-based path for libProteusPass
- Link with libproteus

.gitignore

@@ -144,6 +144,8 @@ examples/amgx/sol.gf
 examples/amgx/mesh.*
 examples/amgx/sol.*
 
+examples/dfem/minimal_surface
+
 examples/caliper/ex1
 examples/caliper/ex1p
 examples/caliper/refined.mesh

CMakeLists.txt

@@ -592,6 +592,13 @@ if (MFEM_USE_ENZYME)
     set(ENZYME_INCLUDE_DIRS ${ENZYME_DIR}/include)
 endif()
 
+if (MFEM_USE_PROTEUS)
+    enable_language(C)
+    find_package(proteus REQUIRED PATHS "${PROTEUS_DIR}")
+    message(STATUS "${PROTEUS_DIR}/include")
+    include_directories("${PROTEUS_DIR}/include")
+endif()
+
 # MFEM_TIMER_TYPE
 if (NOT DEFINED MFEM_TIMER_TYPE)
   if (APPLE)
@@ -728,6 +735,16 @@ mfem_add_library(mfem ${SOURCES} ${HEADERS} ${MASTER_HEADERS})
 target_compile_features(mfem PUBLIC cxx_std_${CMAKE_CXX_STANDARD})
 # message(STATUS "TPL_LIBRARIES = ${TPL_LIBRARIES}")
 target_link_libraries(mfem PUBLIC ${TPL_LIBRARIES} ${TPL_TARGETS})
+
+if (MFEM_USE_PROTEUS)
+    add_library(ClangProteusFlags INTERFACE IMPORTED)
+    set_target_properties(ClangProteusFlags PROPERTIES
+        INTERFACE_COMPILE_OPTIONS "-fpass-plugin=$<TARGET_FILE:ProteusPass>"
+    )
+    target_link_libraries(mfem PUBLIC ClangProteusFlags)
+    target_link_libraries(mfem PUBLIC proteus)
+endif()
+
 if (TPL_TARGETS)
   add_dependencies(mfem ${TPL_TARGETS})
 endif()

config/config.hpp

@@ -157,4 +157,20 @@ constexpr real_t operator""_r(unsigned long long v)
 #endif
 #endif // MFEM_USE_MPI not defined
 
+#ifdef NVTX_DBG_HPP
+#include NVTX_DBG_HPP
+#else
+#define db1(...)
+#define dbg(...)
+#define dbl(...)
+#define dba(...)
+#define dbc(...)
+#define NVTX_MARK_FUNCTION
+#define NVTX_MARK_BEGIN(...)
+#define NVTX_MARK_INI(...)
+#define NVTX_MARK_END(...)
+#define NVTX_MARK(...)
+#define NVTX(...)
+#endif
+
 #endif // MFEM_CONFIG_HPP

config/defaults.mk

@@ -522,6 +522,8 @@ GSLIB_LIB = -L$(GSLIB_DIR)/lib -lgs
 
 # CUDA library configuration
 CUDA_OPT =
+# base CUDA install directory, only needed if building with clang+cuda
+CUDA_DIR = /usr/local/cuda/
 CUDA_LIB = -lcusparse -lcublas
 CLANG_CUDA_LIB = -L$(CUDA_DIR)/lib64 -L$(CUDA_DIR)/lib \
  $(XLINKER)-rpath,$(CUDA_DIR)/lib64,-rpath,$(CUDA_DIR)/lib \

examples/CMakeLists.txt

@@ -47,6 +47,7 @@ list(APPEND ALL_EXE_SRCS
   ex39.cpp
   ex40.cpp
   ex41.cpp
+  jitplayground.cpp
   )
 
 if (MFEM_USE_MPI)

examples/jitplayground.cpp

@@ -0,0 +1,536 @@
+#include <mfem.hpp>
+
+#include "../fem/dfem/util.hpp"
+
+#include <proteus/CppJitModule.h>
+
+#include "jitplayground.hpp"
+
+#include <algorithm>
+#include <array>
+#include <cctype>
+#include <cmath>
+#include <fstream>
+#include <initializer_list>
+#include <iostream>
+#include <memory>
+#include <sstream>
+#include <string>
+#include <string_view>
+#include <type_traits>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+namespace util
+{
+constexpr std::string_view Dirname(std::string_view path)
+{
+   const size_t last_sep = path.find_last_of("/\\");
+   if (last_sep == std::string_view::npos) { return {}; }
+   return path.substr(0, last_sep);
+}
+
+constexpr std::string_view thisFileDir = Dirname(__FILE__);
+}
+
+template <typename T>
+static std::string TypeNameString()
+{
+   return std::string(mfem::future::get_type_name<T>());
+}
+
+template <typename Tuple, size_t... Is>
+static auto ParamTypeStringsImpl(std::index_sequence<Is...>)
+{
+   return std::array<std::string, sizeof...(Is)>
+   {
+      TypeNameString<std::remove_reference_t<decltype(mfem::future::get<Is>(std::declval<Tuple&>()))>>()...
+   };
+}
+
+template <typename Tuple>
+static auto ParamTypeStrings()
+{
+   return ParamTypeStringsImpl<Tuple>(
+             std::make_index_sequence<mfem::future::tuple_size<Tuple>::value> {});
+}
+
+static std::string_view Trim(std::string_view s)
+{
+   size_t begin = 0;
+   while (begin < s.size() && std::isspace(static_cast<unsigned char>(s[begin])))
+   {
+      ++begin;
+   }
+   size_t end = s.size();
+   while (end > begin &&
+          std::isspace(static_cast<unsigned char>(s[end - 1])))
+   {
+      --end;
+   }
+   return s.substr(begin, end - begin);
+}
+
+static bool IsValidIdentifier(std::string_view s)
+{
+   if (s.empty()) { return false; }
+   const unsigned char c0 = static_cast<unsigned char>(s[0]);
+   if (!(std::isalpha(c0) || c0 == '_')) { return false; }
+   for (size_t i = 1; i < s.size(); ++i)
+   {
+      const unsigned char c = static_cast<unsigned char>(s[i]);
+      if (!(std::isalnum(c) || c == '_')) { return false; }
+   }
+   return true;
+}
+
+static bool ParseJitDirective(std::string_view line,
+                              std::string &type,
+                              std::string &var,
+                              std::string &kind)
+{
+   const size_t jit_pos = line.find("$JIT");
+   if (jit_pos == std::string_view::npos) { return false; }
+
+   const size_t open = line.find('[', jit_pos);
+   const size_t close = line.find(']', jit_pos);
+   MFEM_VERIFY(open != std::string_view::npos &&
+               close != std::string_view::npos &&
+               close > open,
+               "malformed $JIT directive (expected brackets): " << line);
+
+   const std::string_view payload = line.substr(open + 1, close - open - 1);
+   const size_t comma1 = payload.find(',');
+   const size_t comma2 = (comma1 == std::string_view::npos)
+                         ? std::string_view::npos
+                         : payload.find(',', comma1 + 1);
+   MFEM_VERIFY(comma1 != std::string_view::npos &&
+               comma2 != std::string_view::npos,
+               "malformed $JIT directive (expected 3 comma-separated fields): "
+               << line);
+
+   const std::string_view f0 = Trim(payload.substr(0, comma1));
+   const std::string_view f1 = Trim(payload.substr(comma1 + 1,
+                                                   comma2 - comma1 - 1));
+   const std::string_view f2 = Trim(payload.substr(comma2 + 1));
+   MFEM_VERIFY(!f0.empty() && !f1.empty() && !f2.empty(),
+               "malformed $JIT directive (empty field): " << line);
+
+   type.assign(f0);
+   var.assign(f1);
+   kind.assign(f2);
+   return true;
+}
+
+static std::string ReadFileOrEmpty(const std::string &fn)
+{
+   std::ifstream file(fn);
+   if (!file.is_open())
+   {
+      std::cerr << "could not open file " << fn << "\n";
+      return {};
+   }
+   std::stringstream buffer;
+   buffer << file.rdbuf();
+   return buffer.str();
+}
+
+static std::vector<std::string> ExtractJitVarNames(const std::string
+                                                   &kernel_code)
+{
+   std::stringstream ss(kernel_code);
+   std::string line;
+   std::vector<std::string> var_names;
+   std::unordered_set<std::string> seen_vars;
+
+   while (std::getline(ss, line))
+   {
+      std::string type, var, kind;
+      if (ParseJitDirective(line, type, var, kind))
+      {
+         MFEM_VERIFY(IsValidIdentifier(var),
+                     "$JIT variable must be a valid identifier: " << var);
+         MFEM_VERIFY(seen_vars.insert(var).second,
+                     "duplicate $JIT variable name: " << var);
+         var_names.push_back(var);
+      }
+   }
+   return var_names;
+}
+
+static std::string RewriteKernelForJit(std::string kernel_code,
+                                       const std::vector<std::string> &jit_values)
+{
+   std::stringstream ss(kernel_code);
+   std::string line;
+
+   std::string out;
+   out.reserve(kernel_code.size() + 128);
+
+   bool have_pending = false;
+   size_t pending_index = 0;
+   std::string pending_type;
+   std::string pending_var;
+   std::unordered_set<std::string> seen_vars;
+
+   while (std::getline(ss, line))
+   {
+      line.push_back('\n');
+
+      if (have_pending)
+      {
+         MFEM_VERIFY(pending_index < jit_values.size(),
+                     "not enough JIT values provided");
+         const size_t indent_end = line.find_first_not_of(" \t");
+         const std::string indent =
+            (indent_end == std::string::npos) ? std::string() :
+            line.substr(0, indent_end);
+         out += indent + "const " + pending_type + " " + pending_var + " = " +
+                jit_values[pending_index] + ";\n";
+         have_pending = false;
+         ++pending_index;
+         continue;
+      }
+
+      std::string type, var, kind;
+      if (ParseJitDirective(line, type, var, kind))
+      {
+         MFEM_VERIFY(IsValidIdentifier(var),
+                     "$JIT variable must be a valid identifier: " << var);
+         MFEM_VERIFY(kind == "generic",
+                     "unsupported $JIT kind: " << kind);
+         MFEM_VERIFY(seen_vars.insert(var).second,
+                     "duplicate $JIT variable name: " << var);
+
+         pending_type = std::move(type);
+         pending_var = std::move(var);
+         have_pending = true;
+         continue; // drop directive line
+      }
+
+      out += line;
+   }
+
+   MFEM_VERIFY(!have_pending,
+               "$JIT directive must annotate a following line");
+   MFEM_VERIFY(jit_values.size() == pending_index,
+               "JIT value count must match number of $JIT directives");
+   return out;
+}
+
+static std::string GeneratedOutputPath(std::string_view original_path)
+{
+   const size_t last_sep = original_path.find_last_of("/\\");
+   const size_t dot = original_path.find_last_of('.');
+   const bool dot_in_filename =
+      (dot != std::string_view::npos) &&
+      (last_sep == std::string_view::npos || dot > last_sep);
+
+   const std::string_view base =
+      dot_in_filename ? original_path.substr(0, dot) : original_path;
+   return std::string(base) + "_generated.hpp";
+}
+
+static void WriteFileOrWarn(const std::string &path,
+                            const std::string &contents)
+{
+   std::ofstream out(path);
+   if (!out.is_open())
+   {
+      std::cerr << "could not write generated file " << path << "\n";
+      return;
+   }
+   out << contents;
+}
+
+class JitQFunction
+{
+public:
+   template <typename ImplT, size_t N>
+   JitQFunction(ImplT, const std::string &fn,
+                const std::array<bool, N> &activity_map)
+   {
+      using qf_signature = typename
+                           mfem::future::get_function_signature<
+                           decltype(&ImplT::operator())>::type;
+      using qf_param_ts = typename qf_signature::parameter_ts;
+      constexpr size_t nparams = mfem::future::tuple_size<qf_param_ts>::value;
+      static_assert(N == nparams, "activity_map size must match qfunc arity");
+
+      this->fn = fn;
+      this->nparams = nparams;
+      this->activity_map.reserve(N);
+      for (size_t i = 0; i < N; ++i)
+      {
+         this->activity_map.push_back(activity_map[i]);
+      }
+      {
+         const auto param_types_arr = ParamTypeStrings<qf_param_ts>();
+         this->param_types.assign(param_types_arr.begin(), param_types_arr.end());
+      }
+      this->return_type = TypeNameString<typename qf_signature::return_t>();
+      this->return_is_void = std::is_same_v<typename qf_signature::return_t, void>;
+      this->impl_type_name = TypeNameString<ImplT>();
+      this->jit_var_names = ExtractJitVarNames(ReadFileOrEmpty(fn));
+   }
+
+   template <typename ReturnT, typename... Args>
+   ReturnT run(std::string_view name,
+               std::initializer_list<std::pair<std::string_view, std::string_view>> jit_values,
+               Args&&... args)
+   {
+      auto ordered_values = MatchJitValues(jit_values);
+      auto &mod = GetOrCreateModule(ordered_values);
+      auto &instance = mod.instantiate(std::string(name), std::string());
+      return instance.template run<ReturnT>(std::forward<Args>(args)...);
+   }
+
+   template <typename ReturnT, typename... Args>
+   ReturnT run_primal(
+      std::initializer_list<std::pair<std::string_view, std::string_view>> jit_values,
+      Args&&... args)
+   {
+      return run<ReturnT>(qfunc_name, jit_values,
+                          std::forward<Args>(args)...);
+   }
+
+   template <typename ReturnT, typename... Args>
+   ReturnT run_derivative(
+      std::initializer_list<std::pair<std::string_view, std::string_view>> jit_values,
+      Args&&... args)
+   {
+      return run<ReturnT>(qfunc_name + "_fwddiff", jit_values,
+                          std::forward<Args>(args)...);
+   }
+
+private:
+   std::vector<std::string_view> MatchJitValues(
+      std::initializer_list<std::pair<std::string_view, std::string_view>>
+      named_values) const
+   {
+      std::unordered_map<std::string_view, std::string_view> value_map;
+      for (const auto &[name, value] : named_values)
+      {
+         value_map[name] = value;
+      }
+
+      std::vector<std::string_view> ordered_values;
+      ordered_values.reserve(jit_var_names.size());
+      for (const auto &var_name : jit_var_names)
+      {
+         auto it = value_map.find(var_name);
+         MFEM_VERIFY(it != value_map.end(),
+                     "missing JIT value for variable: " << var_name);
+         ordered_values.push_back(it->second);
+      }
+
+      MFEM_VERIFY(ordered_values.size() == named_values.size(),
+                  "provided " << named_values.size() << " JIT values but expected "
+                  << jit_var_names.size());
+      return ordered_values;
+   }
+
+
+   std::string BuildModuleCode(const std::vector<std::string> &jit_values) const
+   {
+      std::string module_code =
+         RewriteKernelForJit(ReadFileOrEmpty(fn), jit_values);
+      module_code += "\n\n";
+      module_code += "// --- generated ---\n";
+      module_code +=
+         "template <typename return_type, typename... Args>\n"
+         "return_type __enzyme_fwddiff(Args...);\n"
+         "\n"
+         "extern int enzyme_const;\n"
+         "extern int enzyme_dup;\n"
+         "\n";
+
+      // Generate a primal wrapper with the requested symbol name, so the kernel
+      // header can just define the qfunc as a functor.
+      //
+      // Note: Proteus instantiates entrypoints via `qfunc_wrapper<>(...)` even
+      // when there are no user template args, so keep the wrapper itself a
+      // template (with a default parameter) while still doing literal `$JIT`
+      // replacements in the kernel code.
+      module_code += "template <typename = void>\n";
+      module_code += return_type + " " +
+                     std::string(qfunc_name) + "(";
+      bool first = true;
+      for (size_t i = 0; i < nparams; ++i)
+      {
+         if (!first) { module_code += ", "; }
+         first = false;
+         module_code += param_types[i] + " Arg" + std::to_string(i);
+      }
+      module_code += ")\n";
+      module_code += "{\n";
+      module_code += "   " + impl_type_name + " qf;\n";
+      if (return_is_void)
+      {
+         module_code += "   ";
+      }
+      else
+      {
+         module_code += "   return ";
+      }
+      module_code += "qf(";
+      for (size_t i = 0; i < nparams; ++i)
+      {
+         if (i) { module_code += ", "; }
+         module_code += "Arg" + std::to_string(i);
+      }
+      module_code += ");\n";
+      module_code += "}\n\n";
+
+      module_code += "template <typename = void>\n";
+      module_code += return_type + " " +
+                     std::string(qfunc_name) + "_fwddiff(";
+
+      first = true;
+      for (size_t i = 0; i < nparams; ++i)
+      {
+         if (!first) { module_code += ", "; }
+         first = false;
+         module_code += param_types[i] + " Arg" + std::to_string(i);
+         if (activity_map[i])
+         {
+            module_code += ", " + param_types[i] + " dArg" + std::to_string(i);
+         }
+      }
+      module_code += ")\n";
+      module_code += "{\n";
+      if (return_is_void)
+      {
+         module_code += "   __enzyme_fwddiff<void>(\n";
+      }
+      else
+      {
+         module_code += "   return __enzyme_fwddiff<" +
+                        return_type + ">(\n";
+      }
+      module_code += "      (void*)" + std::string(qfunc_name) + "<>";
+      module_code += ",\n";
+      for (size_t i = 0; i < nparams; ++i)
+      {
+         if (activity_map[i])
+         {
+            module_code += "      enzyme_dup, Arg" + std::to_string(i) +
+                           ", dArg" + std::to_string(i);
+         }
+         else
+         {
+            module_code += "      enzyme_const, Arg" + std::to_string(i);
+         }
+         module_code += (i + 1 == nparams) ? ");\n" : ",\n";
+      }
+      module_code += "}\n";
+
+      WriteFileOrWarn(GeneratedOutputPath(fn), module_code);
+      return module_code;
+   }
+
+   proteus::CppJitModule &GetOrCreateModule(
+      const std::vector<std::string_view> &jit_values)
+   {
+      std::string key;
+      for (const auto &val : jit_values)
+      {
+         if (!key.empty()) { key += ","; }
+         key += val;
+      }
+
+      auto it = modules.find(key);
+      if (it != modules.end())
+      {
+         return *it->second;
+      }
+
+      std::vector<std::string> values(jit_values.begin(), jit_values.end());
+      std::string code = BuildModuleCode(values);
+      auto mod = std::make_unique<proteus::CppJitModule>("host", code,
+                                                         DefaultExtraArgs());
+      auto [inserted, ok] = modules.emplace(key, std::move(mod));
+      MFEM_VERIFY(ok, "failed to cache JIT module");
+      return *inserted->second;
+   }
+
+   static std::vector<std::string> DefaultExtraArgs()
+   {
+      return {"-fplugin=/Users/andrej1/local/enzyme/lib/ClangEnzyme-20.dylib"};
+   }
+
+   std::string qfunc_name = "qfunc_wrapper";
+   std::string fn;
+   size_t nparams = 0;
+   std::vector<bool> activity_map;
+   std::vector<std::string> param_types;
+   std::string return_type;
+   bool return_is_void = false;
+   std::string impl_type_name;
+   std::vector<std::string> jit_var_names;
+   std::unordered_map<std::string, std::unique_ptr<proteus::CppJitModule>> modules;
+};
+
+int main()
+{
+   const size_t N = 4;
+   const size_t M = 5;
+   const double A = 123.4;
+
+   std::vector<double> X(N);
+   std::vector<double> Y(N);
+   for (size_t i = 0; i < N; ++i)
+   {
+      X[i] = static_cast<double>(i + 1);
+      Y[i] = static_cast<double>(N - i);
+   }
+
+   // // >>> user interface calls
+   // const std::string kernel_path = std::string(util::thisFileDir) +
+   //                                 "/jitplayground.hpp";
+   // JitQFunction qf(daxpy_op{}, kernel_path, std::array{false, true, false});
+   // // <<< user interface calls
+
+   // // this will happen internally in dFEM
+
+   daxpy_op op;
+   printf("\n\nfunction call\n");
+   op(&A, X.data(), Y.data(), &N);
+
+   // reset X for the derivative test
+   for (size_t i = 0; i < N; ++i)
+   {
+      X[i] = static_cast<double>(i + 1);
+      Y[i] = static_cast<double>(N - i);
+   }
+
+   std::vector<double> dX(N, 1.0);
+   printf("\n\nforward diff call\n");
+   daxpy_op_fwddiff(&A, X.data(), dX.data(), Y.data(), &N);
+
+   std::vector<double> dX_manual(N, A);
+
+   printf("\n\nderivative checks\n");
+   std::cout << "dX: ";
+   for (size_t i = 0; i < N; ++i)
+   {
+      std::cout << dX[i] << (i + 1 == N ? '\n' : ' ');
+   }
+
+   std::cout << "dX_manual: ";
+   for (size_t i = 0; i < N; ++i)
+   {
+      std::cout << dX_manual[i] << (i + 1 == N ? '\n' : ' ');
+   }
+
+   double max_abs_err = 0.0;
+   for (size_t i = 0; i < N; ++i)
+   {
+      max_abs_err = std::max(max_abs_err, std::abs(dX[i] - dX_manual[i]));
+   }
+   std::cout << "max |dX - dX_manual| = " << max_abs_err << "\n";
+
+   return 0;
+}

examples/jitplayground.hpp

@@ -0,0 +1,58 @@
+#pragma once
+
+#include <cstddef>
+#include <vector>
+#include <type_traits>
+
+#include "proteus/JitInterface.h"
+
+struct daxpy_op
+{
+   void operator()(
+      const double *a,
+      double *x,
+      const double *y,
+      const size_t *N) const
+   {
+      const size_t n = *N;
+      auto lam = [=, n = proteus::jit_variable(n)]
+                 () __attribute__((annotate("jit")))
+      {
+         printf("N = %zu\n", n);
+         for (size_t i = 0; i < n; ++i)
+         {
+            printf("x[%zu] = %f, y[%zu] = %f\n", i, x[i], i, y[i]);
+            x[i] = *a * x[i] + y[i];
+            printf("updated x[%zu] = %f\n", i, x[i]);
+         }
+      };
+
+      proteus::register_lambda(lam);
+
+      lam();
+   }
+};
+
+template <typename return_type, typename... Args>
+return_type __enzyme_fwddiff(Args...);
+
+extern int enzyme_const;
+extern int enzyme_dup;
+
+void daxpy_op_wrapper(const double * Arg0, double * Arg1,
+                      const double * Arg2, const size_t *Arg3)
+{
+   daxpy_op qf;
+   qf(Arg0, Arg1, Arg2, Arg3);
+}
+
+void daxpy_op_fwddiff(const double * Arg0, double * Arg1,
+                      double * dArg1, const double * Arg2, const size_t *Arg3)
+{
+   __enzyme_fwddiff<void>(
+      (void*)daxpy_op_wrapper,
+      enzyme_const, Arg0,
+      enzyme_dup, Arg1, dArg1,
+      enzyme_const, Arg2,
+      enzyme_const, Arg3);
+}

fem/CMakeLists.txt

@@ -121,6 +121,11 @@ set(SRCS
   qinterp/eval_hdiv.cpp
   qinterp/grad_by_nodes.cpp
   qinterp/grad_by_vdim.cpp
+  qinterp/grad_transpose.cpp
+  qinterp/grad_transpose_by_nodes.cpp
+  qinterp/grad_transpose_by_vdim.cpp
+  qinterp/eval_transpose.cpp
+  qinterp/eval_transpose_by_vdim.cpp
   qspace.cpp
   quadinterpolator.cpp
   quadinterpolator_face.cpp
@@ -133,7 +138,7 @@ set(SRCS
   tmop/assemble/diag2.cpp
   tmop/assemble/grad2_limit.cpp
   tmop/assemble/grad2.cpp
-  tmop/assemble/diag3_limit.cpp  
+  tmop/assemble/diag3_limit.cpp
   tmop/assemble/diag3.cpp
   tmop/assemble/grad3_limit.cpp
   tmop/assemble/grad3.cpp
@@ -278,8 +283,10 @@ set(HDRS
   qfunction.hpp
   qinterp/det.hpp
   qinterp/eval.hpp
+  qinterp/eval_transpose.hpp
   qinterp/eval_hdiv.hpp
   qinterp/grad.hpp
+  qinterp/grad_transpose.hpp
   qspace.hpp
   quadinterpolator.hpp
   quadinterpolator_face.hpp
@@ -313,36 +320,36 @@ set(HDRS
   )
 
 if (MFEM_USE_SIDRE)
-  list(APPEND SRCS sidredatacollection.cpp)
-  list(APPEND HDRS sidredatacollection.hpp)
+    list(APPEND SRCS sidredatacollection.cpp)
+    list(APPEND HDRS sidredatacollection.hpp)
 endif()
 
 if (MFEM_USE_CONDUIT)
-  list(APPEND SRCS conduitdatacollection.cpp)
-  list(APPEND HDRS conduitdatacollection.hpp)
+    list(APPEND SRCS conduitdatacollection.cpp)
+    list(APPEND HDRS conduitdatacollection.hpp)
 endif()
 
 if (MFEM_USE_ADIOS2)
-  list(APPEND SRCS adios2datacollection.cpp)
-  list(APPEND HDRS adios2datacollection.hpp)
+    list(APPEND SRCS adios2datacollection.cpp)
+    list(APPEND HDRS adios2datacollection.hpp)
 endif()
 
 if (MFEM_USE_FMS)
-  list(APPEND SRCS fmsdatacollection.cpp fmsconvert.cpp)
-  list(APPEND HDRS fmsdatacollection.hpp fmsconvert.hpp)
+    list(APPEND SRCS fmsdatacollection.cpp fmsconvert.cpp)
+    list(APPEND HDRS fmsdatacollection.hpp fmsconvert.hpp)
 endif()
 
 if (MFEM_USE_MPI)
-  list(APPEND SRCS
+    list(APPEND SRCS
     pbilinearform.cpp
     pfespace.cpp
     pgridfunc.cpp
     plinearform.cpp
     pnonlinearform.cpp
     prestriction.cpp)
-  # If this list (HDRS -> HEADERS) is used for install, we probably want the
-  # headers added all the time.
-  list(APPEND HDRS
+    # If this list (HDRS -> HEADERS) is used for install, we probably want the
+    # headers added all the time.
+    list(APPEND HDRS
     pbilinearform.hpp
     pfespace.hpp
     pgridfunc.hpp

fem/bilininteg.hpp

@@ -2209,6 +2209,7 @@ private:
    const FiniteElementSpace *fespace;
    const DofToQuad *maps;         ///< Not owned
    const GeometricFactors *geom;  ///< Not owned
+public:
    int dim, ne, dofs1D, quad1D;
    Vector pa_data;
    bool symmetric = true; ///< False if using a nonsymmetric matrix coefficient
@@ -2350,8 +2351,8 @@ public:
    template <int DIM, int D1D, int Q1D>
    static void AddSpecialization()
    {
-      ApplyPAKernels::Specialization<DIM,D1D,Q1D>::Add();
-      DiagonalPAKernels::Specialization<DIM,D1D,Q1D>::Add();
+      ApplyPAKernels::Add<DIM,D1D,Q1D>();
+      DiagonalPAKernels::Add<DIM,D1D,Q1D>();
    }
 protected:
    const IntegrationRule* GetDefaultIntegrationRule(
@@ -2450,8 +2451,8 @@ public:
    template <int DIM, int D1D, int Q1D>
    static void AddSpecialization()
    {
-      ApplyPAKernels::Specialization<DIM,D1D,Q1D>::Add();
-      DiagonalPAKernels::Specialization<DIM,D1D,Q1D>::Add();
+      ApplyPAKernels::Add<DIM,D1D,Q1D>();
+      DiagonalPAKernels::Add<DIM,D1D,Q1D>();
    }
 
 protected:
@@ -2545,8 +2546,8 @@ public:
    template <int DIM, int D1D, int Q1D>
    static void AddSpecialization()
    {
-      ApplyPAKernels::Specialization<DIM, D1D, Q1D>::Add();
-      ApplyPATKernels::Specialization<DIM, D1D, Q1D>::Add();
+      ApplyPAKernels::Add<DIM, D1D, Q1D>();
+      ApplyPATKernels::Add<DIM, D1D, Q1D>();
    }
 
    struct Kernels { Kernels(); };
@@ -2891,8 +2892,8 @@ public:
 
    template <int DIM, int D1D, int Q1D> static void AddSpecialization()
    {
-      ApplyPAKernels::Specialization<DIM, D1D, Q1D>::Add();
-      DiagonalPAKernels::Specialization<DIM, D1D, Q1D>::Add();
+      ApplyPAKernels::Add<DIM, D1D, Q1D>();
+      DiagonalPAKernels::Add<DIM, D1D, Q1D>();
    }
 };
 
@@ -3218,7 +3219,7 @@ public:
    template <int DIM, int VDIM, int D1D, int Q1D>
    static void AddSpecialization()
    {
-      ApplyPAKernels::Specialization<DIM, VDIM, D1D, Q1D>::Add();
+      ApplyPAKernels::Add<DIM, VDIM, D1D, Q1D>();
    }
 
    // struct Kernels { Kernels(); };
@@ -3450,8 +3451,8 @@ public:
 
    template <int DIM, int D1D, int Q1D> static void AddSpecialization()
    {
-      ApplyPAKernels::Specialization<DIM, D1D, Q1D>::Add();
-      ApplyPATKernels::Specialization<DIM, D1D, Q1D>::Add();
+      ApplyPAKernels::Add<DIM, D1D, Q1D>();
+      ApplyPATKernels::Add<DIM, D1D, Q1D>();
    }
 
    struct Kernels { Kernels(); };
@@ -3559,7 +3560,7 @@ public:
 
    template <int DIM, int D1D, int Q1D> static void AddSpecialization()
    {
-      ApplyPAKernels::Specialization<DIM, D1D, Q1D>::Add();
+      ApplyPAKernels::Add<DIM, D1D, Q1D>();
    }
 
    struct Kernels { Kernels(); };

fem/dfem/backends/devices.hpp

@@ -0,0 +1,687 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include <tuple>
+#include <utility>
+
+#include "fem/dfem/fieldoperator.hpp"
+#include "fem/dfem/integrator_ctx.hpp"
+
+#include "fem/pfespace.hpp"
+#include "fem/quadinterpolator.hpp"
+#include "general/forall.hpp"
+#include "linalg/dtensor.hpp"
+#include "linalg/tensor_arrays.hpp"
+#include "linalg/vector.hpp"
+
+namespace mfem::future::device
+{
+
+// FieldBasis /////////////////////////////////////////////////////////////////
+struct FieldBasis
+{
+   // E-vector -> Q-vector
+   std::function<void(const Vector &, Vector &)> forward;
+   // Q-vector -> E-vector
+   std::function<void(const Vector &, Vector &)> transpose;
+};
+
+// FunctionSignature //////////////////////////////////////////////////////////
+template <class F> struct FunctionSignature;
+
+template <typename output_t, typename... input_ts>
+struct FunctionSignature<output_t(input_ts...)>
+{
+   using return_t = output_t;
+   using parameter_ts = std::tuple<input_ts...>;
+};
+
+template <class T> struct create_function_signature;
+
+// Specialization for member functions (lambdas)
+template <typename output_t, typename T, typename... input_ts>
+struct create_function_signature<output_t (T::*)(input_ts...) const>
+{
+   using type = FunctionSignature<output_t(input_ts...)>;
+};
+
+// Specialization for function pointers
+template <typename output_t, typename... input_ts>
+struct create_function_signature<output_t (*)(input_ts...)>
+{
+   using type = FunctionSignature<output_t(input_ts...)>;
+};
+
+// get_function_signature /////////////////////////////////////////////////////
+template <typename...>
+using void_t = void;
+
+template <typename T, typename = void>
+struct get_function_signature
+{
+   using type = typename create_function_signature<T>::type;
+};
+
+template <typename T>
+struct get_function_signature<T, void_t<decltype(&T::operator())>>
+{
+   using type = typename create_function_signature<decltype(&T::operator())>::type;
+};
+
+// all_true ///////////////////////////////////////////////////////////////////
+template <size_t N, size_t... Is>
+constexpr std::array<bool, N> all_true_impl(std::index_sequence<Is...>)
+{
+   return {{((void)Is, true)...}};
+}
+
+template <size_t N>
+constexpr std::array<bool, N> all_true()
+{
+   return all_true_impl<N>(std::make_index_sequence<N> {});
+}
+
+// set_layout /////////////////////////////////////////////////////////////////
+template <typename ndarray_t>
+inline void set_layout_default(ndarray_t &a)
+{
+   NVTX_MARK_FUNCTION;
+   if constexpr (ndarray_t::tensor_rank() == 0) { return; }
+   constexpr std::size_t nd = ndarray_t::rank();
+   constexpr std::size_t td = ndarray_t::tensor_rank();
+   std::array<std::size_t, nd + td> perm{};
+   for (std::size_t i = 0; i < td; i++) { perm[i] = nd + i; }
+   for (std::size_t i = 0; i < nd; i++) { perm[td + i] = i; }
+   a.set_layout(perm);
+}
+
+template <typename ndarray_t>
+inline void set_layout(ndarray_t& a, const std::vector<int>& layout)
+{
+   NVTX_MARK_FUNCTION;
+   if constexpr (ndarray_t::tensor_rank() == 0) { return; }
+   constexpr std::size_t nd = ndarray_t::rank();
+   constexpr std::size_t td = ndarray_t::tensor_rank();
+   constexpr std::size_t N  = nd + td;
+   // missing means default
+   if (layout.empty()) { set_layout_default(a); return; }
+   MFEM_VERIFY(layout.size() == N,
+               "layout size mismatch: expected " << N << " got " << layout.size());
+   // TODO: make a version of set_layout that takes `std::vector<int>`
+   std::array<std::size_t, N> perm{};
+   for (std::size_t i = 0; i < N; i++)
+   {
+      MFEM_VERIFY(layout[i] >= 0, "layout index must be >=0");
+      perm[i] = static_cast<std::size_t>(layout[i]);
+   }
+   a.set_layout(perm);
+}
+
+// make_tensor_array //////////////////////////////////////////////////////////
+/// Primary template: intentionally undefined — gives a clear error for unsupported types.
+template <typename T>
+struct tensor_array_traits;
+
+/// Matches tensor<scalar_t, sizes...>
+template <typename scalar_t, int... sizes>
+struct tensor_array_traits<tensor<scalar_t, sizes...>>
+{
+   using scalar_type = scalar_t;
+   template <std::size_t ndims>
+   using array_type = tensor_ndarray<scalar_t, ndims, sizes...>;
+};
+
+/// Matches tensor_ndarray<scalar_t, ndims, tensor_sizes...>
+template <typename scalar_t, int ndims, int... tensor_sizes>
+struct tensor_array_traits<tensor_ndarray<scalar_t, ndims, tensor_sizes...>>
+{
+   using scalar_type = scalar_t;
+   template <std::size_t N>
+   using array_type = tensor_ndarray<scalar_t, N, tensor_sizes...>;
+};
+
+/// Entry point: explicit tensor type T as template argument.
+template <typename T, typename ptr_scalar_t, typename... dyn_sizes_t>
+decltype(auto) make_tensor_array(ptr_scalar_t *ptr,
+                                 const std::vector<int>* layout,
+                                 dyn_sizes_t... dynamic_sizes)
+{
+   NVTX_MARK_FUNCTION;
+   using traits = tensor_array_traits<T>;
+   using array_t = typename traits::template array_type<sizeof...(dynamic_sizes)>;
+   auto a = array_t(ptr, {std::size_t(dynamic_sizes)...});
+   if (layout) { set_layout(a, *layout); }
+   else        { set_layout_default(a); }
+   return a;
+}
+
+// constexpr_for //////////////////////////////////////////////////////////////
+template <auto start, auto end, auto inc = 1, typename F>
+constexpr void constexpr_for(F&& f)
+{
+   if constexpr (start < end)
+   {
+      f(std::integral_constant<decltype(start), start>());
+      constexpr_for<start + inc, end, inc>(f);
+   }
+}
+
+// is_tensor_array ////////////////////////////////////////////////////////////
+template <typename T>
+struct is_tensor_array : std::false_type {};
+
+template <typename scalar_t, int... Dims>
+struct is_tensor_array<tensor_array<scalar_t, Dims...>> : std::true_type {};
+
+template <typename T>
+struct is_tensor_array_mut : std::false_type {};
+
+template <typename scalar_t, int... Dims>
+struct is_tensor_array_mut<tensor_array<scalar_t, Dims...>>:
+/*  */ std::bool_constant<!std::is_const_v<scalar_t>> {};
+
+// supports_tensor_array_qfunc ////////////////////////////////////////////////
+template <typename qfunc_t, typename inputs_t, typename outputs_t>
+struct supports_tensor_array_qfunc
+{
+   using qf_signature = typename get_function_signature<qfunc_t>::type;
+   using qf_param_ts = typename qf_signature::parameter_ts;
+
+   static constexpr int ninputs = std::tuple_size_v<inputs_t>;
+   static constexpr int noutputs = std::tuple_size_v<outputs_t>;
+   static constexpr int nparams = std::tuple_size_v<qf_param_ts>;
+
+   template <std::size_t... Is>
+   static constexpr bool InputsOk(std::index_sequence<Is...>)
+   {
+      return (is_tensor_array<std::remove_cv_t<std::remove_reference_t<
+              std::tuple_element_t<Is, qf_param_ts>>>>::value && ...);
+   }
+
+   template <std::size_t... Is>
+   static constexpr bool OutputsOk(std::index_sequence<Is...>)
+   {
+      return (is_tensor_array_mut<std::remove_cv_t<std::remove_reference_t<
+              std::tuple_element_t<ninputs + Is, qf_param_ts>>>>::value && ...);
+   }
+
+   static constexpr bool value =
+      (nparams == ninputs + noutputs) &&
+      InputsOk(std::make_index_sequence<ninputs> {}) &&
+      OutputsOk(std::make_index_sequence<noutputs> {});
+};
+
+// FieldBasisFromWeight ///////////////////////////////////////////////////////
+inline FieldBasis FieldBasisFromWeight(const IntegrationRule &ir)
+{
+   NVTX_MARK_FUNCTION;
+   return
+   {
+      [&ir](const Vector &, Vector &xq)
+      {
+         NVTX("Weights"); // could be done once 🔥
+         const int nqp = ir.GetNPoints();
+         MFEM_ASSERT(xq.Size() % nqp == 0, "weight block has unexpected size");
+         const int ne = xq.Size() / nqp;
+         const auto wref = ir.GetWeights().Read();
+         auto xq_w = Reshape(xq.Write(), nqp, ne);
+         mfem::forall(ne * nqp, [=] MFEM_HOST_DEVICE(int eq)
+         {
+            const int q = eq % nqp, e = eq / nqp;
+            xq_w(q,e) = wref[q];
+         });
+      },
+      [](const Vector &, Vector &) { }
+   };
+}
+
+inline FieldBasis FromQI(const QuadratureInterpolator *qi,
+                         QuadratureInterpolator::EvalFlags mode)
+{
+   NVTX_MARK_FUNCTION;
+   return
+   {
+      [qi, mode](const Vector &xe, Vector &xq)
+      {
+         qi->SetOutputLayout(QVectorLayout::byVDIM);
+         if (mode == QuadratureInterpolator::VALUES)
+         {
+            NVTX("VALUES");
+            qi->Values(xe, xq);
+         }
+         else
+         {
+            NVTX("DERIVATIVES");
+            qi->Derivatives(xe, xq);
+         }
+      },
+      [qi, mode](const Vector &yq, Vector &ye)
+      {
+         Vector empty;
+         qi->SetOutputLayout(QVectorLayout::byVDIM);
+         if (mode == QuadratureInterpolator::VALUES)
+         {
+            NVTX("Transposed VALUES");
+            qi->AddMultTranspose(QuadratureInterpolator::VALUES, yq, empty, ye);
+         }
+         else
+         {
+            NVTX("Transposed DERIVATIVES");
+            qi->AddMultTranspose(QuadratureInterpolator::DERIVATIVES, empty, yq, ye);
+         }
+      }
+   };
+}
+
+// QuadratureFunction identity copy
+inline FieldBasis FromQF()
+{
+   NVTX_MARK_FUNCTION;
+   return
+   {
+      [](const Vector &xe, Vector &xq)
+      {
+         NVTX("FromQF(e->q)");
+         xq.NewMemoryAndSize(xe.GetMemory(), xe.Size(), false);
+      },
+      [](const Vector &yq, Vector &ye) { NVTX("FromQF(q->e)"); ye = yq; }
+   };
+}
+
+// User-defined parameter space B
+inline FieldBasis FromPS(const Operator *B, const Operator *Bt)
+{
+   NVTX_MARK_FUNCTION;
+   return
+   {
+      [B](const Vector &xe, Vector &xq) { NVTX("B->Mult(e->q)");  B->Mult(xe, xq); },
+      [Bt](const Vector &yq, Vector &ye) { NVTX("Bt->Mult(q->e)"); Bt->Mult(yq, ye); }
+   };
+}
+
+// GetFieldBasis //////////////////////////////////////////////////////////////
+inline const FieldBasis GetFieldBasis(const FieldDescriptor &f,
+                                      const IntegrationRule &ir,
+                                      QuadratureInterpolator::EvalFlags mode)
+{
+   NVTX_MARK_FUNCTION;
+   return std::visit([&ir, &mode](auto && arg) -> FieldBasis
+   {
+      using T = std::decay_t<decltype(arg)>;
+
+      if constexpr (std::is_same_v<T, const FiniteElementSpace *>)
+      {
+         return FromQI(arg->GetQuadratureInterpolator(ir), mode);
+      }
+      else if constexpr (std::is_same_v<T, const ParFiniteElementSpace *>)
+      {
+         return FromQI(arg->GetQuadratureInterpolator(ir), mode);
+      }
+      else if constexpr (std::is_same_v<T, const QuadratureFunction *>)
+      {
+         return FromQF();
+      }
+      else if constexpr (std::is_same_v<T, const ParameterSpace *>)
+      {
+         return FromPS(arg->GetB(), arg->GetBt());
+      }
+      else if constexpr (std::is_same_v<T, const IntegrationRule *>)
+      {
+         return FieldBasis{};
+      }
+      else
+      {
+         static_assert(false, "internal error");
+      }
+   }, f.data);
+}
+
+// create_fieldbases //////////////////////////////////////////////////////////
+template <typename fops_t, size_t nfops> inline
+std::array<FieldBasis, nfops> get_bases(fops_t &fops,
+                                        const std::array<size_t, nfops> &fop_to_fd,
+                                        const std::vector<FieldDescriptor> &fds,
+                                        const IntegrationRule &ir)
+{
+   NVTX_MARK_FUNCTION;
+   std::array<FieldBasis, nfops> bases;
+   constexpr_for<0, nfops>([&](auto i)
+   {
+      const auto fop = std::get<i>(fops);
+      using fop_t = std::decay_t<decltype(fop)>;
+
+      const auto fd = fds[fop_to_fd[i]];
+
+      constexpr QuadratureInterpolator::EvalFlags dummy_mode =
+         QuadratureInterpolator::VALUES;
+      if constexpr (is_identity_fop<fop_t>::value)
+      {
+         bases[i] = device::GetFieldBasis(fd, ir, dummy_mode);
+      }
+      else if constexpr (is_weight_fop<fop_t>::value)
+      {
+         bases[i] = device::FieldBasisFromWeight(ir);
+      }
+      else if constexpr (is_value_fop<fop_t>::value)
+      {
+         bases[i] = device::GetFieldBasis(fd, ir, QuadratureInterpolator::VALUES);
+      }
+      else if constexpr (is_gradient_fop<fop_t>::value)
+      {
+         bases[i] = device::GetFieldBasis(fd, ir, QuadratureInterpolator::DERIVATIVES);
+      }
+      else { static_assert(false, "internal error"); }
+   });
+   return bases;
+}
+
+// check_types ////////////////////////////////////////////////////////////////
+template <typename fops_t, size_t nfops> inline
+bool check_types(fops_t &fops,
+                 const std::array<size_t, nfops> &fop_to_fd,
+                 const std::vector<FieldDescriptor> &fields)
+{
+   NVTX_MARK_FUNCTION;
+   constexpr_for<0, nfops>([&](auto i)
+   {
+      const auto input = std::get<i.value>(fops);
+      using input_t = std::decay_t<decltype(input)>;
+
+      [[maybe_unused]] const auto fd = fields[fop_to_fd[i]];
+
+      if constexpr (is_identity_fop<input_t>::value)
+      {
+         MFEM_ASSERT(std::holds_alternative<const QuadratureFunction *>(fd.data),
+                     "Identity FieldOperator requested on non "
+                     "QuadratureFunction");
+      }
+      else if constexpr (is_weight_fop<input_t>::value)
+      {
+      }
+      else if constexpr (is_value_fop<input_t>::value)
+      {
+         MFEM_ASSERT(std::holds_alternative<const FiniteElementSpace *>(fd.data) ||
+                     std::holds_alternative<const ParFiniteElementSpace *>(fd.data) ||
+                     std::holds_alternative<const ParameterSpace *>(fd.data),
+                     "Value FieldOperator requested on non "
+                     "QuadratureFunction");
+      }
+      else if constexpr (is_gradient_fop<input_t>::value)
+      {
+         MFEM_ASSERT(std::holds_alternative<const FiniteElementSpace *>(fd.data) ||
+                     std::holds_alternative<const ParFiniteElementSpace *>(fd.data),
+                     "Value FieldOperator requested on non "
+                     "QuadratureFunction");
+      }
+   });
+   return true;
+}
+
+// create_fop_to_fd ///////////////////////////////////////////////////////////
+// Create quadrature function fop to fields map
+template <size_t M, typename fops_t, size_t N = std::tuple_size_v<fops_t>>
+std::array<size_t, M> fop_to_fd(const fops_t &fops,
+                                const std::vector<FieldDescriptor> &fields)
+{
+   static_assert(N == M, "sizes must match");
+   std::array<size_t, M> fop_to_fd;
+   constexpr_for<0, N>([&](auto i)
+   {
+      const auto fop = std::get<i>(fops);
+      fop_to_fd[i] = std::numeric_limits<size_t>::max();
+      for (size_t j = 0; j < fields.size(); j++)
+      {
+         // TODO: output.GetFieldId() should probably store/return size_t
+         if (static_cast<int>(fields[j].id) == fop.GetFieldId())
+         {
+            fop_to_fd[i] = j;
+         }
+      }
+      // Handle Weight type. There is no FieldDescriptor for the weight.
+      // TODO: Create weight descriptor for the weight for internal use?
+      // TODO: this is a hack...
+      if (is_weight_fop<std::remove_cv_t<decltype(fop)>>::value)
+      {
+         fop_to_fd[i] = 0;
+      }
+      else if (fop_to_fd[i] == std::numeric_limits<size_t>::max())
+      {
+         MFEM_ABORT("not found");
+      }
+   });
+   return fop_to_fd;
+}
+
+// interpolate ////////////////////////////////////////////////////////////////
+template <size_t ninputs>
+inline void interpolate(const std::array<size_t, ninputs> &input_to_infd,
+                        const std::array<FieldBasis, ninputs> &input_bases,
+                        const std::vector<Vector *> &xe,
+                        BlockVector &xq,
+                        const std::array<bool, ninputs> &conditional = all_true<ninputs>())
+{
+   NVTX_MARK_FUNCTION;
+   constexpr_for<0, ninputs>([&](auto i)
+   {
+      if (!conditional.empty() && !conditional[i]) { return; }
+      NVTX_MARK("input forward block #{}", i.value);
+      input_bases[i].forward(*xe[input_to_infd[i]], xq.GetBlock(i));
+   });
+}
+
+// call_qfunc /////////////////////////////////////////////////////////////////
+template <typename qfunc_t, std::size_t... Is, std::size_t... Os>
+inline void call_qfunc(const qfunc_t &qfunc,
+                       const BlockVector &xq,
+                       BlockVector &yq,
+                       int gnqp,
+                       const std::array<std::vector<int>, sizeof...(Is)>& in_layouts,
+                       const std::array<std::vector<int>, sizeof...(Os)>& out_layouts,
+                       std::index_sequence<Is...>,
+                       std::index_sequence<Os...>)
+{
+   NVTX_MARK_FUNCTION;
+   constexpr std::size_t ninputs = sizeof...(Is);
+
+   using qf_signature = typename get_function_signature<qfunc_t>::type;
+   using qf_param_ts = typename qf_signature::parameter_ts;
+
+   NVTX_MARK_INI("inputs");
+   auto inputs = std::make_tuple(
+                    make_tensor_array<std::remove_cv_t<std::remove_reference_t<
+                    std::tuple_element_t<Is, qf_param_ts>>>>(
+                       xq.GetBlock(Is).Read(), &in_layouts[Is], gnqp)...);
+   NVTX_MARK_END("inputs");
+
+   NVTX_MARK_INI("outputs");
+   auto outputs = std::make_tuple(
+                     make_tensor_array<std::remove_cv_t<std::remove_reference_t<
+                     std::tuple_element_t<ninputs + Os, qf_param_ts>>>>(
+                        yq.GetBlock(Os).ReadWrite(), &out_layouts[Os], gnqp)...);
+   NVTX_MARK_END("outputs");
+
+   std::apply([&](auto&&... args)
+   {
+      NVTX("QFunction()");
+      qfunc(args...);
+   }, std::tuple_cat(inputs, outputs));
+}
+
+// integrate //////////////////////////////////////////////////////////////////
+template <size_t noutputs>
+inline void integrate(const std::array<size_t, noutputs> &output_to_outfd,
+                      const std::array<FieldBasis, noutputs> &output_bases,
+                      const BlockVector &yq,
+                      std::vector<Vector *> &ye)
+{
+   NVTX_MARK_FUNCTION;
+   for (auto v : ye) { NVTX("ye = 0.0"); *v = 0.0; }
+
+   constexpr_for<0, noutputs>([&](auto i)
+   {
+      NVTX("out transpose block #{}", i.value);
+      output_bases[i].transpose(yq.GetBlock(i), *ye[output_to_outfd[i]]);
+   });
+}
+
+// ACTION /////////////////////////////////////////////////////////////////////
+template<typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t,
+         size_t N = std::tuple_size_v<inputs_t>,
+         size_t M = std::tuple_size_v<outputs_t>>
+class Action
+{
+   IntegratorContext ctx;
+   qfunc_t qfunc;
+   inputs_t inputs;
+   outputs_t outputs;
+
+   std::array<size_t, N> input_to_infd;
+   std::array<size_t, M> output_to_outfd;
+   const bool input_checks, output_checks;
+
+   std::array<FieldBasis, N> input_bases;
+   std::array<FieldBasis, M> output_bases;
+
+   std::array<std::vector<int>, N> input_qlayouts;
+   std::array<std::vector<int>, M> output_qlayouts;
+
+   int gnqp = 0;
+   Array<int> xq_offsets, yq_offsets;
+   mutable BlockVector xq, yq;
+
+public:
+   Action(IntegratorContext ctx,
+          qfunc_t qfunc,
+          inputs_t inputs,
+          outputs_t outputs):
+      ctx(ctx),
+      qfunc(std::move(qfunc)),
+      inputs(inputs),
+      outputs(outputs),
+      input_to_infd(fop_to_fd<N>(inputs, ctx.infds)),
+      output_to_outfd(fop_to_fd<M>(outputs, ctx.outfds)),
+      input_checks(check_types(inputs, input_to_infd, ctx.infds)),
+      output_checks(check_types(outputs, output_to_outfd, ctx.outfds)),
+      input_bases(get_bases(inputs, input_to_infd, ctx.infds, ctx.ir)),
+      output_bases(get_bases(outputs, output_to_outfd, ctx.outfds, ctx.ir))
+   {
+      NVTX_MARK_FUNCTION;
+
+      // Prepare inputs q-layouts maps for the qfunc call
+      constexpr_for<0, N>([&](auto i)
+      {
+         using in_t = std::decay_t<decltype(inputs)>;
+         using fop_t = std::remove_cv_t<std::remove_reference_t<
+                       std::tuple_element_t<decltype(i)::value, in_t>>>;
+         const auto it = ctx.in_qlayouts.find(std::type_index(typeid(fop_t)));
+         if (it != ctx.in_qlayouts.end()) { input_qlayouts[i] = it->second; }
+         else
+         {
+            input_qlayouts[i].clear();
+         }
+      });
+
+      // Prepare outputs q-layouts maps for the qfunc call
+      constexpr_for<0, M>([&](auto i)
+      {
+         using out_t = std::decay_t<decltype(outputs)>;
+         using fop_t = std::remove_cv_t<std::remove_reference_t<
+                       std::tuple_element_t<decltype(i)::value, out_t>>>;
+         const auto it = ctx.out_qlayouts.find(std::type_index(typeid(fop_t)));
+         if (it != ctx.out_qlayouts.end())
+         {
+            output_qlayouts[i] = it->second;
+         }
+         else
+         {
+            output_qlayouts[i].clear();
+         }
+      });
+
+      const int nqp = ctx.ir.GetNPoints();
+      gnqp = nqp * ctx.nentities;
+
+      // prepare xq and yq BlockVectors
+      xq_offsets.SetSize(N + 1);
+      xq_offsets[0] = 0;
+      constexpr_for<0, N>([&](auto i)
+      {
+         const auto input = std::get<i>(inputs);
+         xq_offsets[i + 1] = nqp * input.size_on_qp * ctx.nentities;
+      });
+      xq_offsets.PartialSum();
+      xq.Update(xq_offsets, Device::GetMemoryType());
+      xq.UseDevice(true);
+      xq = 0.0;
+      xq.SyncToBlocks();
+
+      yq_offsets.SetSize(M + 1);
+      yq_offsets[0] = 0;
+      constexpr_for<0, M>([&](auto i)
+      {
+         const auto output = std::get<i>(outputs);
+         yq_offsets[i.value + 1] = nqp * output.size_on_qp * ctx.nentities;
+      });
+      yq_offsets.PartialSum();
+      yq.Update(yq_offsets, Device::GetMemoryType());
+      yq.UseDevice(true);
+      yq = 0.0;
+      yq.SyncToBlocks();
+   }
+
+   //////////////////////////////////////////////////////////////////
+   void operator()(const std::vector<Vector *> &xe,
+                   std::vector<Vector *> &ye) const
+   {
+      NVTX_MARK_FUNCTION;
+      if (ctx.attr.Size() == 0) { return; }
+
+      // E -> Q
+      interpolate(input_to_infd, input_bases, xe, xq);
+      // Q -> Q
+      static_assert(
+         supports_tensor_array_qfunc<qfunc_t, inputs_t, outputs_t>::value,
+         "qfunc signature not supported by default backend Action");
+      call_qfunc(qfunc,
+                 xq,
+                 yq,
+                 gnqp,
+                 input_qlayouts,
+                 output_qlayouts,
+                 std::make_index_sequence<N> {},
+                 std::make_index_sequence<M> {});
+      // Q -> E
+      integrate(output_to_outfd, output_bases, yq, ye);
+   }
+};
+
+} // namespace mfem::future::device
+
+namespace mfem::future
+{
+
+struct DeviceBackend
+{
+   template<typename qfunc_t, typename inputs_t, typename outputs_t>
+   auto static MakeAction(const IntegratorContext &ctx,
+                          qfunc_t qfunc,
+                          inputs_t inputs,
+                          outputs_t outputs)
+   {
+      NVTX_MARK_FUNCTION;
+      return device::Action(ctx, qfunc, inputs, outputs);
+   }
+};
+
+} // namespace mfem::future

fem/dfem/backends/global_qf/action.hpp

@@ -0,0 +1,112 @@
+#pragma once
+
+#include "../util.hpp"
+#include "../../integrator_ctx.hpp"
+#include <utility>
+
+namespace mfem::future
+{
+
+namespace GlobalQFImpl
+{
+
+template<
+   typename qfunc_t,
+   typename inputs_t,
+   typename outputs_t,
+   size_t ninputs = std::tuple_size_v<inputs_t>,
+   size_t noutputs = std::tuple_size_v<outputs_t>>
+struct Action
+{
+   Action(
+      IntegratorContext ctx,
+      qfunc_t qfunc,
+      inputs_t inputs,
+      outputs_t outputs) :
+      ctx(ctx),
+      qfunc(std::move(qfunc)),
+      inputs(inputs),
+      outputs(outputs)
+   {
+      NVTX_MARK_FUNCTION;
+      create_fop_to_fd(inputs, ctx.infds, input_to_infd);
+      create_fop_to_fd(outputs, ctx.outfds, output_to_outfd);
+
+      check_consistency(inputs, input_to_infd, ctx.infds);
+      check_consistency(outputs, output_to_outfd, ctx.outfds);
+
+      create_fieldbases(inputs, input_to_infd, ctx.infds, ctx.ir, input_bases);
+      create_fieldbases(outputs, output_to_outfd, ctx.outfds, ctx.ir, output_bases);
+
+      create_qlayouts(inputs, ctx.in_qlayouts, input_qlayouts);
+      create_qlayouts(outputs, ctx.out_qlayouts, output_qlayouts);
+
+      const int nqp = ctx.ir.GetNPoints();
+      gnqp = nqp * ctx.nentities;
+
+      xq_offsets.SetSize(ninputs + 1);
+      xq_offsets[0] = 0;
+      constexpr_for<0, ninputs>([&](auto i)
+      {
+         const auto input = get<i>(inputs);
+         xq_offsets[i + 1] = nqp * input.size_on_qp * ctx.nentities;
+      });
+      xq_offsets.PartialSum();
+      xq.Update(xq_offsets);
+
+      yq_offsets.SetSize(noutputs + 1);
+      yq_offsets[0] = 0;
+      constexpr_for<0, noutputs>([&](auto i)
+      {
+         const auto output = get<i>(outputs);
+         yq_offsets[i + 1] = nqp * output.size_on_qp * ctx.nentities;
+      });
+      yq_offsets.PartialSum();
+      yq.Update(yq_offsets);
+   }
+
+   void operator()(
+      const std::vector<Vector *> &xe,
+      std::vector<Vector *> &ye) const
+   {
+      NVTX_MARK_FUNCTION;
+      if (ctx.attr.Size() == 0) { return; }
+
+      // E -> Q
+      interpolate(input_to_infd, input_bases, xe, xq);
+
+      // Q -> Q
+      static_assert(
+         detail::supports_tensor_array_qfunc<qfunc_t, inputs_t, outputs_t>::value,
+         "qfunc signature not supported by default backend Action");
+
+      detail::call_qfunc(
+         qfunc, xq, yq, gnqp, input_qlayouts, output_qlayouts,
+         std::make_index_sequence<ninputs> {},
+         std::make_index_sequence<noutputs> {});
+
+      // Q -> E
+      integrate(output_to_outfd, output_bases, yq, ye);
+   }
+
+   IntegratorContext ctx;
+   qfunc_t qfunc;
+   inputs_t inputs;
+   outputs_t outputs;
+
+   std::array<size_t, ninputs> input_to_infd;
+   std::array<size_t, noutputs> output_to_outfd;
+
+   std::array<FieldBasis, ninputs> input_bases;
+   std::array<FieldBasis, noutputs> output_bases;
+
+   std::array<std::vector<int>, ninputs>  input_qlayouts;
+   std::array<std::vector<int>, noutputs> output_qlayouts;
+
+   int gnqp = 0;
+   Array<int> xq_offsets, yq_offsets;
+   mutable BlockVector xq, yq;
+};
+
+}
+}

fem/dfem/backends/global_qf/derivative_action_enzyme.hpp

@@ -0,0 +1,131 @@
+#pragma once
+
+#include "fem/quadinterpolator.hpp"
+#include "../../integrator_ctx.hpp"
+#include "../util.hpp"
+#include <utility>
+
+namespace mfem::future
+{
+
+namespace GlobalQFImpl
+{
+
+template<
+   int derivative_id,
+   typename qfunc_t,
+   typename inputs_t,
+   typename outputs_t,
+   size_t ninputs = std::tuple_size_v<inputs_t>,
+   size_t noutputs = std::tuple_size_v<outputs_t>>
+struct DerivativeActionEnzyme
+{
+   DerivativeActionEnzyme(
+      IntegratorContext ctx,
+      qfunc_t &qfunc,
+      inputs_t inputs,
+      outputs_t outputs) :
+      ctx(ctx),
+      qfunc(qfunc),
+      inputs(inputs),
+      outputs(outputs)
+   {
+      create_fop_to_fd(inputs, ctx.infds, input_to_infd);
+      create_fop_to_fd(outputs, ctx.outfds, output_to_outfd);
+
+      check_consistency(inputs, input_to_infd, ctx.infds);
+      check_consistency(outputs, output_to_outfd, ctx.outfds);
+
+      create_fieldbases(inputs, input_to_infd, ctx.infds, ctx.ir, input_bases);
+      create_fieldbases(outputs, output_to_outfd, ctx.outfds, ctx.ir, output_bases);
+
+      create_qlayouts(inputs, ctx.in_qlayouts, input_qlayouts);
+      create_qlayouts(outputs, ctx.out_qlayouts, output_qlayouts);
+
+      const int nqp = ctx.ir.GetNPoints();
+      gnqp = nqp * ctx.nentities;
+
+      xq_offsets.SetSize(ninputs + 1);
+      xq_offsets[0] = 0;
+      constexpr_for<0, ninputs>([&](auto i)
+      {
+         const auto input = get<i>(inputs);
+         xq_offsets[i + 1] = nqp * input.size_on_qp * ctx.nentities;
+      });
+      xq_offsets.PartialSum();
+      xq.Update(xq_offsets);
+
+      yq_offsets.SetSize(noutputs + 1);
+      yq_offsets[0] = 0;
+      constexpr_for<0, noutputs>([&](auto i)
+      {
+         const auto output = get<i>(outputs);
+         yq_offsets[i + 1] = nqp * output.size_on_qp * ctx.nentities;
+      });
+      yq_offsets.PartialSum();
+      yq.Update(yq_offsets);
+
+      // For each dependent input in the dependency map we create a shadow
+      // memory variable at the quadrature point level.
+      const auto activity_map = detail::make_activity_map<derivative_id>(inputs);
+      shadow_xq_offsets.SetSize(ninputs + 1);
+      shadow_xq_offsets = 0;
+      constexpr_for<0, ninputs>([&](auto i)
+      {
+         if (activity_map[i])
+         {
+            shadow_xq_offsets[i + 1] =
+               xq_offsets[i + 1] - xq_offsets[i];;
+         }
+      });
+      shadow_xq_offsets.PartialSum();
+      shadow_xq.Update(shadow_xq_offsets);
+   }
+
+   void operator()(
+      const std::vector<Vector *> &xe,
+      const Vector *de,
+      std::vector<Vector *> &ye) const
+   {
+      if (ctx.attr.Size() == 0) { return; }
+      // E -> Q
+      interpolate(input_to_infd, input_bases, xe, xq);
+
+      const auto activity_map = detail::make_activity_map<derivative_id>(inputs);
+      interpolate(input_to_infd, input_bases, xe, shadow_xq, activity_map);
+
+      // Q -> Q
+      static_assert(
+         detail::supports_tensor_array_qfunc<qfunc_t, inputs_t, outputs_t>::value,
+         "qfunc signature not supported by default backend Action");
+
+      detail::enzyme_fwddiff<derivative_id, qfunc_t, inputs_t, outputs_t>(
+         qfunc, xq, shadow_xq, yq, gnqp, input_qlayouts, output_qlayouts,
+         std::make_index_sequence<ninputs> {},
+         std::make_index_sequence<noutputs> {});
+
+      // Q -> E
+      integrate(output_to_outfd, output_bases, yq, ye);
+   }
+
+   IntegratorContext ctx;
+   qfunc_t &qfunc;
+   inputs_t inputs;
+   outputs_t outputs;
+
+   std::array<size_t, ninputs> input_to_infd;
+   std::array<size_t, noutputs> output_to_outfd;
+
+   std::array<FieldBasis, ninputs> input_bases;
+   std::array<FieldBasis, noutputs> output_bases;
+
+   std::array<std::vector<int>, ninputs>  input_qlayouts;
+   std::array<std::vector<int>, noutputs> output_qlayouts;
+
+   int gnqp = 0;
+   Array<int> xq_offsets, shadow_xq_offsets, yq_offsets;
+   mutable BlockVector xq, shadow_xq, yq;
+};
+
+}
+}

fem/dfem/backends/global_qf/prelude.hpp

@@ -0,0 +1,42 @@
+#pragma once
+
+#include "action.hpp"
+#include "derivative_action_enzyme.hpp"
+
+namespace mfem::future
+{
+
+struct GlobalQFBackend
+{
+   template<
+      typename qfunc_t,
+      typename inputs_t,
+      typename outputs_t>
+   auto static MakeAction(
+      const IntegratorContext &ctx,
+      qfunc_t qfunc,
+      inputs_t inputs,
+      outputs_t outputs)
+   {
+      return GlobalQFImpl::Action(ctx, qfunc, inputs, outputs);
+   }
+
+   template<
+      int derivative_id,
+      typename qfunc_t,
+      typename inputs_t,
+      typename outputs_t>
+   auto static MakeDerivativeAction(
+      const IntegratorContext &ctx,
+      qfunc_t qfunc,
+      inputs_t inputs,
+      outputs_t outputs)
+   {
+      return GlobalQFImpl::DerivativeActionEnzyme<
+             derivative_id, qfunc_t, inputs_t, outputs_t>(
+                ctx, qfunc, inputs, outputs);
+   }
+
+};
+
+}

fem/dfem/backends/local_qf/action.hpp

@@ -0,0 +1,166 @@
+#pragma once
+
+#include "../util.hpp"
+#include "../../integrator_ctx.hpp"
+
+#include <utility>
+
+namespace mfem::future
+{
+
+namespace LocalQFImpl
+{
+
+template<
+   typename qfunc_t,
+   typename inputs_t,
+   typename outputs_t,
+   size_t ninputs = tuple_size<inputs_t>::value,
+   size_t noutputs = tuple_size<outputs_t>::value>
+struct Action
+{
+   Action(
+      IntegratorContext ctx,
+      qfunc_t qfunc,
+      inputs_t inputs,
+      outputs_t outputs) :
+      ctx(ctx),
+      qfunc(std::move(qfunc)),
+      inputs(inputs),
+      outputs(outputs)
+   {
+      create_fop_to_fd(inputs, ctx.infds, input_to_infd);
+      create_fop_to_fd(outputs, ctx.outfds, output_to_outfd);
+
+      check_consistency(inputs, input_to_infd, ctx.infds);
+      check_consistency(outputs, output_to_outfd, ctx.outfds);
+
+      const int nqp = ctx.ir.GetNPoints();
+
+      // Initialize DofToQuad maps for inputs
+      for_constexpr<ninputs>([&](auto i)
+      {
+         const auto &fd = ctx.infds[input_to_infd[i]];
+         std::visit([&](auto* space_ptr)
+         {
+            using T = std::decay_t<decltype(*space_ptr)>;
+            if constexpr (std::is_same_v<T, FiniteElementSpace> ||
+                          std::is_same_v<T, ParFiniteElementSpace>)
+            {
+               const auto *fe = space_ptr->GetTypicalFE();
+               input_dtq_maps[i] = &fe->GetDofToQuad(ctx.ir, DofToQuad::TENSOR);
+            }
+         }, fd.data);
+      });
+
+      // Initialize DofToQuad maps for outputs
+      for_constexpr<noutputs>([&](auto i)
+      {
+         const auto &fd = ctx.outfds[output_to_outfd[i]];
+         std::visit([&](auto* space_ptr)
+         {
+            using T = std::decay_t<decltype(*space_ptr)>;
+            if constexpr (std::is_same_v<T, FiniteElementSpace> ||
+                          std::is_same_v<T, ParFiniteElementSpace>)
+            {
+               const auto *fe = space_ptr->GetTypicalFE();
+               output_dtq_maps[i] = &fe->GetDofToQuad(ctx.ir, DofToQuad::TENSOR);
+            }
+         }, fd.data);
+      });
+   }
+
+   void operator()(
+      const std::vector<Vector *> &xe,
+      std::vector<Vector *> &ye) const
+   {
+      if (ctx.attr.Size() == 0) { return; }
+
+      // input_dtq_maps
+
+      // const auto B = (const real_t*)input_dtq_maps[0/*i*/].B;
+      // const auto G = (const real_t*)input_dtq_maps[0/*i*/].G;
+
+      //    dfem::forall<T_Q1D*T_Q1D*T_Q1D>([=] MFEM_HOST_DEVICE (int e, void *)
+      //    {
+      //       if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
+
+      //       constexpr int MQ1 = T_Q1D > 0 ? T_Q1D : 8;
+
+      //       MFEM_SHARED real_t sm0[MQ1][MQ1][MQ1][3];
+      //       MFEM_SHARED real_t sm1[MQ1][MQ1][MQ1][3];
+
+      //       low::regs3d_t<DIM, MQ1> reg;
+      //       const real_t *rd = dx_ptr;
+
+      //       MFEM_SHARED real_t sB[MQ1][MQ1], sG[MQ1][MQ1];
+      //       {
+      //          low::LoadMatrix(d1d, q1d, B, sB);
+      //          low::LoadMatrix(d1d, q1d, G, sG);
+      //          {
+      //             low::LoadDofs3d(e, d1d, XE, sm0);
+      //             low::Grad3d(d1d, q1d, sB, sG, sm0, sm1, reg);
+      //          }
+      //       }
+      //       // else if constexpr (is_identity_fop<field_operator_t>::value)   // Identity
+      //       {
+      //          // db1("Identity");
+      //          // rd = fields_e_ptr[input_to_field[i]];
+      //          // rd = dx_ptr;
+      //       }
+      //    }
+
+      //    MFEM_FOREACH_THREAD_DIRECT(qz,z,q1d)
+      //    {
+      //       MFEM_FOREACH_THREAD_DIRECT(qy,y,q1d)
+      //       {
+      //          MFEM_FOREACH_THREAD_DIRECT(qx,x,q1d)
+      //          {
+
+      //             auto args = decay_tuple<qf_param_ts> {};
+      //             get<0>(args) = as_tensor<real_t, 3>(&reg[qz][qy][qx][0]);
+      //             if constexpr (T_Q1D > 0)
+      //             {
+      //                get<1>(args) = as_tensor<real_t, 3, 3>(rd + 9*(qx*T_Q1D*T_Q1D + qy*T_Q1D + qz));
+      //             }
+      //             else
+      //             {
+      //                get<1>(args) = as_tensor<real_t, 3, 3>(rd + 9*(qx*q1d*q1d + qy*q1d + qz));
+      //             }
+      //             auto r = get<0>(apply(qfunc, args));
+      //             if constexpr (decltype(r)::ndim == 1)
+      //             {
+      //                as_tensor<real_t, 3>(&reg[qz][qy][qx][0]) = r;
+      //             }
+      //             else { static_assert(false); }
+      //          }
+      //       }
+      //    }
+      //    MFEM_SYNC_THREAD;
+      //    // Integrate
+      //    // if constexpr (is_gradient_fop<std::decay_t<output_fop_t>>::value) // Gradient
+      //    {
+      //       // const auto sB = reinterpret_cast<const real_t (*)[MQ1]>(Bo);
+      //       // const auto sG = reinterpret_cast<const real_t (*)[MQ1]>(Go);
+      //       low::GradTranspose3d(d1d, q1d, sB, sG, reg, sm1, sm0);
+      //       low::WriteDofs3d(d1d, 0, e, reg, YE);
+      //    }
+      // },
+      // num_entities, thread_blocks, 0, nullptr);
+   }
+
+
+   IntegratorContext ctx;
+   qfunc_t qfunc;
+   inputs_t inputs;
+   outputs_t outputs;
+
+   std::array<size_t, ninputs> input_to_infd;
+   std::array<size_t, noutputs> output_to_outfd;
+
+   std::array<const DofToQuad*, ninputs> input_dtq_maps;
+   std::array<const DofToQuad*, noutputs> output_dtq_maps;
+};
+
+}
+}

fem/dfem/backends/local_qf/prelude.hpp

@@ -0,0 +1,39 @@
+#pragma once
+
+#include "../../integrator_ctx.hpp"
+#include "action.hpp"
+
+namespace mfem::future
+{
+
+struct LocalQFBackend
+{
+   template<
+      typename qfunc_t,
+      typename inputs_t,
+      typename outputs_t>
+   auto static MakeAction(
+      const IntegratorContext &ctx,
+      qfunc_t qfunc,
+      inputs_t inputs,
+      outputs_t outputs)
+   {
+      return LocalQFImpl::Action(ctx, qfunc, inputs, outputs);
+   }
+
+   template<
+      int derivative_id,
+      typename qfunc_t,
+      typename inputs_t,
+      typename outputs_t>
+   auto static MakeDerivativeAction(
+      const IntegratorContext &ctx,
+      qfunc_t qfunc,
+      inputs_t inputs,
+      outputs_t outputs)
+   {
+      MFEM_ABORT("LocalQFBackend does not support derivative actions.");
+   }
+};
+
+}

fem/dfem/backends/util.hpp

@@ -0,0 +1,667 @@
+#pragma once
+
+
+#include <tuple>
+
+#include "../fem/quadinterpolator.hpp"
+#include "../util.hpp"
+#include "../integrator_ctx.hpp"
+
+#include "general/enzyme.hpp"
+#include "../../../linalg/tensor_arrays.hpp"
+
+namespace mfem::future
+{
+
+template <size_t N, size_t... Is>
+constexpr std::array<bool, N> all_true_impl(std::index_sequence<Is...>)
+{
+   return {{((void)Is, true)...}};
+}
+
+template <size_t N>
+constexpr std::array<bool, N> all_true()
+{
+   return all_true_impl<N>(std::make_index_sequence<N> {});
+}
+
+struct FieldBasis
+{
+   // E-vector -> Q-vector
+   std::function<void(const Vector &, Vector &)> forward;
+
+   // Q-vector -> E-vector
+   std::function<void(const Vector &, Vector &)> transpose;
+};
+
+inline FieldBasis FromQI(const QuadratureInterpolator *qi,
+                         QuadratureInterpolator::EvalFlags mode)
+{
+   return
+   {
+      [qi, mode](const Vector &xe, Vector &xq)
+      {
+         qi->SetOutputLayout(QVectorLayout::byVDIM);
+         if (mode == QuadratureInterpolator::VALUES)
+         {
+            qi->Values(xe, xq);
+         }
+         else
+         {
+            qi->Derivatives(xe, xq);
+         }
+      },
+      [qi, mode](const Vector &yq, Vector &ye)
+      {
+         Vector empty;
+         qi->SetOutputLayout(QVectorLayout::byVDIM);
+         if (mode == QuadratureInterpolator::VALUES)
+         {
+            qi->AddMultTranspose(QuadratureInterpolator::VALUES, yq, empty, ye);
+         }
+         else
+         {
+            qi->AddMultTranspose(QuadratureInterpolator::DERIVATIVES, empty, yq, ye);
+         }
+      }
+   };
+}
+
+// QuadratureFunction identity copy
+inline FieldBasis FromQF()
+{
+   return
+   {
+      [](const Vector &xe, Vector &xq) { xq = xe; },
+      [](const Vector &yq, Vector &ye) { ye = yq; }
+   };
+}
+
+// User-defined parameter space B
+inline FieldBasis FromPS(const Operator *B, const Operator *Bt)
+{
+   return
+   {
+      [B](const Vector &xe, Vector &xq) { B->Mult(xe, xq); },
+      [Bt](const Vector &yq, Vector &ye) { Bt->Mult(yq, ye); }
+   };
+}
+
+inline FieldBasis FieldBasisFromWeight(const IntegrationRule &ir)
+{
+   return
+   {
+      [&ir](const Vector &, Vector &xq)
+      {
+         const int nqp = ir.GetNPoints();
+         MFEM_ASSERT(xq.Size() % nqp == 0, "weight block has unexpected size");
+
+         const int ne = xq.Size() / nqp;
+         const real_t *wref = ir.GetWeights().HostRead();
+
+         for (int e = 0; e < ne; e++)
+         {
+            std::memcpy(xq.HostReadWrite() + e*nqp, wref, nqp*sizeof(real_t));
+         }
+      },
+      [](const Vector &, Vector &) { }
+   };
+}
+
+inline const FieldBasis GetFieldBasis(const FieldDescriptor &f,
+                                      const IntegrationRule &ir,
+                                      QuadratureInterpolator::EvalFlags mode)
+{
+   return std::visit([&ir, &mode](auto && arg) -> FieldBasis
+   {
+      using T = std::decay_t<decltype(arg)>;
+
+      if constexpr (std::is_same_v<T, const FiniteElementSpace *>)
+      {
+         return FromQI(arg->GetQuadratureInterpolator(ir), mode);
+      }
+      else if constexpr (std::is_same_v<T, const ParFiniteElementSpace *>)
+      {
+         return FromQI(arg->GetQuadratureInterpolator(ir), mode);
+      }
+      else if constexpr (std::is_same_v<T, const QuadratureFunction *>)
+      {
+         return FromQF();
+      }
+      else if constexpr (std::is_same_v<T, const ParameterSpace *>)
+      {
+         return FromPS(arg->GetB(), arg->GetBt());
+      }
+      else if constexpr (std::is_same_v<T, const IntegrationRule *>)
+      {
+         return FieldBasis{};
+      }
+      else
+      {
+         static_assert(dfem::always_false<T>, "internal error");
+      }
+   }, f.data);
+}
+
+template <typename fops_t, size_t nfops>
+inline void create_fieldbases(
+   fops_t &fops,
+   const std::array<size_t, nfops> &fop_to_fd,
+   const std::vector<FieldDescriptor> &fds,
+   const IntegrationRule &ir,
+   std::array<FieldBasis, nfops> &bases)
+{
+   constexpr_for<0, nfops>([&](auto i)
+   {
+      const auto fop = get<i>(fops);
+      using fop_t = std::decay_t<decltype(fop)>;
+
+      const auto fd = fds[fop_to_fd[i]];
+
+      constexpr QuadratureInterpolator::EvalFlags dummy_mode =
+         QuadratureInterpolator::VALUES;
+      if constexpr (is_identity_fop<fop_t>::value)
+      {
+         bases[i] = GetFieldBasis(fd, ir, dummy_mode);
+      }
+      else if constexpr (is_weight_fop<fop_t>::value)
+      {
+         bases[i] = FieldBasisFromWeight(ir);
+      }
+      else if constexpr (is_value_fop<fop_t>::value)
+      {
+         bases[i] = GetFieldBasis(fd, ir, QuadratureInterpolator::VALUES);
+      }
+      else if constexpr (is_gradient_fop<fop_t>::value)
+      {
+         bases[i] = GetFieldBasis(fd, ir, QuadratureInterpolator::DERIVATIVES);
+      }
+   });
+}
+
+template <typename fops_t, size_t nfops>
+inline void check_consistency(
+   fops_t &fops,
+   const std::array<size_t, nfops> &fop_to_fd,
+   const std::vector<FieldDescriptor> &fields)
+{
+   constexpr_for<0, nfops>([&](auto i)
+   {
+      const auto input = get<i>(fops);
+      using input_t = std::decay_t<decltype(input)>;
+
+      [[maybe_unused]] const auto fd = fields[fop_to_fd[i]];
+
+      if constexpr (is_identity_fop<input_t>::value)
+      {
+         MFEM_ASSERT(std::holds_alternative<const QuadratureFunction *>(fd.data),
+                     "Identity FieldOperator requested on non "
+                     "QuadratureFunction");
+      }
+      else if constexpr (is_weight_fop<input_t>::value)
+      {
+      }
+      else if constexpr (is_value_fop<input_t>::value)
+      {
+         MFEM_ASSERT(std::holds_alternative<const FiniteElementSpace *>(fd.data) ||
+                     std::holds_alternative<const ParFiniteElementSpace *>(fd.data) ||
+                     std::holds_alternative<const ParameterSpace *>(fd.data),
+                     "Value FieldOperator requested on non "
+                     "QuadratureFunction");
+      }
+      else if constexpr (is_gradient_fop<input_t>::value)
+      {
+         MFEM_ASSERT(std::holds_alternative<const FiniteElementSpace *>(fd.data) ||
+                     std::holds_alternative<const ParFiniteElementSpace *>(fd.data),
+                     "Value FieldOperator requested on non "
+                     "QuadratureFunction");
+      }
+   });
+}
+
+template <size_t ninputs>
+inline void interpolate(
+   const std::array<size_t, ninputs> &input_to_infd,
+   const std::array<FieldBasis, ninputs> &input_bases,
+   const std::vector<Vector *> &xe,
+   BlockVector &xq,
+   const std::array<bool, ninputs> &conditional = all_true<ninputs>())
+{
+   constexpr_for<0, ninputs>([&](auto i)
+   {
+      if (!conditional.empty() && !conditional[i]) { return; }
+
+      input_bases[i].forward(*xe[input_to_infd[i]], xq.GetBlock(i));
+   });
+}
+
+template <size_t noutputs>
+inline void integrate(
+   const std::array<size_t, noutputs> &output_to_outfd,
+   const std::array<FieldBasis, noutputs> &output_bases,
+   const BlockVector &yq,
+   std::vector<Vector *> &ye)
+{
+   for (auto v : ye) { *v = 0.0; }
+
+   constexpr_for<0, noutputs>([&](auto i)
+   {
+      output_bases[i].transpose(yq.GetBlock(i), *ye[output_to_outfd[i]]);
+   });
+}
+
+
+namespace detail
+{
+
+template <typename T>
+struct is_tensor_array : std::false_type {};
+
+template <typename scalar_t, int... Dims>
+struct is_tensor_array<tensor_array<scalar_t, Dims...>> : std::true_type {};
+
+template <typename T>
+struct is_tensor_array_mut : std::false_type {};
+
+template <typename scalar_t, int... Dims>
+struct is_tensor_array_mut<tensor_array<scalar_t, Dims...>> :
+                                                            std::bool_constant<!std::is_const_v<scalar_t>> {};
+
+
+template <typename ndarray_t>
+inline void set_layout_default(ndarray_t &a)
+{
+   if constexpr (ndarray_t::tensor_rank() == 0) { return; }
+
+   constexpr std::size_t nd = ndarray_t::rank();
+   constexpr std::size_t td = ndarray_t::tensor_rank();
+   std::array<std::size_t, nd + td> perm{};
+
+   for (std::size_t i = 0; i < td; i++) { perm[i] = nd + i; }
+   for (std::size_t i = 0; i < nd; i++) { perm[td + i] = i; }
+
+   a.set_layout(perm);
+}
+
+template <typename ndarray_t>
+inline void set_layout(ndarray_t& a, const std::vector<int>& layout)
+{
+   if constexpr (ndarray_t::tensor_rank() == 0) { return; }
+
+   constexpr std::size_t nd = ndarray_t::rank();
+   constexpr std::size_t td = ndarray_t::tensor_rank();
+   constexpr std::size_t N  = nd + td;
+
+   // missing means default
+   if (layout.empty()) { set_layout_default(a); return; }
+
+   MFEM_VERIFY(layout.size() == N,
+               "layout size mismatch: expected " << N << " got " << layout.size());
+
+   // TODO: make a version of set_layout that takes `std::vector<int>`
+   std::array<std::size_t, N> perm{};
+   for (std::size_t i = 0; i < N; i++)
+   {
+      MFEM_VERIFY(layout[i] >= 0, "layout index must be >=0");
+      perm[i] = static_cast<std::size_t>(layout[i]);
+   }
+
+   a.set_layout(perm);
+}
+
+/// Primary template: intentionally undefined — gives a clear error for unsupported types.
+template <typename T>
+struct tensor_array_traits;
+
+/// Matches tensor<scalar_t, sizes...>
+template <typename scalar_t, int... sizes>
+struct tensor_array_traits<tensor<scalar_t, sizes...>>
+{
+   using scalar_type = scalar_t;
+   template <std::size_t ndims>
+   using array_type = tensor_ndarray<scalar_t, ndims, sizes...>;
+};
+
+/// Matches tensor_ndarray<scalar_t, ndims, tensor_sizes...>
+template <typename scalar_t, int ndims, int... tensor_sizes>
+struct tensor_array_traits<tensor_ndarray<scalar_t, ndims, tensor_sizes...>>
+{
+   using scalar_type = scalar_t;
+   template <std::size_t N>
+   using array_type = tensor_ndarray<scalar_t, N, tensor_sizes...>;
+};
+
+/// Entry point: explicit tensor type T as template argument.
+template <typename T, typename ptr_scalar_t, typename... dyn_sizes_t>
+decltype(auto) make_tensor_array(ptr_scalar_t *ptr,
+                                 const std::vector<int>* layout,
+                                 dyn_sizes_t... dynamic_sizes)
+{
+   using traits = tensor_array_traits<T>;
+   using array_t = typename traits::template array_type<sizeof...(dynamic_sizes)>;
+   auto a = array_t(ptr, {std::size_t(dynamic_sizes)...});
+   if (layout) { set_layout(a, *layout); }
+   else        { set_layout_default(a); }
+   return a;
+}
+
+template <typename qfunc_t, typename inputs_t, typename outputs_t>
+struct supports_tensor_array_qfunc
+{
+   using qf_signature = typename get_function_signature<qfunc_t>::type;
+   using qf_param_ts = typename qf_signature::parameter_ts;
+
+   static constexpr int ninputs = std::tuple_size<inputs_t>::value;
+   static constexpr int noutputs = std::tuple_size<outputs_t>::value;
+   static constexpr int nparams = std::tuple_size<qf_param_ts>::value;
+
+   template <std::size_t... Is>
+   static constexpr bool InputsOk(std::index_sequence<Is...>)
+   {
+      return (is_tensor_array<std::remove_cv_t<std::remove_reference_t<
+              typename std::tuple_element<Is, qf_param_ts>::type>>>::value && ...);
+   }
+
+   template <std::size_t... Is>
+   static constexpr bool OutputsOk(std::index_sequence<Is...>)
+   {
+      return (is_tensor_array_mut<std::remove_cv_t<std::remove_reference_t<
+              typename std::tuple_element<ninputs + Is, qf_param_ts>::type>>>::value && ...);
+   }
+
+   static constexpr bool value =
+      (nparams == ninputs + noutputs) &&
+      InputsOk(std::make_index_sequence<ninputs> {}) &&
+      OutputsOk(std::make_index_sequence<noutputs> {});
+};
+
+template <typename qfunc_t, std::size_t... Is, std::size_t... Os>
+inline void call_qfunc(
+   const qfunc_t &qfunc,
+   const BlockVector &xq,
+   BlockVector &yq,
+   int gnqp,
+   const std::array<std::vector<int>, sizeof...(Is)>& in_layouts,
+   const std::array<std::vector<int>, sizeof...(Os)>& out_layouts,
+   std::index_sequence<Is...>,
+   std::index_sequence<Os...>)
+{
+   constexpr std::size_t ninputs = sizeof...(Is);
+
+   using qf_signature = typename get_function_signature<qfunc_t>::type;
+   using qf_param_ts = typename qf_signature::parameter_ts;
+
+   auto inputs = std::make_tuple(
+                    make_tensor_array<std::remove_cv_t<std::remove_reference_t<
+                    typename std::tuple_element<Is, qf_param_ts>::type>>>(
+                       xq.GetBlock(Is).Read(), &in_layouts[Is], gnqp)...);
+
+   auto outputs = std::make_tuple(
+                     make_tensor_array<std::remove_cv_t<std::remove_reference_t<
+                     typename std::tuple_element<ninputs + Os, qf_param_ts>::type>>>(
+                        yq.GetBlock(Os).ReadWrite(), &out_layouts[Os], gnqp)...);
+
+   std::apply([&](auto&&... args)
+   {
+      qfunc(args...);
+   }, std::tuple_cat(inputs, outputs));
+}
+
+template <typename func_t, typename... arg_ts>
+MFEM_HOST_DEVICE inline
+auto qfunction_wrapper(const func_t &f, arg_ts...args)
+{
+   return f(args...);
+}
+
+template <std::size_t derivative_id, std::size_t I, typename Tuple, std::size_t... Is>
+constexpr std::array<bool, sizeof...(Is)>
+make_activity_array(std::index_sequence<Is...>)
+{
+   return { (std::decay_t< std::tuple_element_t<Is, Tuple>>::GetFieldId() == derivative_id)... };
+}
+
+template <std::size_t derivative_id, typename inputs_t, std::size_t... Is>
+constexpr auto make_activity_map_impl(std::index_sequence<Is...>)
+{
+   constexpr std::size_t N = sizeof...(Is);
+
+   if constexpr (N == 0)
+      return std::array<bool, 0> {};
+
+   return make_activity_array<derivative_id, 0, inputs_t>
+          (std::make_index_sequence<N> {});
+}
+
+template <std::size_t derivative_id, typename inputs_t>
+constexpr auto make_activity_map(inputs_t)
+{
+   return make_activity_map_impl<derivative_id, inputs_t>(
+             std::make_index_sequence<std::tuple_size_v<inputs_t>> {});
+}
+
+namespace enzyme_detail
+{
+
+template <auto wrapper_fn, typename qf_return_t, typename... AccArgs>
+__attribute__((always_inline)) inline void
+do_enzyme_call(AccArgs... acc)
+{
+#ifdef MFEM_USE_ENZYME
+   __enzyme_fwddiff<qf_return_t>(wrapper_fn, acc...);
+#endif
+}
+
+template <auto wrapper_fn, typename qf_return_t,
+          size_t CurO, size_t NO,
+          typename primals_t, typename derivs_t,
+          typename... AccArgs>
+__attribute__((always_inline)) inline void
+process_outputs(primals_t &primals, derivs_t &derivs, AccArgs... acc)
+{
+   if constexpr (CurO == NO)
+   {
+      do_enzyme_call<wrapper_fn, qf_return_t>(acc...);
+   }
+   else
+   {
+      process_outputs<wrapper_fn, qf_return_t, CurO + 1, NO>(
+         primals, derivs,
+         acc...,
+         enzyme_dupnoneed,
+         &std::get<CurO>(primals),
+         &std::get<CurO>(derivs));
+   }
+}
+
+template <auto wrapper_fn, typename qf_return_t,
+          size_t CurI, size_t NI, bool... ActivityMap,
+          typename inputs_t, typename shadows_t,
+          typename primals_t, typename derivs_t,
+          typename... AccArgs>
+__attribute__((always_inline)) inline void
+process_inputs(inputs_t &inputs, shadows_t &shadows,
+               primals_t &primals, derivs_t &derivs,
+               AccArgs... acc)
+{
+   if constexpr (CurI == NI)
+   {
+      constexpr size_t NO = std::tuple_size_v<primals_t>;
+      process_outputs<wrapper_fn, qf_return_t, 0, NO>(
+         primals, derivs, acc...);
+   }
+   else
+   {
+      constexpr bool active =
+         std::array<bool, sizeof...(ActivityMap)> {ActivityMap...} [CurI];
+
+      if constexpr (active)
+      {
+         std::cout << "Input[" << CurI << "]: ACTIVE (enzyme_dup)\n"
+                   << "  primal ptr type:  "
+                   << get_type_name<decltype(&std::get<CurI>(inputs))>() << "\n"
+                   << "  shadow ptr type:  "
+                   << get_type_name<decltype(&std::get<CurI>(shadows))>() << "\n";
+      }
+      else
+      {
+         std::cout << "Input[" << CurI << "]: INACTIVE (enzyme_const)\n"
+                   << "  primal ptr type:  "
+                   << get_type_name<decltype(&std::get<CurI>(inputs))>() << "\n";
+      }
+
+      if constexpr (active)
+      {
+         process_inputs<wrapper_fn, qf_return_t, CurI + 1, NI, ActivityMap...>(
+            inputs, shadows, primals, derivs,
+            acc...,
+            enzyme_dup,
+            &std::get<CurI>(inputs),
+            &std::get<CurI>(shadows));
+      }
+      else
+      {
+         process_inputs<wrapper_fn, qf_return_t, CurI + 1, NI, ActivityMap...>(
+            inputs, shadows, primals, derivs,
+            acc...,
+            enzyme_const,
+            &std::get<CurI>(inputs));
+      }
+   }
+}
+
+} // namespace enzyme_detail
+
+template <size_t derivative_id, typename qfunc_t, typename inputs_t, typename outputs_t,
+          std::size_t... Is, std::size_t... Os>
+inline void enzyme_fwddiff(
+   qfunc_t &qfunc,
+   const BlockVector &xq,
+   const BlockVector &shadow_xq,
+   BlockVector &yq,
+   const int &gnqp,
+   const std::array<std::vector<int>, sizeof...(Is)>& in_layouts,
+   const std::array<std::vector<int>, sizeof...(Os)>& out_layouts,
+   std::index_sequence<Is...>,
+   std::index_sequence<Os...>)
+{
+#ifdef MFEM_USE_ENZYME
+   constexpr std::size_t ninputs  = sizeof...(Is);
+   constexpr std::size_t noutputs = sizeof...(Os);
+
+   using qf_signature = typename get_function_signature<qfunc_t>::type;
+   using qf_param_ts  = typename qf_signature::parameter_ts;
+   using qf_return_t  = typename qf_signature::return_t;
+
+   constexpr auto activity_map = make_activity_map<derivative_id>(inputs_t{});
+   static_assert(activity_map.size() == ninputs, "activity map size mismatch");
+
+   std::cout << "activity_map: ";
+   for (const auto &v : activity_map)
+   {
+      std::cout << v << " ";
+   }
+   std::cout << "\n";
+
+   auto inputs = std::make_tuple(
+                    make_tensor_array<std::remove_cv_t<std::remove_reference_t<
+                    typename std::tuple_element<Is, qf_param_ts>::type>>>(
+                       xq.GetBlock(Is).Read(), &in_layouts[Is], gnqp)...);
+
+   auto shadows = std::make_tuple(
+                     make_tensor_array<std::remove_cv_t<std::remove_reference_t<
+                     typename std::tuple_element<Is, qf_param_ts>::type>>>(
+                        shadow_xq.GetBlock(Is).Read(), &in_layouts[Is], gnqp)...);
+
+   std::array<Vector, noutputs> primal_storage;
+   ((primal_storage[Os].SetSize(yq.GetBlock(Os).Size())), ...);
+
+   auto primals_out = std::make_tuple(
+                         make_tensor_array<std::remove_cv_t<std::remove_reference_t<
+                         typename std::tuple_element<ninputs + Os, qf_param_ts>::type>>>(
+                            primal_storage[Os].ReadWrite(), &out_layouts[Os], gnqp)...);
+
+   auto derivs_out = std::make_tuple(
+                        make_tensor_array<std::remove_cv_t<std::remove_reference_t<
+                        typename std::tuple_element<ninputs + Os, qf_param_ts>::type>>>(
+                           yq.GetBlock(Os).ReadWrite(), &out_layouts[Os], gnqp)...);
+
+   using wrapper_fn_t = qf_return_t (*)(
+                           const qfunc_t &,
+                           std::remove_reference_t<decltype(std::get<Is>(inputs))>...,
+                           std::remove_reference_t<decltype(std::get<Os>(primals_out))>...);
+
+   constexpr wrapper_fn_t wrapper_fn =
+      qfunction_wrapper<qfunc_t,
+      std::remove_reference_t<decltype(std::get<Is>(inputs))>...,
+      std::remove_reference_t<decltype(std::get<Os>(primals_out))>...>;
+
+   // wrapper_fn travels as a non-type template parameter throughout without
+   // being stored.
+   enzyme_detail::process_inputs<
+   wrapper_fn,
+   qf_return_t,
+   0,
+   ninputs,
+   activity_map[Is]...
+   >(inputs, shadows,
+     primals_out, derivs_out,
+     enzyme_const, &qfunc // seed: qfunc is always inactive
+    );
+
+#else
+   MFEM_ABORT("enzyme_fwddiff requires MFEM_USE_ENZYME");
+#endif
+}
+
+} // namespace detail
+
+// Create quadrature function fop to fields map
+template <typename fops_t, size_t N = std::tuple_size_v<fops_t>, size_t M>
+void create_fop_to_fd(const fops_t &fops,
+                      const std::vector<FieldDescriptor> &fields,
+                      std::array<size_t, M> &fop_to_fd)
+{
+   static_assert(N == M, "sizes must match");
+   constexpr_for<0, N>([&](auto i)
+   {
+      const auto fop = get<i>(fops);
+      fop_to_fd[i] = std::numeric_limits<size_t>::max();
+      for (size_t j = 0; j < fields.size(); j++)
+      {
+         // TODO: output.GetFieldId() should probably store/return size_t
+         if (static_cast<int>(fields[j].id) == fop.GetFieldId())
+         {
+            fop_to_fd[i] = j;
+         }
+      }
+      // Handle Weight type. There is no FieldDescriptor for the weight.
+      // TODO: Create weight descriptor for the weight for internal use?
+      // TODO: this is a hack...
+      if (is_weight_fop<std::remove_cv_t<decltype(fop)>>::value)
+      {
+         fop_to_fd[i] = 0;
+      }
+      else if (fop_to_fd[i] == std::numeric_limits<size_t>::max())
+      {
+         MFEM_ABORT("not found");
+      }
+   });
+}
+
+template <typename fops_t, size_t nfops>
+void create_qlayouts(const fops_t &fops,
+                     const std::unordered_map<std::type_index, std::vector<int>> &a,
+                     std::array<std::vector<int>, nfops> &b)
+{
+   constexpr_for<0, nfops>([&](auto i)
+   {
+      using fop_t =
+         std::remove_cv_t<std::remove_reference_t<decltype(get<i>(fops))>>;
+      auto it = a.find(std::type_index(typeid(fop_t)));
+      if (it != a.end()) { b[i] = it->second; }
+      else               { b[i].clear(); }
+   });
+}
+
+}

fem/dfem/doperator.cpp

@@ -11,44 +11,122 @@
 
 #include "doperator.hpp"
 
+#include <algorithm>
+
 #ifdef MFEM_USE_MPI
 
 using namespace mfem;
 using namespace mfem::future;
 
-void DifferentiableOperator::SetParameters(std::vector<Vector *> p) const
+DifferentiableOperator::DifferentiableOperator(
+   int height, int width,
+   const std::vector<FieldDescriptor> &infds,
+   const std::vector<FieldDescriptor> &outfds,
+   const ParMesh &mesh) :
+   Operator(height, width),
+   mesh(mesh),
+   infds(infds),
+   outfds(outfds)
 {
-   MFEM_ASSERT(parameters.size() == p.size(),
-               "number of parameters doesn't match descriptors");
-   for (size_t i = 0; i < parameters.size(); i++)
+   NVTX_MARK_FUNCTION;
+   unionfds.clear();
+   unionfds.insert(unionfds.end(), infds.begin(), infds.end());
+   unionfds.insert(unionfds.end(), outfds.begin(), outfds.end());
+   std::sort(unionfds.begin(), unionfds.end());
+   auto last = std::unique(unionfds.begin(), unionfds.end());
+   unionfds.erase(last, unionfds.end());
+
+   infields_l.resize(infds.size());
+   for (size_t i = 0; i < infds.size(); i++)
    {
-      p[i]->Read();
-      parameters_l[i] = *p[i];
+      infields_l[i] = new Vector(GetVSize(infds[i]));
    }
+
+   infields_e.resize(infds.size());
 }
 
-DifferentiableOperator::DifferentiableOperator(
-   const std::vector<FieldDescriptor> &solutions,
-   const std::vector<FieldDescriptor> &parameters,
-   const ParMesh &mesh) :
-   mesh(mesh),
-   solutions(solutions),
-   parameters(parameters)
+void DifferentiableOperator::SetMultLevel(MultLevel level)
 {
-   fields.resize(solutions.size() + parameters.size());
-   fields_e.resize(fields.size());
-   solutions_l.resize(solutions.size());
-   parameters_l.resize(parameters.size());
+   mult_level = level;
+}
 
-   for (size_t i = 0; i < solutions.size(); i++)
+void DifferentiableOperator::Mult(const Vector &x, Vector &y) const
+{
+   NVTX_MARK_FUNCTION;
+   MFEM_ASSERT(!action_callbacks.empty(),
+               "no integrators have been set");
+
+   MFEM_ASSERT(dynamic_cast<const BlockVector*>(&x),
+               "x needs to be a BlockVector");
+
+   MFEM_ASSERT(dynamic_cast<const BlockVector*>(&y),
+               "y needs to be a BlockVector");
+
+   const auto &bx = static_cast<const BlockVector &>(x);
+   auto &by = static_cast<BlockVector &>(y);
+
+   Mult(bx, by);
+}
+
+void DifferentiableOperator::DisableTensorProductStructure(bool disable)
+{
+   use_tensor_product_structure = !disable;
+}
+
+std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetDerivative(
+   size_t derivative_id, const Vector &x)
+{
+   MFEM_ASSERT(derivative_action_callbacks.find(derivative_id) !=
+               derivative_action_callbacks.end(),
+               "no derivative action has been found for ID " << derivative_id);
+
+   const size_t dfidx = FindIdx(derivative_id, infds);
+
+   // Get transpose callbacks if available, otherwise pass empty vector
+   std::vector<derivative_action_t> transpose_callbacks;
+   auto it = daction_transpose_callbacks.find(derivative_id);
+   if (it != daction_transpose_callbacks.end())
    {
-      fields[i] = solutions[i];
+      transpose_callbacks = it->second;
    }
 
-   for (size_t i = 0; i < parameters.size(); i++)
+   return std::make_shared<DerivativeOperator>(
+             height,
+             GetTrueVSize(infds[dfidx]),
+             derivative_action_callbacks[derivative_id],
+             transpose_callbacks,
+             infds[dfidx],
+             x,
+             infds,
+             outfds);
+}
+
+std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetDerivative(
+   size_t derivative_id, const MultiVector &x)
+{
+   MFEM_ASSERT(derivative_action_callbacks.find(derivative_id) !=
+               derivative_action_callbacks.end(),
+               "no derivative action has been found for ID " << derivative_id);
+
+   const size_t dfidx = FindIdx(derivative_id, infds);
+
+   // Get transpose callbacks if available, otherwise pass empty vector
+   std::vector<derivative_action_t> transpose_callbacks;
+   auto it = daction_transpose_callbacks.find(derivative_id);
+   if (it != daction_transpose_callbacks.end())
    {
-      fields[i + solutions.size()] = parameters[i];
+      transpose_callbacks = it->second;
    }
+
+   return std::make_shared<DerivativeOperator>(
+             height,
+             GetTrueVSize(infds[dfidx]),
+             derivative_action_callbacks[derivative_id],
+             transpose_callbacks,
+             infds[dfidx],
+             x,
+             infds,
+             outfds);
 }
 
 #endif // MFEM_USE_MPI

fem/dfem/fielddescriptor.hpp

@@ -0,0 +1,63 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../fespace.hpp"
+#include "parameterspace.hpp"
+
+namespace mfem::future
+{
+
+/// @brief FieldDescriptor struct
+///
+/// This struct is used to store information about a field.
+struct FieldDescriptor
+{
+   using data_variant_t =
+      std::variant<const FiniteElementSpace *,
+      const ParFiniteElementSpace *,
+      const QuadratureFunction *,
+      const ParameterSpace *>;
+
+   /// Field ID
+   std::size_t id;
+
+   /// Field variant
+   data_variant_t data;
+
+   /// Default constructor
+   FieldDescriptor() :
+      id(SIZE_MAX), data(data_variant_t{}) {}
+
+   /// Constructor
+   template <typename T>
+   FieldDescriptor(std::size_t field_id, const T* v) :
+      id(field_id), data(v) {}
+
+   bool operator==(const FieldDescriptor& other) const
+   {
+      return id == other.id;
+   }
+
+   bool operator<(const FieldDescriptor& other) const
+   {
+      return id < other.id;
+   }
+
+   friend void swap(FieldDescriptor& a, FieldDescriptor& b)
+   {
+      using std::swap;
+      swap(a.id, b.id);
+      swap(a.data, b.data);
+   }
+};
+
+}

fem/dfem/integrator_ctx.hpp

@@ -0,0 +1,26 @@
+#pragma once
+
+#include <typeindex>
+#include <unordered_map>
+#include <vector>
+
+#include "fielddescriptor.hpp"
+
+namespace mfem::future
+{
+
+struct IntegratorContext
+{
+   const ParMesh &mesh;
+   const Array<int> *elem_attr;
+   Array<int> attr;
+   int nentities;
+   const std::vector<FieldDescriptor> &infds;
+   const std::vector<FieldDescriptor> &outfds;
+   const std::vector<FieldDescriptor> &unionfds;
+   const IntegrationRule &ir;
+   std::unordered_map<std::type_index, std::vector<int>> &in_qlayouts;
+   std::unordered_map<std::type_index, std::vector<int>> &out_qlayouts;
+};
+
+}

fem/dfem/parameterspace.hpp

@@ -43,7 +43,7 @@ public:
    /// Get spatial dimension
    ///
    /// returns always 1.
-   int Dimension() const
+   constexpr int Dimension() const
    {
       return 1;
    }
@@ -74,11 +74,14 @@ public:
       return elem_restr.get();
    }
 
+   virtual const Operator* GetB() const = 0;
+
+   virtual const Operator* GetBt() const = 0;
+
 protected:
    int vdim;
    DofToQuad dtq;
-   mutable std::unique_ptr<Operator> prolongation;
-   mutable std::unique_ptr<Operator> elem_restr;
+   mutable std::unique_ptr<Operator> prolongation, elem_restr, B, Bt;
 };
 
 /// @brief Uniform parameter space

fem/dfem/tuo_rocm_dfem_build_instructions.md

@@ -0,0 +1,49 @@
+export LC_USER=andrej1
+module load rocmcc/6.3.1-cce-19.0.0-magic cmake/3.29.2
+
+export MPICH_CC=amdclang
+export MPICH_CXX=amdclang++
+export ROCM_PATH=/opt/rocm-6.3.1
+export LLVM_DIR=$ROCM_PATH/lib/llvm
+export MPI_DIR=/usr/tce/packages/cray-mpich/cray-mpich-8.1.32-rocmcc-6.3.1-cce-19.0.0-magic
+
+export CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH:$ROCM_PATH/lib/cmake/hip:$ROCM_PATH/lib/cmake/hipblas:$ROCM_PATH/lib/cmake/hipblas-common:$ROCM_PATH/lib/cmake/hipsparse:$ROCM_PATH/lib/cmake/rocsparse:$ROCM_PATH/lib/cmake/rocrand
+
+export BASE_DIR=/usr/workspace/$LC_USER/dfem-tuo-magic
+export LOCAL_DIR=/usr/workspace/$LC_USER/dfem-tuo-magic/local
+mkdir -p $LOCAL_DIR
+export PATH=$LOCAL_DIR/bin:$PATH
+cd $BASE_DIR
+
+## Enzyme
+git clone --depth 1 https://github.com/EnzymeAD/Enzyme.git
+pushd Enzyme/enzyme
+CC=amdclang CXX=amdclang++ cmake -B build -DLLVM_DIR=$LLVM_DIR -DCMAKE_INSTALL_PREFIX=$LOCAL_DIR
+cmake --build build -j && cmake --install build
+popd
+
+## hypre
+curl https://github.com/hypre-space/hypre/archive/refs/tags/v2.32.0.tar.gz -o hypre-v2.32.0.tar.gz -L
+tar xzf hypre-v2.32.0.tar.gz
+pushd hypre-2.32.0/src
+CC=mpicc CXX=mpicxx CXXFLAGS="std=c++17 -fPIC" CFLAGS="-fPIC" ROCM_PATH=$ROCM_PATH ./configure --disable-fortran --prefix=$LOCAL_DIR --with-MPI-libs="mpi mpich" --with-MPI-lib-dirs=$MPI_DIR/lib --with-MPI-include=$MPI_DIR/include --enable-shared --with-hip
+make -j install
+popd
+
+## metis
+curl -OL https://github.com/mfem/tpls/raw/gh-pages/parmetis-4.0.3.tar.gz
+tar xzf parmetis-4.0.3.tar.gz
+pushd parmetis-4.0.3
+cmake -B build -DCMAKE_CXX_FLAGS="-fPIC" -DCMAKE_C_FLAGS="-fPIC" -DGKLIB_PATH=$BASE_DIR/parmetis-4.0.3/metis/GKlib -DMETIS_PATH=$BASE_DIR/parmetis-4.0.3/metis -DCMAKE_INSTALL_PREFIX=$LOCAL_DIR -DSHARED=1 -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx
+cmake --build build -j && cmake --install build
+popd
+pushd parmetis-4.0.3/metis
+cmake -B build -DCMAKE_CXX_FLAGS="-fPIC" -DCMAKE_C_FLAGS="-fPIC" -DGKLIB_PATH=$BASE_DIR/parmetis-4.0.3/metis/GKlib -DCMAKE_INSTALL_PREFIX=$LOCAL_DIR -DSHARED=1 -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx
+cmake --build build -j && cmake --install build
+popd
+
+git clone https://github.com/mfem/mfem.git
+git switch dfem-phase1-dev
+pushd mfem
+CXX=mpicxx cmake -B build-opt -DCMAKE_BUILD_TYPE=Release -DMFEM_USE_HIP=ON -DCMAKE_HIP_ARCHITECTURES="gfx942" -DCMAKE_HIP_PLATFORM="amd"
+cmake --build build-opt -j

fem/dfem/tuo_rocm_dfem_user.cmake

@@ -0,0 +1,31 @@
+if (NOT CMAKE_BUILD_TYPE)
+  set(CMAKE_BUILD_TYPE "Release" CACHE STRING
+      "Build type: Debug, Release, RelWithDebInfo, or MinSizeRel." FORCE)
+endif()
+
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+set(CMAKE_CXX_STANDARD 17)
+# set(CMAKE_CXX_FLAGS "--save-temps -Rpass-analysis=kernel-resource-usage -mllvm -amdgpu-early-inline-all=true -mllvm -amdgpu-function-calls=false")
+
+set(MFEM_PRECISION "double" CACHE STRING
+    "Floating-point precision to use: single, or double")
+
+option(BUILD_SHARED_LIBS "Enable shared library build of MFEM" ON)
+option(MFEM_USE_MPI "Enable MPI parallel build" ON)
+option(MFEM_USE_METIS "Enable METIS usage" ${MFEM_USE_MPI})
+option(MFEM_USE_ENZYME "Enable Enzyme" ON)
+option(MFEM_USE_HIP "Enable HIP" ON)
+
+set(MFEM_MPI_NP 4 CACHE STRING "Number of processes used for MPI tests")
+
+option(MFEM_ENABLE_TESTING ON)
+
+set(HIP_ARCH "gfx942" CACHE STRING "Target HIP architecture.")
+
+# Make sure all dirs are absolute
+set(ENZYME_DIR "/usr/workspace/andrej1/dfem-tuo-magic/local/cmake/Enzyme" CACHE PATH "Path to the Enzyme library.")
+set(HYPRE_DIR "/usr/workspace/andrej1/dfem-tuo-magic/local" CACHE PATH "Path to the hypre library.")
+set(METIS_DIR "/usr/workspace/andrej1/dfem-tuo-magic/local" CACHE PATH "Path to the METIS library.")
+
+set(CMAKE_SKIP_PREPROCESSED_SOURCE_RULES ON) # Skip *.i rules
+set(CMAKE_SKIP_ASSEMBLY_SOURCE_RULES ON) # Skip *.s rules

fem/dfem/util.hpp

@@ -21,6 +21,7 @@
 #include <type_traits>
 #include <numeric>
 #include <iomanip>
+#include <typeindex>
 
 #include "../../general/communication.hpp"
 #include "../../general/forall.hpp"
@@ -28,12 +29,16 @@
 #include "../fe/fe_base.hpp"
 #include "../fespace.hpp"
 #include "../pfespace.hpp"
+#include "../qfunction.hpp"
 #include "../../mesh/mesh.hpp"
 #include "../../linalg/dtensor.hpp"
+#include "../quadinterpolator.hpp"
 
+#include "fielddescriptor.hpp"
 #include "fieldoperator.hpp"
 #include "parameterspace.hpp"
-#include "tuple.hpp"
+// #include "tuple.hpp"
+#include <tuple>
 
 namespace mfem::future
 {
@@ -103,11 +108,21 @@ template <typename lambda, typename arg_t>
 constexpr void for_constexpr_with_arg(lambda&& f, arg_t&& arg)
 {
    using indices =
-      std::make_index_sequence<tuple_size<std::remove_reference_t<arg_t>>::value>;
+      std::make_index_sequence<std::tuple_size_v<std::remove_reference_t<arg_t>>>;
    for_constexpr_with_arg(std::forward<lambda>(f), std::forward<arg_t>(arg),
                           indices{});
 }
 
+template <auto start, auto end, auto inc = 1, typename F>
+constexpr void constexpr_for(F&& f)
+{
+   if constexpr (start < end)
+   {
+      f(std::integral_constant<decltype(start), start>());
+      constexpr_for<start + inc, end, inc>(f);
+   }
+}
+
 template <std::size_t I, typename Tuple, std::size_t... Is>
 std::array<bool, sizeof...(Is)>
 make_dependency_array(const Tuple& inputs, std::index_sequence<Is...>)
@@ -116,7 +131,7 @@ make_dependency_array(const Tuple& inputs, std::index_sequence<Is...>)
 }
 
 template <typename... input_ts, std::size_t... Is>
-auto make_dependency_map_impl(tuple<input_ts...> inputs,
+auto make_dependency_map_impl(std::tuple<input_ts...> inputs,
                               std::index_sequence<Is...>)
 {
    constexpr std::size_t N = sizeof...(input_ts);
@@ -144,7 +159,7 @@ auto make_dependency_map_impl(tuple<input_ts...> inputs,
 // @returns an unordered_map where the keys are the field IDs and the values
 // are arrays of booleans indicating which inputs depend on each field ID.
 template <typename... input_ts>
-auto make_dependency_map(tuple<input_ts...> inputs)
+auto make_dependency_map(std::tuple<input_ts...> inputs)
 {
    return make_dependency_map_impl(inputs, std::index_sequence_for<input_ts...> {});
 }
@@ -413,8 +428,8 @@ void pretty_print_mpi(const mfem::Vector& v)
 
 
 template <typename ... Ts>
-constexpr auto decay_types(tuple<Ts...> const &)
--> tuple<std::remove_cv_t<std::remove_reference_t<Ts>>...>;
+constexpr auto decay_types(std::tuple<Ts...> const &)
+-> std::tuple<std::remove_cv_t<std::remove_reference_t<Ts>>...>;
 
 template <typename T>
 using decay_tuple = decltype(decay_types(std::declval<T>()));
@@ -425,7 +440,7 @@ template <typename output_t, typename... input_ts>
 struct FunctionSignature<output_t(input_ts...)>
 {
    using return_t = output_t;
-   using parameter_ts = tuple<input_ts...>;
+   using parameter_ts = std::tuple<input_ts...>;
 };
 
 template <class T> struct create_function_signature;
@@ -444,6 +459,21 @@ struct create_function_signature<output_t (*)(input_ts...)>
    using type = FunctionSignature<output_t(input_ts...)>;
 };
 
+template <typename...>
+using void_t = void;
+
+template <typename T, typename = void>
+struct get_function_signature
+{
+   using type = typename create_function_signature<T>::type;
+};
+
+template <typename T>
+struct get_function_signature<T, void_t<decltype(&T::operator())>>
+{
+   using type = typename create_function_signature<decltype(&T::operator())>::type;
+};
+
 template <typename T>
 constexpr int GetFieldId()
 {
@@ -544,32 +574,6 @@ constexpr auto filter_fields(const std::tuple<Ts...>& t)
              std::conditional_t<Ts::GetFieldId() != -1, std::tuple<Ts>, std::tuple<>> {}...);
 }
 
-/// @brief FieldDescriptor struct
-///
-/// This struct is used to store information about a field.
-struct FieldDescriptor
-{
-   using data_variant_t =
-      std::variant<const FiniteElementSpace *,
-      const ParFiniteElementSpace *,
-      const ParameterSpace *>;
-
-   /// Field ID
-   std::size_t id;
-
-   /// Field variant
-   data_variant_t data;
-
-   /// Default constructor
-   FieldDescriptor() :
-      id(SIZE_MAX), data(data_variant_t{}) {}
-
-   /// Constructor
-   template <typename T>
-   FieldDescriptor(std::size_t field_id, const T* v) :
-      id(field_id), data(v) {}
-};
-
 namespace dfem
 {
 template <class... T> constexpr bool always_false = false;
@@ -617,6 +621,7 @@ void forall(func_t f,
             int num_shmem = 0,
             real_t *shmem = nullptr)
 {
+   NVTX_MARK_FUNCTION;
    if (Device::Allows(Backend::CUDA_MASK) ||
        Device::Allows(Backend::HIP_MASK))
    {
@@ -772,6 +777,10 @@ int GetVSize(const FieldDescriptor &f)
       {
          return arg->GetVSize();
       }
+      else if constexpr (std::is_same_v<T, const QuadratureFunction *>)
+      {
+         return arg->Size();
+      }
       else if constexpr (std::is_same_v<T, const ParameterSpace *>)
       {
          return arg->GetVSize();
@@ -810,6 +819,10 @@ void GetElementVDofs(const FieldDescriptor &f, int el, Array<int> &vdofs)
       {
          arg->GetElementVDofs(el, vdofs);
       }
+      else if constexpr (std::is_same_v<T, const QuadratureFunction *>)
+      {
+         MFEM_ABORT("internal error");
+      }
       else if constexpr (std::is_same_v<T, const ParameterSpace *>)
       {
          MFEM_ABORT("internal error");
@@ -844,6 +857,10 @@ int GetTrueVSize(const FieldDescriptor &f)
       {
          return arg->GetTrueVSize();
       }
+      else if constexpr (std::is_same_v<T, const QuadratureFunction *>)
+      {
+         return arg->Size();
+      }
       else if constexpr (std::is_same_v<T, const ParameterSpace *>)
       {
          return arg->GetTrueVSize();
@@ -874,6 +891,10 @@ int GetVDim(const FieldDescriptor &f)
       {
          return arg->GetVDim();
       }
+      else if constexpr (std::is_same_v<T, const QuadratureFunction *>)
+      {
+         return arg->GetVDim();
+      }
       else if constexpr (std::is_same_v<T, const ParameterSpace *>)
       {
          return arg->GetVDim();
@@ -909,6 +930,10 @@ int GetDimension(const FieldDescriptor &f)
             return arg->GetMesh()->Dimension() - 1;
          }
       }
+      else if constexpr (std::is_same_v<T, const QuadratureFunction *>)
+      {
+         return arg->GetSpace()->GetMesh()->Dimension();
+      }
       else if constexpr (std::is_same_v<T, const ParameterSpace *>)
       {
          return arg->Dimension();
@@ -921,6 +946,36 @@ int GetDimension(const FieldDescriptor &f)
    }, f.data);
 }
 
+inline
+std::variant<const QuadratureInterpolator *, const Operator *>get_qinterp(
+   const FieldDescriptor &f,
+   const IntegrationRule &ir)
+{
+   return std::visit([&ir](auto && arg) -> const QuadratureInterpolator*
+   {
+      using T = std::decay_t<decltype(arg)>;
+      if constexpr (std::is_same_v<T, const FiniteElementSpace *> ||
+                    std::is_same_v<T, const ParFiniteElementSpace *>)
+      {
+         return arg->GetQuadratureInterpolator(ir);
+      }
+      else if constexpr (std::is_same_v<T, const QuadratureFunction *>)
+      {
+         // QuadratureFunction doesn't need a QuadratureInterpolator
+         return nullptr;
+      }
+      else if constexpr (std::is_same_v<T, const ParameterSpace *>)
+      {
+         return nullptr;
+      }
+      else
+      {
+         static_assert(dfem::always_false<T>, "internal error");
+      }
+
+      return nullptr; // Unreachable, but avoids compiler warning
+   }, f.data);
+}
 
 /// @brief Get the prolongation operator for a field descriptor.
 ///
@@ -937,6 +992,10 @@ const Operator *get_prolongation(const FieldDescriptor &f)
       {
          return arg->GetProlongationMatrix();
       }
+      else if constexpr (std::is_same_v<T, const QuadratureFunction *>)
+      {
+         return nullptr;
+      }
       else if constexpr (std::is_same_v<T, const ParameterSpace *>)
       {
          return arg->GetProlongationMatrix();
@@ -967,6 +1026,10 @@ const Operator *get_element_restriction(const FieldDescriptor &f,
       {
          return arg->GetElementRestriction(o);
       }
+      else if constexpr (std::is_same_v<T, const QuadratureFunction *>)
+      {
+         return nullptr;
+      }
       else if constexpr (std::is_same_v<T, const ParameterSpace *>)
       {
          return arg->GetElementRestriction(o);
@@ -1002,6 +1065,11 @@ const Operator *get_face_restriction(const FieldDescriptor &f,
       {
          return arg->GetFaceRestriction(o, ft, m);
       }
+      else if constexpr (std::is_same_v<T, const QuadratureFunction *>)
+      {
+         // QuadratureFunction does not support face restrictions
+         MFEM_ABORT("internal error");
+      }
       else if constexpr (std::is_same_v<T, const ParameterSpace *>)
       {
          // ParameterSpace does not support face restrictions
@@ -1054,10 +1122,12 @@ get_restriction_transpose(
    const ElementDofOrdering &o,
    const fop_t &fop)
 {
+   NVTX_MARK_FUNCTION;
    if constexpr (is_sum_fop<fop_t>::value)
    {
       auto RT = [=](const Vector &v_e, Vector &v_l)
       {
+         NVTX_MARK_FUNCTION;
          v_l += v_e;
       };
       return std::make_tuple(RT, 1);
@@ -1067,6 +1137,7 @@ get_restriction_transpose(
       const Operator *R = get_restriction<entity_t>(f, o);
       std::function<void(const Vector&, Vector&)> RT = [=](const Vector &x, Vector &y)
       {
+         NVTX_MARK_FUNCTION;
          R->AddMultTranspose(x, y);
       };
       return std::make_tuple(RT, R->Height());
@@ -1086,11 +1157,22 @@ get_restriction_transpose(
 inline
 void prolongation(const FieldDescriptor field, const Vector &x, Vector &field_l)
 {
+   NVTX_MARK_FUNCTION;
    const auto P = get_prolongation(field);
    field_l.SetSize(P->Height());
    P->Mult(x, field_l);
 }
 
+inline
+void prolongation_transpose(
+   const FieldDescriptor &field, const Vector &field_l, Vector &x)
+{
+   NVTX_MARK_FUNCTION;
+   const auto P = get_prolongation(field);
+   x.SetSize(P->Width());
+   P->MultTranspose(field_l, x);
+}
+
 /// @brief Apply the prolongation operator to a vector of fields.
 ///
 /// x is a long vector containing the data for all fields on tdofs and
@@ -1107,6 +1189,7 @@ void prolongation(const std::array<FieldDescriptor, N> fields,
                   const Vector &x,
                   std::array<Vector, M> &fields_l)
 {
+   NVTX_MARK_FUNCTION;
    int data_offset = 0;
    for (int i = 0; i < N; i++)
    {
@@ -1130,20 +1213,275 @@ void prolongation(const std::array<FieldDescriptor, N> fields,
 /// @param fields the array of field descriptors.
 /// @param x the input vector in tdofs.
 /// @param fields_l the array of output vectors in vdofs.
-inline
-void prolongation(const std::vector<FieldDescriptor> fields,
-                  const Vector &x,
-                  std::vector<Vector> &fields_l)
+// inline
+// void prolongation(const std::vector<FieldDescriptor> fields,
+//                   const Vector &x,
+//                   std::vector<Vector> &fields_l)
+// {
+//    int data_offset = 0;
+//    for (std::size_t i = 0; i < fields.size(); i++)
+//    {
+//       const auto P = get_prolongation(fields[i]);
+//       const int width = P->Width();
+//       const Vector x_i(const_cast<Vector&>(x), data_offset, width);
+//       fields_l[i].SetSize(P->Height());
+//       P->Mult(x_i, fields_l[i]);
+//       data_offset += width;
+//    }
+// }
+
+inline void prolongation(const std::vector<FieldDescriptor> fields,
+                         const BlockVector &x,
+                         std::vector<Vector *> &x_l)
 {
-   int data_offset = 0;
-   for (std::size_t i = 0; i < fields.size(); i++)
+   NVTX_MARK_FUNCTION;
+   MFEM_ASSERT(x.NumBlocks() == static_cast<int>(x_l.size()),
+               "error " << x.NumBlocks() << " vs " << x_l.size());
+   for (int i = 0; i < x.NumBlocks(); i++)
    {
       const auto P = get_prolongation(fields[i]);
-      const int width = P->Width();
-      const Vector x_i(const_cast<Vector&>(x), data_offset, width);
-      fields_l[i].SetSize(P->Height());
-      P->Mult(x_i, fields_l[i]);
-      data_offset += width;
+
+      // If nullptr, assume Identity.
+      if (P == nullptr)
+      {
+         NVTX_MARK("P(id)");
+         *x_l[i] = x.GetBlock(i);
+      }
+      else
+      {
+         const auto prolongation = get_prolongation(fields[i]);
+         MFEM_ASSERT(prolongation->Width() == x.GetBlock(i).Size(),
+                     "prolongation not applicable to given input data size " <<
+                     prolongation->Width() << " vs " << x.GetBlock(i).Size());
+         MFEM_ASSERT(prolongation->Height() == x_l[i]->Size(),
+                     "prolongation not applicable to given output data size " <<
+                     prolongation->Height() << " vs " << x_l[i]->Size());
+         NVTX_MARK("P(x_l)");
+         prolongation->Mult(x.GetBlock(i), *x_l[i]);
+      }
+   }
+   dbg("done");
+}
+
+inline void prolongation(const std::vector<FieldDescriptor> fields,
+                         const MultiVector &x,
+                         std::vector<Vector *> &x_l)
+{
+   NVTX_MARK_FUNCTION;
+   MFEM_ASSERT(x.NumBlocks() == static_cast<int>(x_l.size()),
+               "error " << x.NumBlocks() << " vs " << x_l.size());
+   for (int i = 0; i < x.NumBlocks(); i++)
+   {
+      const auto P = get_prolongation(fields[i]);
+
+      // If nullptr, assume Identity.
+      if (P == nullptr)
+      {
+         NVTX_MARK("!P #{} size:{}", i, x[i].Size());
+         x_l[i]->NewMemoryAndSize(x[i].GetMemory(), x[i].Size(), false);
+      }
+      else
+      {
+         const auto prolongation = get_prolongation(fields[i]);
+         MFEM_ASSERT(prolongation->Width() == x[i].Size(),
+                     "prolongation not applicable to given input data size " <<
+                     prolongation->Width() << " vs " << x[i].Size());
+         MFEM_ASSERT(prolongation->Height() == x_l[i]->Size(),
+                     "prolongation not applicable to given output data size " <<
+                     prolongation->Height() << " vs " << x_l[i]->Size());
+         NVTX_MARK("P(x_l[{}])",i);
+         prolongation->Mult(x[i], *x_l[i]);
+      }
+   }
+}
+
+inline
+void prolongation_transpose(
+   const std::vector<FieldDescriptor> fields,
+   const std::vector<Vector *> &x_l,
+   BlockVector &x)
+{
+   NVTX_MARK_FUNCTION;
+   MFEM_ASSERT(static_cast<int>(x_l.size()) == x.NumBlocks(),
+               "error " << x_l.size() << " vs " << x.NumBlocks());
+   for (size_t i = 0; i < x_l.size(); i++)
+   {
+      const auto P = get_prolongation(fields[i]);
+
+      // If nullptr, assume Identity.
+      if (P == nullptr)
+      {
+         NVTX_MARK("P^T(id)");
+         x.GetBlock(i) = *x_l[i];
+      }
+      else
+      {
+         MFEM_ASSERT(P->Height() == x_l[i]->Size(),
+                     "prolongation not applicable to given input data size " <<
+                     P->Height() << " vs " << x_l[i]->Size());
+         MFEM_ASSERT(P->Width() == x.GetBlock(i).Size(),
+                     "prolongation not applicable to given output data size " <<
+                     P->Width() << " vs " << x.GetBlock(i).Size());
+         NVTX_MARK("P^T(x_l)");
+         P->MultTranspose(*x_l[i], x.GetBlock(i));
+      }
+   }
+}
+
+inline
+void prolongation_transpose(
+   const std::vector<FieldDescriptor> fields,
+   const std::vector<Vector *> &x_l,
+   MultiVector &x)
+{
+   NVTX_MARK_FUNCTION;
+   MFEM_ASSERT(static_cast<int>(x_l.size()) == x.NumBlocks(),
+               "error " << x_l.size() << " vs " << x.NumBlocks());
+   for (size_t i = 0; i < x_l.size(); i++)
+   {
+      const auto P = get_prolongation(fields[i]);
+
+      // If nullptr, assume Identity.
+      if (P == nullptr)
+      {
+         NVTX_MARK("P^T(id)");
+         x[i] = *x_l[i];
+      }
+      else
+      {
+         MFEM_ASSERT(P->Height() == x_l[i]->Size(),
+                     "prolongation not applicable to given input data size " <<
+                     P->Height() << " vs " << x_l[i]->Size());
+         MFEM_ASSERT(P->Width() == x[i].Size(),
+                     "prolongation not applicable to given output data size " <<
+                     P->Width() << " vs " << x[i].Size());
+         NVTX_MARK("P^T(x_l)");
+         P->MultTranspose(*x_l[i], x[i]); // D2D copy 🔥🔥🔥
+      }
+   }
+}
+
+template <typename entity_t>
+void restriction(
+   const std::vector<FieldDescriptor> fields,
+   const std::vector<Vector *> &x_l,
+   std::vector<Vector *> &x_e)
+{
+   NVTX_MARK("fields.size(): {}", fields.size());
+   MFEM_ASSERT(x_l.size() == x_e.size(),
+               "internal error " << x_l.size() << " vs " << x_e.size());
+   for (size_t i = 0; i < fields.size(); i++)
+   {
+      NVTX("Field #{}", i);
+      int s = 0;
+      const auto R = get_restriction<entity_t>(
+                        fields[i], ElementDofOrdering::LEXICOGRAPHIC);
+
+      // If nullptr, assume Identity.
+      if (R == nullptr)
+      {
+         s = x_l[i]->Size();
+      }
+      else
+      {
+         s = R->Height();
+      }
+
+      // TODO
+      if (x_e[i] == nullptr)
+      {
+         NVTX("x_e[{}] null, size {}", i, s);
+         x_e[i] = new Vector(s);
+      }
+      x_e[i]->SetSize(s);
+
+      if (R == nullptr)
+      {
+         NVTX("!R #{} s:{} x_l:{}", i, s, x_l[i]->Size());
+         x_e[i]->NewMemoryAndSize(x_l[i]->GetMemory(), x_l[i]->Size(), false);
+      }
+      else
+      {
+         MFEM_ASSERT(R->Width() == x_l[i]->Size(),
+                     "restriction not applicable to given input data size " <<
+                     R->Width() << " vs " << x_l[i]->Size());
+         NVTX("R->Mult(#{})", i);
+         R->Mult(*x_l[i], *x_e[i]);
+      }
+   }
+}
+
+template <typename entity_t>
+void prepare_residual(
+   const std::vector<FieldDescriptor> &fields,
+   std::vector<Vector *> &r_e)
+{
+   NVTX_MARK_FUNCTION;
+   for (size_t i = 0; i < fields.size(); i++)
+   {
+      int s = 0;
+      if (std::holds_alternative<const QuadratureFunction *>(fields[i].data))
+      {
+         const auto fd = std::get<const QuadratureFunction *>(fields[i].data);
+         s = fd->Size();
+      }
+      else
+      {
+         const auto R = get_restriction<entity_t>(
+                           fields[i], ElementDofOrdering::LEXICOGRAPHIC);
+         s = R->Height();
+      }
+
+      // TODO
+      if (r_e[i] == nullptr)
+      {
+         r_e[i] = new Vector(s);
+      }
+      else
+      {
+         r_e[i]->SetSize(s);
+      }
+   }
+}
+
+template <typename entity_t>
+void restriction_transpose(
+   const std::vector<FieldDescriptor> &fields,
+   const std::vector<Vector *> &x_e,
+   std::vector<Vector *> &x_l)
+{
+   NVTX_MARK_FUNCTION;
+   for (size_t i = 0; i < fields.size(); i++)
+   {
+      int s = 0;
+      const auto R = get_restriction<entity_t>(
+                        fields[i], ElementDofOrdering::LEXICOGRAPHIC);
+      // TODO: if nullptr, assume Identity
+      if (R == nullptr)
+      {
+         s = x_e[i]->Size();
+      }
+      else
+      {
+         s = R->Width();
+      }
+
+      // TODO
+      if (x_l[i] == nullptr)
+      {
+         x_l[i] = new Vector(s);
+      }
+      x_l[i]->SetSize(s);
+
+      // TODO: if nullptr, assume Identity
+      if (R == nullptr)
+      {
+         x_l[i] = x_e[i];
+      }
+      else
+      {
+         R->MultTranspose(*x_e[i], *x_l[i]);
+      }
    }
 }
 
@@ -1220,6 +1558,7 @@ void restriction(const FieldDescriptor u,
                  Vector &field_e,
                  ElementDofOrdering ordering)
 {
+   NVTX_MARK_FUNCTION;
    const auto R = get_restriction<entity_t>(u, ordering);
    MFEM_ASSERT(R->Width() == u_l.Size(),
                "restriction not applicable to given data size");
@@ -1243,6 +1582,7 @@ void restriction(const std::vector<FieldDescriptor> u,
                  ElementDofOrdering ordering,
                  const int offset = 0)
 {
+   NVTX_MARK_FUNCTION;
    for (std::size_t i = 0; i < u.size(); i++)
    {
       const auto R = get_restriction<entity_t>(u[i], ordering);
@@ -1262,6 +1602,7 @@ void element_restriction(const std::array<FieldDescriptor, N> u,
                          ElementDofOrdering ordering,
                          const int offset = 0)
 {
+   NVTX_MARK_FUNCTION;
    for (int i = 0; i < N; i++)
    {
       const auto R = get_element_restriction(u[i], ordering);
@@ -1326,6 +1667,10 @@ const DofToQuad *GetDofToQuad(const FieldDescriptor &f,
             return &arg->GetTypicalTraceElement()->GetDofToQuad(ir, mode);
          }
       }
+      else if constexpr (std::is_same_v<T, const QuadratureFunction *>)
+      {
+         return nullptr;
+      }
       else if constexpr (std::is_same_v<T, const ParameterSpace *>)
       {
          return &arg->GetDofToQuad();
@@ -1433,12 +1778,12 @@ int GetSizeOnQP(const field_operator_t &, const FieldDescriptor &f)
 /// @tparam entity_t the entity type (see Entity).
 /// @returns an array mapping field operator types to field descriptor indices.
 template <typename entity_t, typename field_operator_ts>
-std::array<size_t, tuple_size<field_operator_ts>::value>
-create_descriptors_to_fields_map(
-   const std::vector<FieldDescriptor> &fields,
-   field_operator_ts &fops)
+std::array<size_t, std::tuple_size_v<field_operator_ts>>
+                                                      create_descriptors_to_fields_map(
+                                                         const std::vector<FieldDescriptor> &fields,
+                                                         field_operator_ts &fops)
 {
-   std::array<size_t, tuple_size<field_operator_ts>::value> map;
+   std::array<size_t, std::tuple_size_v<field_operator_ts>> map;
 
    auto find_id = [](const std::vector<FieldDescriptor> &fields, std::size_t i)
    {
@@ -1457,7 +1802,7 @@ create_descriptors_to_fields_map(
 
    auto f = [&](auto &fop, auto &map)
    {
-      if constexpr (std::is_same_v<std::decay_t<decltype(fop)>, Weight>)
+      if constexpr (is_weight_fop<std::decay_t<decltype(fop)>>::value)
       {
          // TODO-bug: stealing dimension from the first field
          fop.dim = GetDimension<entity_t>(fields[0]);
@@ -1482,7 +1827,7 @@ create_descriptors_to_fields_map(
       }
    };
 
-   for_constexpr<tuple_size<field_operator_ts>::value>([&](auto idx)
+   for_constexpr<std::tuple_size_v<field_operator_ts>>([&](auto idx)
    {
       f(get<idx>(fops), map[idx]);
    });
@@ -2006,7 +2351,7 @@ auto unpack_shmem(
    MFEM_SYNC_THREAD;
 
    // nvcc needs make_tuple to be fully qualified
-   return mfem::future::make_tuple(
+   return std::make_tuple(
              input_dtq_shmem, output_dtq_shmem, fields_shmem,
              input_shmem, residual_shmem, scratch_mem);
 }
@@ -2085,7 +2430,7 @@ auto unpack_shmem(
    MFEM_SYNC_THREAD;
 
    // nvcc needs make_tuple to be fully qualified
-   return mfem::future::make_tuple(
+   return std::make_tuple(
              input_dtq_shmem, output_dtq_shmem, fields_shmem,
              direction_shmem, input_shmem, shadow_shmem,
              residual_shmem, scratch_mem);
@@ -2244,7 +2589,7 @@ int accumulate_sizes_on_qp(
 template <
    typename entity_t,
    typename field_operator_ts,
-   std::size_t N = tuple_size<field_operator_ts>::value,
+   std::size_t N = std::tuple_size_v<field_operator_ts>,
    std::size_t... Is>
 std::array<DofToQuadMap, N> create_dtq_maps_impl(
    field_operator_ts &fops,
@@ -2342,5 +2687,25 @@ std::array<DofToQuadMap, num_fields> create_dtq_maps(
              std::make_index_sequence<num_fields> {});
 }
 
+struct QLayoutEntry
+{
+   std::type_index type;
+   std::vector<int> layout;
+
+   template <class Fop>
+   QLayoutEntry(Fop, std::initializer_list<int> idx) :
+      type(typeid(Fop)), layout(idx) {}
+};
+
+inline static void ExtractQLayouts(
+   const std::initializer_list<QLayoutEntry> entries,
+   std::unordered_map<std::type_index, std::vector<int>>& out)
+{
+   for (const auto& e : entries)
+   {
+      out[e.type] = e.layout;
+   }
+}
+
 } // namespace mfem::future
 #endif

fem/fem.hpp

@@ -52,7 +52,7 @@
 #include "bounds.hpp"
 #include "particleset.hpp"
 
-#include "dfem/doperator.hpp"
+// #include "dfem/doperator.hpp"
 
 #ifdef MFEM_USE_MPI
 #include "pfespace.hpp"

fem/kernel_dispatch.hpp

@@ -13,16 +13,122 @@
 #define MFEM_KERNEL_DISPATCH_HPP
 
 #include "../config/config.hpp"
-#include "kernel_reporter.hpp"
 #include "../general/hash_util.hpp"
-#include <unordered_map>
+#include "kernel_reporter.hpp"
+
 #include <tuple>
 #include <type_traits>
-#include <cstddef>
+#include <unordered_map>
 
 namespace mfem
 {
 
+namespace internal
+{
+template <typename... Types> struct KernelTypeList {};
+} // namespace internal
+
+template <typename... T> class KernelDispatchTable
+{
+public:
+   template <auto...> static inline void Add();
+};
+
+template <typename Kernels, typename Signature, typename... Params,
+          typename... OptParams>
+class KernelDispatchTable<Kernels, Signature,
+         internal::KernelTypeList<Params...>,
+         internal::KernelTypeList<OptParams...>>
+{
+   using TableType =
+      std::unordered_map<std::tuple<Params...>, Signature, TupleHasher>;
+   TableType table;
+
+   /// @brief Call function @a f with arguments @a args (perfect forwaring).
+   ///
+   /// Only valid when the function @a f is not a member function.
+   template <
+      typename F, typename... Args,
+      std::enable_if_t<std::is_pointer_v<F>, bool> = true>
+   static void Invoke(F f, Args &&...args)
+   {
+      f(std::forward<Args>(args)...);
+   }
+
+   /// @brief Calls member function @a f on object @a t with arguments @a args
+   /// (perfect forwarding).
+   ///
+   /// Only valid when @a f is a member function of class @a T.
+   template <typename F, typename T, typename... Args,
+             std::enable_if_t<std::is_member_function_pointer_v<F>,
+                              bool> = true>
+   static void Invoke(F f, T &&t, Args &&...args)
+   {
+      (t.*f)(std::forward<Args>(args)...);
+   }
+
+public:
+   /// @brief Run the kernel with the given dispatch parameters and arguments.
+   ///
+   /// If a compile-time specialized version of the kernel with the given
+   /// parameters has been registered, it will be called. Otherwise, the
+   /// fallback kernel will be called.
+   ///
+   /// If the kernel is a member function, then the first argument after @a
+   /// params should be the object on which it is called.
+   template <typename... Args>
+   static void Run(Params... params, Args &&...args)
+   {
+      const auto &table = Kernels::Get().table;
+      const std::tuple<Params...> key = std::make_tuple(params...);
+      const auto it = table.find(key);
+      if (it != table.end())
+      {
+         Invoke(it->second, std::forward<Args>(args)...);
+      }
+      else
+      {
+         KernelReporter::ReportFallback(Kernels::Get().kernel_name, params...);
+         Invoke(Kernels::Fallback(params...), std::forward<Args>(args)...);
+      }
+   }
+
+   // Version without optional parameters
+   template <auto... PARAMS>
+   static inline void Add()
+   {
+      std::tuple<Params...> param_tuple(PARAMS...);
+      Kernels::Get().table[param_tuple] =
+         Kernels::template Kernel<PARAMS..., OptParams{}...>();
+   };
+
+   /// Register a specialized kernel for dispatch.
+   template <auto... PARAMS>
+   struct Specialization
+   {
+      // Version without optional parameters
+      static void Add()
+      {
+         std::tuple<Params...> param_tuple(PARAMS...);
+         Kernels::Get().table[param_tuple] =
+            Kernels::template Kernel<PARAMS..., OptParams{}...>();
+      };
+      // Version with optional parameters
+      template <OptParams... OPT_PARAMS> struct Opt
+      {
+         static void Add()
+         {
+            std::tuple<Params...> param_tuple(PARAMS...);
+            Kernels::Get().table[param_tuple] =
+               Kernels::template Kernel<PARAMS..., OPT_PARAMS...>();
+         }
+      };
+   };
+
+   /// Return the dispatch map table
+   static const TableType &GetDispatchTable() { return Kernels::Get().table; }
+};
+
 // The MFEM_REGISTER_KERNELS macro registers kernels for runtime dispatch using
 // a dispatch map.
 //
@@ -47,8 +153,8 @@ namespace mfem
 #define MFEM_EXPAND(X) X // Workaround needed for MSVC compiler
 
 #define MFEM_REGISTER_KERNELS(KernelName, KernelType, ...)                     \
-   MFEM_EXPAND(MFEM_EXPAND(MFEM_REGISTER_KERNELS_N(__VA_ARGS__,2,1,))          \
-      (KernelName,KernelType,__VA_ARGS__))
+  MFEM_EXPAND(MFEM_EXPAND(MFEM_REGISTER_KERNELS_N(__VA_ARGS__, 2, 1, ))(       \
+      KernelName, KernelType, __VA_ARGS__))
 
 #define MFEM_REGISTER_KERNELS_N(_1, _2, N, ...) MFEM_REGISTER_KERNELS_##N
 
@@ -59,13 +165,13 @@ namespace mfem
 // Version of MFEM_REGISTER_KERNELS without any "optional" (non-dispatch)
 // parameters.
 #define MFEM_REGISTER_KERNELS_1(KernelName, KernelType, Params)                \
-   MFEM_REGISTER_KERNELS_(KernelName, KernelType, Params, (), Params)
+  MFEM_REGISTER_KERNELS_(KernelName, KernelType, Params, (), Params)
 
 // Version of MFEM_REGISTER_KERNELS without any optional (non-dispatch)
 // parameters (e.g. NBZ).
 #define MFEM_REGISTER_KERNELS_2(KernelName, KernelType, Params, OptParams)     \
-   MFEM_REGISTER_KERNELS_(KernelName, KernelType, Params, OptParams,           \
-                          (MFEM_PARAM_LIST Params, MFEM_PARAM_LIST OptParams))
+  MFEM_REGISTER_KERNELS_(KernelName, KernelType, Params, OptParams,            \
+                         (MFEM_PARAM_LIST Params, MFEM_PARAM_LIST OptParams))
 
 // P1 are the parameters, P2 are the optional (non-dispatch parameters), and P3
 // is the concatenation of P1 and P2. We need to pass it as a separate argument
@@ -79,110 +185,20 @@ namespace mfem
   public:                                                                      \
     const char *kernel_name = MFEM_KERNEL_NAME(KernelName);                    \
     using KernelSignature = KernelType;                                        \
+    using Params = ::mfem::internal::KernelTypeList<MFEM_PARAM_LIST P1>;       \
+    using Options = ::mfem::internal::KernelTypeList<MFEM_PARAM_LIST P2>;      \
     template <MFEM_PARAM_LIST P3> static KernelSignature Kernel();             \
     static MFEM_EXPORT KernelSignature Fallback(MFEM_PARAM_LIST P1);           \
     static MFEM_EXPORT KernelName &Get() {                                     \
       static KernelName table;                                                 \
       return table;                                                            \
     }                                                                          \
+    template<auto... Specials>                                                 \
+    static MFEM_EXPORT void Add(){                                             \
+      KernelDispatchTable<KernelName, KernelType, Params, Options>::           \
+      Add<Specials...>();                                                      \
+    }                                                                          \
   }
-
-namespace internal { template<typename... Types> struct KernelTypeList { }; }
-
-template<typename... T> class KernelDispatchTable { };
-
-template <typename Kernels,
-          typename Signature,
-          typename... Params,
-          typename... OptParams>
-class KernelDispatchTable<Kernels,
-         Signature,
-         internal::KernelTypeList<Params...>,
-         internal::KernelTypeList<OptParams...>>
-{
-   using TableType =
-      std::unordered_map<std::tuple<Params...>, Signature, TupleHasher>;
-   TableType table;
-
-   /// @brief Call function @a f with arguments @a args (perfect forwaring).
-   ///
-   /// Only valid when the function @a f is not a member function.
-   template <typename F, typename... Args,
-             typename std::enable_if<std::is_pointer<F>::value,bool>::type=true>
-   static void Invoke(F f, Args&&... args)
-   {
-      f(std::forward<Args>(args)...);
-   }
-
-   /// @brief Calls member function @a f on object @a t with arguments @a args
-   /// (perfect forwarding).
-   ///
-   /// Only valid when @a f is a member function of class @a T.
-   template <typename F, typename T, typename... Args,
-             typename std::enable_if<
-                std::is_member_function_pointer<F>::value,bool>::type=true>
-   static void Invoke(F f, T&& t, Args&&... args)
-   {
-      (t.*f)(std::forward<Args>(args)...);
-   }
-
-public:
-   /// @brief Run the kernel with the given dispatch parameters and arguments.
-   ///
-   /// If a compile-time specialized version of the kernel with the given
-   /// parameters has been registered, it will be called. Otherwise, the
-   /// fallback kernel will be called.
-   ///
-   /// If the kernel is a member function, then the first argument after @a
-   /// params should be the object on which it is called.
-   template<typename... Args>
-   static void Run(Params... params, Args&&... args)
-   {
-      const auto &table = Kernels::Get().table;
-      const std::tuple<Params...> key = std::make_tuple(params...);
-      const auto it = table.find(key);
-      if (it != table.end())
-      {
-         Invoke(it->second, std::forward<Args>(args)...);
-      }
-      else
-      {
-         KernelReporter::ReportFallback(Kernels::Get().kernel_name, params...);
-         Invoke(Kernels::Fallback(params...), std::forward<Args>(args)...);
-      }
-   }
-
-   /// Register a specialized kernel for dispatch.
-   template <Params... PARAMS>
-   struct Specialization
-   {
-      // Version without optional parameters
-      static void Add()
-      {
-         std::tuple<Params...> param_tuple(PARAMS...);
-         Kernels::Get().table[param_tuple] =
-            Kernels:: template Kernel<PARAMS..., OptParams{}...>();
-      };
-      // Version with optional parameters
-      template <OptParams... OPT_PARAMS>
-      struct Opt
-      {
-         static void Add()
-         {
-            std::tuple<Params...> param_tuple(PARAMS...);
-            Kernels::Get().table[param_tuple] =
-               Kernels:: template Kernel<PARAMS..., OPT_PARAMS...>();
-         }
-      };
-   };
-
-   /// Return the dispatch map table
-   static const TableType &GetDispatchTable()
-   {
-      return Kernels::Get().table;
-   }
-};
-
-}
+} // namespace mfem
 
 #endif

fem/nonlinearform.hpp

@@ -224,6 +224,9 @@ public:
    /** @see GetGradient(const Vector &) */
    Operator &GetGradient(const Vector &x, bool finalize) const;
 
+   /// Suppress a warning about hiding overloaded virtual function.
+   using Operator::GetGradient;
+
    /// Update the NonlinearForm to propagate updates of the associated FE space.
    /** After calling this method, the essential boundary conditions need to be
        set again. */

fem/qinterp/eval_transpose.cpp

@@ -0,0 +1,44 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../quadinterpolator.hpp"
+#include "eval_transpose.hpp"
+
+namespace mfem
+{
+
+/// @cond Suppress_Doxygen_warnings
+
+QuadratureInterpolator::TensorEvalTransposeKernelType
+QuadratureInterpolator::TensorEvalTransposeKernels::Fallback(
+   int DIM, QVectorLayout Q_LAYOUT, int, int, int)
+{
+   using namespace internal::quadrature_interpolator;
+
+   if (Q_LAYOUT == QVectorLayout::byNODES)
+   {
+      if (DIM == 1) { return ValuesTranspose1D<QVectorLayout::byNODES>; }
+      else if (DIM == 2) { return ValuesTranspose2D<QVectorLayout::byNODES>; }
+      else if (DIM == 3) { return ValuesTranspose3D<QVectorLayout::byNODES>; }
+   }
+   else
+   {
+      if (DIM == 1) { return ValuesTranspose1D<QVectorLayout::byVDIM>; }
+      else if (DIM == 2) { return ValuesTranspose2D<QVectorLayout::byVDIM>; }
+      else if (DIM == 3) { return ValuesTranspose3D<QVectorLayout::byVDIM>; }
+   }
+   MFEM_ABORT("Invalid dimension");
+   return nullptr;
+}
+
+/// @endcond
+
+} // namespace mfem

fem/qinterp/eval_transpose.hpp

@@ -0,0 +1,300 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#pragma once
+
+#include "../quadinterpolator.hpp"
+#include "../../general/forall.hpp"
+#include "../../linalg/dtensor.hpp"
+#include "../../linalg/kernels.hpp"
+#include "../kernels.hpp"
+
+namespace mfem
+{
+namespace internal
+{
+namespace quadrature_interpolator
+{
+
+template<QVectorLayout Q_LAYOUT>
+static void ValuesTranspose1D(const int NE,
+                              const real_t *b_,
+                              const real_t *q_,
+                              real_t *e_,
+                              const int vdim,
+                              const int d1d,
+                              const int q1d)
+{
+   const auto b = Reshape(b_, q1d, d1d);
+   const auto qd = Q_LAYOUT == QVectorLayout::byNODES ?
+                   Reshape(q_, q1d, vdim, NE) :
+                   Reshape(q_, vdim, q1d, NE);
+   auto e = Reshape(e_, d1d, vdim, NE);
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int el)
+   {
+      for (int c = 0; c < vdim; c++)
+      {
+         for (int d = 0; d < d1d; d++)
+         {
+            real_t u = 0.0;
+            for (int q = 0; q < q1d; q++)
+            {
+               const real_t qval = Q_LAYOUT == QVectorLayout::byVDIM ?
+                                   qd(c, q, el) : qd(q, c, el);
+               u += b(q, d) * qval;
+            }
+            e(d, c, el) += u;
+         }
+      }
+   });
+}
+
+template<QVectorLayout Q_LAYOUT,
+         int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0,
+         int T_NBZ = 1>
+static void ValuesTranspose2D(const int NE,
+                              const real_t *b_,
+                              const real_t *q_,
+                              real_t *e_,
+                              const int vdim = 0,
+                              const int d1d = 0,
+                              const int q1d = 0)
+{
+   static constexpr int NBZ = T_NBZ ? T_NBZ : 1;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   const int VDIM = T_VDIM ? T_VDIM : vdim;
+
+   const auto b = Reshape(b_, Q1D, D1D);
+   const auto q = Q_LAYOUT == QVectorLayout::byNODES ?
+                  Reshape(q_, Q1D, Q1D, VDIM, NE) :
+                  Reshape(q_, VDIM, Q1D, Q1D, NE);
+   auto e = Reshape(e_, D1D, D1D, VDIM, NE);
+
+   mfem::forall_2D_batch(NE, D1D, D1D, NBZ, [=] MFEM_HOST_DEVICE (int el)
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      const int VDIM = T_VDIM ? T_VDIM : vdim;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
+      const int tidz = MFEM_THREAD_ID(z);
+
+      MFEM_SHARED real_t sB[MQ1*MD1];
+      MFEM_SHARED real_t sm0[NBZ][MDQ*MDQ];
+      MFEM_SHARED real_t sm1[NBZ][MDQ*MDQ];
+
+      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,b,sB);
+
+      ConstDeviceMatrix B(sB, D1D, Q1D);
+      DeviceMatrix QQ(sm0[tidz], MQ1, MQ1);
+      DeviceMatrix DQ(sm1[tidz], MD1, MQ1);
+      DeviceMatrix DD(sm0[tidz], MD1, MD1);
+
+      for (int c = 0; c < VDIM; c++)
+      {
+         // Load Q data
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               QQ(qx,qy) = Q_LAYOUT == QVectorLayout::byVDIM ?
+                           q(c,qx,qy,el) : q(qx,qy,c,el);
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Transpose in y: QQ -> DQ (apply B^T in y-direction)
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               real_t u = 0.0;
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  u += B(dy,qy) * QQ(qx,qy);
+               }
+               DQ(dy,qx) = u;
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Transpose in x: DQ -> DD (apply B^T in x-direction)
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(dx,x,D1D)
+            {
+               real_t u = 0.0;
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  u += B(dx,qx) * DQ(dy,qx);
+               }
+               DD(dx,dy) = u;
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Store result
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(dx,x,D1D)
+            {
+               e(dx,dy,c,el) += DD(dx,dy);
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   });
+}
+
+template<QVectorLayout Q_LAYOUT,
+         int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0>
+static void ValuesTranspose3D(const int NE,
+                              const real_t *b_,
+                              const real_t *q_,
+                              real_t *e_,
+                              const int vdim = 0,
+                              const int d1d = 0,
+                              const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   const int VDIM = T_VDIM ? T_VDIM : vdim;
+
+   const auto b = Reshape(b_, Q1D, D1D);
+   const auto q = Q_LAYOUT == QVectorLayout::byNODES ?
+                  Reshape(q_, Q1D, Q1D, Q1D, VDIM, NE) :
+                  Reshape(q_, VDIM, Q1D, Q1D, Q1D, NE);
+   auto e = Reshape(e_, D1D, D1D, D1D, VDIM, NE);
+
+   mfem::forall_3D(NE, D1D, D1D, D1D, [=] MFEM_HOST_DEVICE (int el)
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      const int VDIM = T_VDIM ? T_VDIM : vdim;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_INTERP_1D;
+      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_INTERP_1D;
+      constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
+
+      MFEM_SHARED real_t sB[MQ1*MD1];
+      MFEM_SHARED real_t sm0[MDQ*MDQ*MDQ];
+      MFEM_SHARED real_t sm1[MDQ*MDQ*MDQ];
+
+      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,b,sB);
+
+      ConstDeviceMatrix B(sB, D1D, Q1D);
+      DeviceCube QQQ(sm0, MQ1, MQ1, MQ1);
+      DeviceCube DQQ(sm1, MD1, MQ1, MQ1);
+      DeviceCube DDQ(sm0, MD1, MD1, MQ1);
+      DeviceCube DDD(sm1, MD1, MD1, MD1);
+
+      for (int c = 0; c < VDIM; c++)
+      {
+         // Load Q data
+         MFEM_FOREACH_THREAD(qz,z,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qy,y,Q1D)
+            {
+               MFEM_FOREACH_THREAD(qx,x,Q1D)
+               {
+                  QQQ(qx,qy,qz) = Q_LAYOUT == QVectorLayout::byVDIM ?
+                                  q(c,qx,qy,qz,el) : q(qx,qy,qz,c,el);
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Transpose in z
+         MFEM_FOREACH_THREAD(dz,z,D1D)
+         {
+            MFEM_FOREACH_THREAD(qy,y,Q1D)
+            {
+               MFEM_FOREACH_THREAD(qx,x,Q1D)
+               {
+                  real_t u = 0.0;
+                  for (int qz = 0; qz < Q1D; ++qz)
+                  {
+                     u += B(dz,qz) * QQQ(qx,qy,qz);
+                  }
+                  DQQ(dz,qx,qy) = u;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Transpose in y
+         MFEM_FOREACH_THREAD(dz,z,D1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               MFEM_FOREACH_THREAD(qx,x,Q1D)
+               {
+                  real_t u = 0.0;
+                  for (int qy = 0; qy < Q1D; ++qy)
+                  {
+                     u += B(dy,qy) * DQQ(dz,qx,qy);
+                  }
+                  DDQ(dz,dy,qx) = u;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Transpose in x
+         MFEM_FOREACH_THREAD(dz,z,D1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               MFEM_FOREACH_THREAD(dx,x,D1D)
+               {
+                  real_t u = 0.0;
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     u += B(dx,qx) * DDQ(dz,dy,qx);
+                  }
+                  DDD(dx,dy,dz) = u;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD(dz,z,D1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               MFEM_FOREACH_THREAD(dx,x,D1D)
+               {
+                  e(dx,dy,dz,c,el) += DDD(dx,dy,dz);
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   });
+}
+
+} // namespace quadrature_interpolator
+} // namespace internal
+
+template<int DIM, QVectorLayout Q_LAYOUT,
+         int VDIM, int D1D, int Q1D, int NBZ>
+QuadratureInterpolator::TensorEvalTransposeKernelType
+QuadratureInterpolator::TensorEvalTransposeKernels::Kernel()
+{
+   if (DIM == 1) { return internal::quadrature_interpolator::ValuesTranspose1D<Q_LAYOUT>; }
+   else if (DIM == 2) { return internal::quadrature_interpolator::ValuesTranspose2D<Q_LAYOUT, VDIM, D1D, Q1D, NBZ>; }
+   else if (DIM == 3) { return internal::quadrature_interpolator::ValuesTranspose3D<Q_LAYOUT, VDIM, D1D, Q1D>; }
+   else { MFEM_ABORT(""); }
+}
+
+} // namespace mfem

fem/qinterp/eval_transpose_by_vdim.cpp

@@ -0,0 +1,61 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../quadinterpolator.hpp"
+#include "eval_transpose.hpp"
+
+namespace mfem
+{
+namespace internal
+{
+namespace quadrature_interpolator
+{
+
+void InitEvalTransposeByVDimKernels()
+{
+   using k = QuadratureInterpolator::TensorEvalTransposeKernels;
+   constexpr auto L = QVectorLayout::byVDIM;
+
+   // 2D
+   k::Specialization<2,L,1,2,4>::Opt<8>::Add();
+   k::Specialization<2,L,1,3,6>::Opt<4>::Add();
+   k::Specialization<2,L,1,4,8>::Opt<2>::Add();
+
+   k::Specialization<2,L,2,2,4>::Opt<8>::Add();
+   k::Specialization<2,L,2,3,4>::Opt<8>::Add();
+   k::Specialization<2,L,2,3,6>::Opt<4>::Add();
+   k::Specialization<2,L,2,4,6>::Opt<2>::Add();
+   k::Specialization<2,L,2,4,8>::Opt<2>::Add();
+
+   // 3D
+   k::Specialization<3,L,1,2,4>::Opt<1>::Add();
+   k::Specialization<3,L,1,3,6>::Opt<1>::Add();
+   k::Specialization<3,L,1,4,8>::Opt<1>::Add();
+   k::Specialization<3,L,3,2,4>::Opt<1>::Add();
+   k::Specialization<3,L,3,3,6>::Opt<1>::Add();
+   k::Specialization<3,L,3,4,8>::Opt<1>::Add();
+
+   k::Specialization<3,L,3,2,2>::Opt<1>::Add();
+   k::Specialization<3,L,3,3,3>::Opt<1>::Add();
+   k::Specialization<3,L,3,4,4>::Opt<1>::Add();
+   k::Specialization<3,L,3,5,5>::Opt<1>::Add();
+   k::Specialization<3,L,3,6,6>::Opt<1>::Add();
+   k::Specialization<3,L,3,7,7>::Opt<1>::Add();
+   k::Specialization<3,L,3,8,8>::Opt<1>::Add();
+   k::Specialization<3,L,3,9,9>::Opt<1>::Add();
+
+   k::Specialization<3,L,3,4,6>::Opt<1>::Add();
+   k::Specialization<3,L,3,3,4>::Opt<1>::Add();
+}
+
+} // namespace quadrature_interpolator
+} // namespace internal
+} // namespace mfem

fem/qinterp/grad_transpose.cpp

@@ -0,0 +1,62 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../quadinterpolator.hpp"
+#include "grad_transpose.hpp"
+
+namespace mfem
+{
+
+/// @cond Suppress_Doxygen_warnings
+
+QuadratureInterpolator::GradTransposeKernelType
+QuadratureInterpolator::GradTransposeKernels::Fallback(
+   int DIM, QVectorLayout Q_LAYOUT, bool GRAD_PHYS, int, int, int)
+{
+   using namespace internal::quadrature_interpolator;
+
+   if (Q_LAYOUT == QVectorLayout::byNODES)
+   {
+      if (GRAD_PHYS)
+      {
+         if (DIM == 1) { return DerivativesTranspose1D<QVectorLayout::byNODES, true>; }
+         else if (DIM == 2) { return DerivativesTranspose2D<QVectorLayout::byNODES, true>; }
+         else if (DIM == 3) { return DerivativesTranspose3D<QVectorLayout::byNODES, true>; }
+      }
+      else
+      {
+         if (DIM == 1) { return DerivativesTranspose1D<QVectorLayout::byNODES, false>; }
+         else if (DIM == 2) { return DerivativesTranspose2D<QVectorLayout::byNODES, false>; }
+         else if (DIM == 3) { return DerivativesTranspose3D<QVectorLayout::byNODES, false>; }
+      }
+   }
+   else
+   {
+      if (GRAD_PHYS)
+      {
+         if (DIM == 1) { return DerivativesTranspose1D<QVectorLayout::byVDIM, true>; }
+         else if (DIM == 2) { return DerivativesTranspose2D<QVectorLayout::byVDIM, true>; }
+         else if (DIM == 3) { return DerivativesTranspose3D<QVectorLayout::byVDIM, true>; }
+      }
+      else
+      {
+         if (DIM == 1) { return DerivativesTranspose1D<QVectorLayout::byVDIM, false>; }
+         else if (DIM == 2) { return DerivativesTranspose2D<QVectorLayout::byVDIM, false>; }
+         else if (DIM == 3) { return DerivativesTranspose3D<QVectorLayout::byVDIM, false>; }
+      }
+   }
+   MFEM_ABORT("Invalid dimension");
+   return nullptr;
+}
+
+/// @endcond
+
+} // namespace mfem

fem/qinterp/grad_transpose.hpp

@@ -0,0 +1,737 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#pragma once
+
+#include "../quadinterpolator.hpp"
+#include "../../general/forall.hpp"
+#include "../../linalg/dtensor.hpp"
+#include "../../linalg/kernels.hpp"
+#include "../kernels.hpp"
+
+namespace mfem
+{
+namespace internal
+{
+namespace quadrature_interpolator
+{
+
+// Transpose gradient operation: integrate against shape function derivatives
+// This is the adjoint of the Derivatives operation
+
+template<QVectorLayout Q_LAYOUT, bool GRAD_PHYS>
+static void DerivativesTranspose1D(const int NE,
+                                   const real_t *b_,
+                                   const real_t *g_,
+                                   const real_t *j_,
+                                   const real_t *q_,
+                                   real_t *e_,
+                                   const int sdim,
+                                   const int vdim,
+                                   const int d1d,
+                                   const int q1d)
+{
+   MFEM_CONTRACT_VAR(b_);
+   const int SDIM = GRAD_PHYS ? sdim : 1;
+   const auto g = Reshape(g_, q1d, d1d);
+   const auto j = Reshape(j_, q1d, SDIM, NE);
+   const auto q = Q_LAYOUT == QVectorLayout::byNODES ?
+                  Reshape(q_, q1d, vdim, SDIM, NE):
+                  Reshape(q_, vdim, SDIM, q1d, NE);
+   auto e = Reshape(e_, d1d, vdim, NE);
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int el)
+   {
+      for (int c = 0; c < vdim; c++)
+      {
+         for (int d = 0; d < d1d; d++)
+         {
+            real_t u = 0.0;
+            for (int qx = 0; qx < q1d; qx++)
+            {
+               // Load gradient from q-vector
+               real_t dq[3] = {0.0, 0.0, 0.0};
+               for (int s = 0; s < SDIM; ++s)
+               {
+                  if (Q_LAYOUT == QVectorLayout::byVDIM)  { dq[s] = q(c, s, qx, el); }
+                  if (Q_LAYOUT == QVectorLayout::byNODES) { dq[s] = q(qx, c, s, el); }
+               }
+
+               // Apply inverse Jacobian transpose (adjoint of physical gradient)
+               real_t du = dq[0];
+               if (GRAD_PHYS)
+               {
+                  if (SDIM == 1) { du = dq[0] / j(qx, 0, el); }
+                  else if (SDIM == 2)
+                  {
+                     const real_t Jloc[2] = {j(qx,0,el), j(qx,1,el)};
+                     real_t Jinv[3];
+                     kernels::CalcLeftInverse<2,1>(Jloc, Jinv);
+                     du = Jinv[0]*dq[0] + Jinv[1]*dq[1];
+                  }
+                  else // SDIM == 3
+                  {
+                     const real_t Jloc[3] = {j(qx,0,el), j(qx,1,el), j(qx,2,el)};
+                     real_t Jinv[3];
+                     kernels::CalcLeftInverse<3,1>(Jloc, Jinv);
+                     du = Jinv[0]*dq[0] + Jinv[1]*dq[1] + Jinv[2]*dq[2];
+                  }
+               }
+
+               // Accumulate contribution (transpose of G matrix)
+               u += g(qx, d) * du;
+            }
+            e(d, c, el) += u;
+         }
+      }
+   });
+}
+
+template<QVectorLayout Q_LAYOUT, bool GRAD_PHYS,
+         int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0,
+         int T_NBZ = 1>
+static void DerivativesTranspose2D(const int NE,
+                                   const real_t *b_,
+                                   const real_t *g_,
+                                   const real_t *j_,
+                                   const real_t *q_,
+                                   real_t *e_,
+                                   const int sdim = 2,
+                                   const int vdim = 0,
+                                   const int d1d = 0,
+                                   const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   const int VDIM = T_VDIM ? T_VDIM : vdim;
+   const int SDIM = GRAD_PHYS ? sdim : 2;
+   static constexpr int NBZ = T_NBZ ? T_NBZ : 1;
+
+   const auto b = Reshape(b_, Q1D, D1D);
+   const auto g = Reshape(g_, Q1D, D1D);
+   const auto j = Reshape(j_, Q1D, Q1D, SDIM, 2, NE);
+   const auto q = Q_LAYOUT == QVectorLayout::byNODES ?
+                  Reshape(q_, Q1D, Q1D, VDIM, SDIM, NE):
+                  Reshape(q_, VDIM, SDIM, Q1D, Q1D, NE);
+   auto e = Reshape(e_, D1D, D1D, VDIM, NE);
+
+   mfem::forall_2D_batch(NE, D1D, D1D, NBZ, [=] MFEM_HOST_DEVICE (int el)
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      const int VDIM = T_VDIM ? T_VDIM : vdim;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
+      const int tidz = MFEM_THREAD_ID(z);
+
+      MFEM_SHARED real_t BG[2][MQ1*MD1];
+      kernels::internal::LoadBG<MD1,MQ1>(D1D,Q1D,b,g,BG);
+      DeviceMatrix B(BG[0], D1D, Q1D);
+      DeviceMatrix G(BG[1], D1D, Q1D);
+
+      MFEM_SHARED real_t sm0[NBZ][MDQ*MDQ];
+      MFEM_SHARED real_t sm1[NBZ][MDQ*MDQ];
+
+      DeviceMatrix QQ(sm0[tidz], MQ1, MQ1);
+      DeviceMatrix DQ0(sm1[tidz], MD1, MQ1);
+      DeviceMatrix DQ1(sm1[tidz], MD1, MQ1);  // Reuse sm1 after DQ0 is done
+      DeviceMatrix DD(sm0[tidz], MD1, MD1);   // Reuse sm0 after QQ is done
+
+      for (int c = 0; c < VDIM; c++)
+      {
+         // Load Q data and apply inverse Jacobian
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               // Load gradient components
+               real_t dq[3] = {0.0, 0.0, 0.0};
+               for (int d = 0; d < SDIM; ++d)
+               {
+                  if (Q_LAYOUT == QVectorLayout::byVDIM) { dq[d] = q(c, d, qx, qy, el); }
+                  else { dq[d] = q(qx, qy, c, d, el); }
+               }
+
+               // Apply inverse Jacobian transpose (adjoint of physical gradient)
+               real_t du[2] = {dq[0], dq[1]};
+               if (GRAD_PHYS)
+               {
+                  if (SDIM == 2)
+                  {
+                     real_t Jloc[4], Jinv[4];
+                     Jloc[0] = j(qx,qy,0,0,el);
+                     Jloc[1] = j(qx,qy,1,0,el);
+                     Jloc[2] = j(qx,qy,0,1,el);
+                     Jloc[3] = j(qx,qy,1,1,el);
+                     kernels::CalcInverse<2>(Jloc, Jinv);
+                     const real_t U = Jinv[0]*dq[0] + Jinv[2]*dq[1];
+                     const real_t V = Jinv[1]*dq[0] + Jinv[3]*dq[1];
+                     du[0] = U;
+                     du[1] = V;
+                  }
+                  else // SDIM == 3
+                  {
+                     real_t Jloc[6], Jinv[6];
+                     Jloc[0] = j(qx,qy,0,0,el);
+                     Jloc[1] = j(qx,qy,1,0,el);
+                     Jloc[2] = j(qx,qy,2,0,el);
+                     Jloc[3] = j(qx,qy,0,1,el);
+                     Jloc[4] = j(qx,qy,1,1,el);
+                     Jloc[5] = j(qx,qy,2,1,el);
+                     kernels::CalcLeftInverse<3,2>(Jloc, Jinv);
+                     const real_t U = Jinv[0]*dq[0] + Jinv[2]*dq[1] + Jinv[4]*dq[2];
+                     const real_t V = Jinv[1]*dq[0] + Jinv[3]*dq[1] + Jinv[5]*dq[2];
+                     du[0] = U;
+                     du[1] = V;
+                  }
+               }
+               QQ(qx, qy) = du[0];  // Store du/dx component
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Apply B^T in y-direction: QQ -> DQ0
+         // (Transpose of d/dx which uses DQ1(dy,qx)*B(dy,qy))
+         // Must produce DQ0(dy,qx) to match forward's DQ1 indexing
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               real_t u = 0.0;
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  u += B(dy,qy) * QQ(qx,qy);
+               }
+               DQ0(dy,qx) = u;
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Apply G^T in x-direction: DQ0 -> DD
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(dx,x,D1D)
+            {
+               real_t u = 0.0;
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  u += G(dx,qx) * DQ0(dy,qx);
+               }
+               DD(dx,dy) = u;
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Accumulate to output
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(dx,x,D1D)
+            {
+               e(dx,dy,c,el) += DD(dx,dy);
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Now process du/dy component
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               // Load gradient components
+               real_t dq[3] = {0.0, 0.0, 0.0};
+               for (int d = 0; d < SDIM; ++d)
+               {
+                  if (Q_LAYOUT == QVectorLayout::byVDIM) { dq[d] = q(c, d, qx, qy, el); }
+                  else { dq[d] = q(qx, qy, c, d, el); }
+               }
+
+               // Apply inverse Jacobian transpose
+               real_t du[2] = {dq[0], dq[1]};
+               if (GRAD_PHYS)
+               {
+                  if (SDIM == 2)
+                  {
+                     real_t Jloc[4], Jinv[4];
+                     Jloc[0] = j(qx,qy,0,0,el);
+                     Jloc[1] = j(qx,qy,1,0,el);
+                     Jloc[2] = j(qx,qy,0,1,el);
+                     Jloc[3] = j(qx,qy,1,1,el);
+                     kernels::CalcInverse<2>(Jloc, Jinv);
+                     const real_t U = Jinv[0]*dq[0] + Jinv[2]*dq[1];
+                     const real_t V = Jinv[1]*dq[0] + Jinv[3]*dq[1];
+                     du[0] = U;
+                     du[1] = V;
+                  }
+                  else // SDIM == 3
+                  {
+                     real_t Jloc[6], Jinv[6];
+                     Jloc[0] = j(qx,qy,0,0,el);
+                     Jloc[1] = j(qx,qy,1,0,el);
+                     Jloc[2] = j(qx,qy,2,0,el);
+                     Jloc[3] = j(qx,qy,0,1,el);
+                     Jloc[4] = j(qx,qy,1,1,el);
+                     Jloc[5] = j(qx,qy,2,1,el);
+                     kernels::CalcLeftInverse<3,2>(Jloc, Jinv);
+                     const real_t U = Jinv[0]*dq[0] + Jinv[2]*dq[1] + Jinv[4]*dq[2];
+                     const real_t V = Jinv[1]*dq[0] + Jinv[3]*dq[1] + Jinv[5]*dq[2];
+                     du[0] = U;
+                     du[1] = V;
+                  }
+               }
+               QQ(qx, qy) = du[1];  // Store du/dy component
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Apply G^T in y-direction: QQ -> DQ1
+         // (Transpose of d/dy which uses DQ0(dy,qx)*G(dy,qy))
+         // Must produce DQ1(dy,qx) to match forward's DQ0 indexing
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               real_t u = 0.0;
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  u += G(dy,qy) * QQ(qx,qy);
+               }
+               DQ1(dy,qx) = u;
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Apply B^T in x-direction: DQ1 -> DD
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(dx,x,D1D)
+            {
+               real_t u = 0.0;
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  u += B(dx,qx) * DQ1(dy,qx);
+               }
+               DD(dx,dy) = u;
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Accumulate to output
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(dx,x,D1D)
+            {
+               e(dx,dy,c,el) += DD(dx,dy);
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   });
+}
+
+template<QVectorLayout Q_LAYOUT, bool GRAD_PHYS,
+         int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0>
+static void DerivativesTranspose3D(const int NE,
+                                   const real_t *b_,
+                                   const real_t *g_,
+                                   const real_t *j_,
+                                   const real_t *q_,    // q_der
+                                   real_t *e_,          // e_vec
+                                   const int sdim = 3,
+                                   const int vdim = 0,
+                                   const int d1d = 0,
+                                   const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   const int VDIM = T_VDIM ? T_VDIM : vdim;
+
+   const auto b = Reshape(b_, Q1D, D1D);
+   const auto g = Reshape(g_, Q1D, D1D);
+   const auto j = Reshape(j_, Q1D, Q1D, Q1D, 3, 3, NE);
+   const auto q = Q_LAYOUT == QVectorLayout::byNODES ?
+                  Reshape(q_, Q1D, Q1D, Q1D, VDIM, 3, NE):
+                  Reshape(q_, VDIM, 3, Q1D, Q1D, Q1D, NE);
+   auto e = Reshape(e_, D1D, D1D, D1D, VDIM, NE);
+
+   mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int el)
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      const int VDIM = T_VDIM ? T_VDIM : vdim;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_INTERP_1D;
+      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_INTERP_1D;
+
+      MFEM_SHARED real_t BG[2][MQ1*MD1];
+      kernels::internal::LoadBG<MD1,MQ1>(D1D,Q1D,b,g,BG);
+      DeviceMatrix B(BG[0], D1D, Q1D);
+      DeviceMatrix G(BG[1], D1D, Q1D);
+
+      MFEM_SHARED real_t sm0[1][MQ1*MQ1*MQ1];
+      MFEM_SHARED real_t sm1[1][MQ1*MQ1*MQ1];
+      DeviceCube QQQ(sm0[0], MQ1, MQ1, MQ1);
+      DeviceCube DQQ(sm1[0], MD1, MQ1, MQ1);
+      DeviceCube DDQ(sm0[0], MD1, MD1, MQ1);
+      DeviceCube DDD(sm1[0], MD1, MD1, MD1);
+
+      for (int c = 0; c < VDIM; c++)
+      {
+         // Process du/dx component
+         MFEM_FOREACH_THREAD(qz,z,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qy,y,Q1D)
+            {
+               MFEM_FOREACH_THREAD(qx,x,Q1D)
+               {
+                  real_t dq[3];
+                  if (Q_LAYOUT == QVectorLayout::byVDIM)
+                  {
+                     dq[0] = q(c,0,qx,qy,qz,el);
+                     dq[1] = q(c,1,qx,qy,qz,el);
+                     dq[2] = q(c,2,qx,qy,qz,el);
+                  }
+                  else
+                  {
+                     dq[0] = q(qx,qy,qz,c,0,el);
+                     dq[1] = q(qx,qy,qz,c,1,el);
+                     dq[2] = q(qx,qy,qz,c,2,el);
+                  }
+
+                  real_t du[3] = {dq[0], dq[1], dq[2]};
+                  if (GRAD_PHYS)
+                  {
+                     real_t Jloc[9], Jinv[9];
+                     for (int col = 0; col < 3; col++)
+                     {
+                        for (int row = 0; row < 3; row++)
+                        {
+                           Jloc[row+3*col] = j(qx,qy,qz,row,col,el);
+                        }
+                     }
+                     kernels::CalcInverse<3>(Jloc, Jinv);
+                     const real_t U = Jinv[0]*dq[0] + Jinv[3]*dq[1] + Jinv[6]*dq[2];
+                     const real_t V = Jinv[1]*dq[0] + Jinv[4]*dq[1] + Jinv[7]*dq[2];
+                     const real_t W = Jinv[2]*dq[0] + Jinv[5]*dq[1] + Jinv[8]*dq[2];
+                     du[0] = U; du[1] = V; du[2] = W;
+                  }
+                  QQQ(qx,qy,qz) = du[0];
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Apply G^T in x: QQQ -> DQQ (transpose of G⊗B⊗B)
+         MFEM_FOREACH_THREAD(qz,z,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qy,y,Q1D)
+            {
+               MFEM_FOREACH_THREAD(dx,x,D1D)
+               {
+                  real_t u = 0.0;
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     u += G(dx,qx) * QQQ(qx,qy,qz);
+                  }
+                  DQQ(dx,qy,qz) = u;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Apply B^T in y: DQQ -> DDQ
+         MFEM_FOREACH_THREAD(qz,z,Q1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               MFEM_FOREACH_THREAD(dx,x,D1D)
+               {
+                  real_t u = 0.0;
+                  for (int qy = 0; qy < Q1D; ++qy)
+                  {
+                     u += B(dy,qy) * DQQ(dx,qy,qz);
+                  }
+                  DDQ(dx,dy,qz) = u;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Apply B^T in z: DDQ -> DDD
+         MFEM_FOREACH_THREAD(dz,z,D1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               MFEM_FOREACH_THREAD(dx,x,D1D)
+               {
+                  real_t u = 0.0;
+                  for (int qz = 0; qz < Q1D; ++qz)
+                  {
+                     u += B(dz,qz) * DDQ(dx,dy,qz);
+                  }
+                  DDD(dx,dy,dz) = u;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Accumulate result
+         MFEM_FOREACH_THREAD(dz,z,D1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               MFEM_FOREACH_THREAD(dx,x,D1D)
+               {
+                  e(dx,dy,dz,c,el) += DDD(dx,dy,dz);
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Process du/dy component
+         MFEM_FOREACH_THREAD(qz,z,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qy,y,Q1D)
+            {
+               MFEM_FOREACH_THREAD(qx,x,Q1D)
+               {
+                  real_t dq[3];
+                  if (Q_LAYOUT == QVectorLayout::byVDIM)
+                  {
+                     dq[0] = q(c,0,qx,qy,qz,el);
+                     dq[1] = q(c,1,qx,qy,qz,el);
+                     dq[2] = q(c,2,qx,qy,qz,el);
+                  }
+                  else
+                  {
+                     dq[0] = q(qx,qy,qz,c,0,el);
+                     dq[1] = q(qx,qy,qz,c,1,el);
+                     dq[2] = q(qx,qy,qz,c,2,el);
+                  }
+
+                  real_t du[3] = {dq[0], dq[1], dq[2]};
+                  if (GRAD_PHYS)
+                  {
+                     real_t Jloc[9], Jinv[9];
+                     for (int col = 0; col < 3; col++)
+                     {
+                        for (int row = 0; row < 3; row++)
+                        {
+                           Jloc[row+3*col] = j(qx,qy,qz,row,col,el);
+                        }
+                     }
+                     kernels::CalcInverse<3>(Jloc, Jinv);
+                     const real_t U = Jinv[0]*dq[0] + Jinv[3]*dq[1] + Jinv[6]*dq[2];
+                     const real_t V = Jinv[1]*dq[0] + Jinv[4]*dq[1] + Jinv[7]*dq[2];
+                     const real_t W = Jinv[2]*dq[0] + Jinv[5]*dq[1] + Jinv[8]*dq[2];
+                     du[0] = U; du[1] = V; du[2] = W;
+                  }
+                  QQQ(qx,qy,qz) = du[1];
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Apply B^T in x: QQQ -> DQQ (transpose of B⊗G⊗B)
+         MFEM_FOREACH_THREAD(qz,z,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qy,y,Q1D)
+            {
+               MFEM_FOREACH_THREAD(dx,x,D1D)
+               {
+                  real_t u = 0.0;
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     u += B(dx,qx) * QQQ(qx,qy,qz);
+                  }
+                  DQQ(dx,qy,qz) = u;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Apply G^T in y: DQQ -> DDQ
+         MFEM_FOREACH_THREAD(qz,z,Q1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               MFEM_FOREACH_THREAD(dx,x,D1D)
+               {
+                  real_t u = 0.0;
+                  for (int qy = 0; qy < Q1D; ++qy)
+                  {
+                     u += G(dy,qy) * DQQ(dx,qy,qz);
+                  }
+                  DDQ(dx,dy,qz) = u;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Apply B^T in z: DDQ -> DDD
+         MFEM_FOREACH_THREAD(dz,z,D1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               MFEM_FOREACH_THREAD(dx,x,D1D)
+               {
+                  real_t u = 0.0;
+                  for (int qz = 0; qz < Q1D; ++qz)
+                  {
+                     u += B(dz,qz) * DDQ(dx,dy,qz);
+                  }
+                  DDD(dx,dy,dz) = u;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Accumulate result
+         MFEM_FOREACH_THREAD(dz,z,D1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               MFEM_FOREACH_THREAD(dx,x,D1D)
+               {
+                  e(dx,dy,dz,c,el) += DDD(dx,dy,dz);
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Process du/dz component
+         MFEM_FOREACH_THREAD(qz,z,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qy,y,Q1D)
+            {
+               MFEM_FOREACH_THREAD(qx,x,Q1D)
+               {
+                  real_t dq[3];
+                  if (Q_LAYOUT == QVectorLayout::byVDIM)
+                  {
+                     dq[0] = q(c,0,qx,qy,qz,el);
+                     dq[1] = q(c,1,qx,qy,qz,el);
+                     dq[2] = q(c,2,qx,qy,qz,el);
+                  }
+                  else
+                  {
+                     dq[0] = q(qx,qy,qz,c,0,el);
+                     dq[1] = q(qx,qy,qz,c,1,el);
+                     dq[2] = q(qx,qy,qz,c,2,el);
+                  }
+
+                  real_t du[3] = {dq[0], dq[1], dq[2]};
+                  if (GRAD_PHYS)
+                  {
+                     real_t Jloc[9], Jinv[9];
+                     for (int col = 0; col < 3; col++)
+                     {
+                        for (int row = 0; row < 3; row++)
+                        {
+                           Jloc[row+3*col] = j(qx,qy,qz,row,col,el);
+                        }
+                     }
+                     kernels::CalcInverse<3>(Jloc, Jinv);
+                     const real_t U = Jinv[0]*dq[0] + Jinv[3]*dq[1] + Jinv[6]*dq[2];
+                     const real_t V = Jinv[1]*dq[0] + Jinv[4]*dq[1] + Jinv[7]*dq[2];
+                     const real_t W = Jinv[2]*dq[0] + Jinv[5]*dq[1] + Jinv[8]*dq[2];
+                     du[0] = U; du[1] = V; du[2] = W;
+                  }
+                  QQQ(qx,qy,qz) = du[2];
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Apply G^T in z: QQQ -> DQQ (transpose of B⊗B⊗G)
+         MFEM_FOREACH_THREAD(dz,z,D1D)
+         {
+            MFEM_FOREACH_THREAD(qy,y,Q1D)
+            {
+               MFEM_FOREACH_THREAD(qx,x,Q1D)
+               {
+                  real_t u = 0.0;
+                  for (int qz = 0; qz < Q1D; ++qz)
+                  {
+                     u += G(dz,qz) * QQQ(qx,qy,qz);
+                  }
+                  DQQ(dz,qy,qx) = u;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Apply B^T in y: DQQ -> DDQ
+         MFEM_FOREACH_THREAD(dz,z,D1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               MFEM_FOREACH_THREAD(qx,x,Q1D)
+               {
+                  real_t u = 0.0;
+                  for (int qy = 0; qy < Q1D; ++qy)
+                  {
+                     u += B(dy,qy) * DQQ(dz,qy,qx);
+                  }
+                  DDQ(dz,dy,qx) = u;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Apply B^T in x: DDQ -> DDD
+         MFEM_FOREACH_THREAD(dz,z,D1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               MFEM_FOREACH_THREAD(dx,x,D1D)
+               {
+                  real_t u = 0.0;
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     u += B(dx,qx) * DDQ(dz,dy,qx);
+                  }
+                  DDD(dx,dy,dz) = u;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Accumulate result
+         MFEM_FOREACH_THREAD(dz,z,D1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               MFEM_FOREACH_THREAD(dx,x,D1D)
+               {
+                  e(dx,dy,dz,c,el) += DDD(dx,dy,dz);
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   });
+}
+
+} // namespace quadrature_interpolator
+} // namespace internal
+
+template<int DIM, QVectorLayout Q_LAYOUT, bool GRAD_PHYS, int VDIM, int D1D,
+         int Q1D, int NBZ>
+QuadratureInterpolator::GradTransposeKernelType
+QuadratureInterpolator::GradTransposeKernels::Kernel()
+{
+   if (DIM == 1) { return internal::quadrature_interpolator::DerivativesTranspose1D<Q_LAYOUT, GRAD_PHYS>; }
+   else if (DIM == 2) { return internal::quadrature_interpolator::DerivativesTranspose2D<Q_LAYOUT, GRAD_PHYS, VDIM, D1D, Q1D, NBZ>; }
+   else if (DIM == 3) { return internal::quadrature_interpolator::DerivativesTranspose3D<Q_LAYOUT, GRAD_PHYS, VDIM, D1D, Q1D>; }
+   else { MFEM_ABORT(""); }
+}
+
+} // namespace mfem

fem/qinterp/grad_transpose_by_nodes.cpp

@@ -0,0 +1,50 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../quadinterpolator.hpp"
+#include "grad_transpose.hpp"
+
+namespace mfem
+{
+namespace internal
+{
+namespace quadrature_interpolator
+{
+
+template <bool P>
+void InitGradTransposeByNodesKernels()
+{
+   using k = QuadratureInterpolator::GradTransposeKernels;
+   constexpr auto L = QVectorLayout::byNODES;
+   // 2D
+   k::Specialization<2,L,P,1,3,4>::template Opt<8>::Add();
+   k::Specialization<2,L,P,1,4,6>::template Opt<4>::Add();
+   k::Specialization<2,L,P,1,5,8>::template Opt<2>::Add();
+
+   k::Specialization<2,L,P,2,3,3>::template Opt<8>::Add();
+   k::Specialization<2,L,P,2,3,4>::template Opt<8>::Add();
+   k::Specialization<2,L,P,2,4,6>::template Opt<4>::Add();
+   k::Specialization<2,L,P,2,5,8>::template Opt<2>::Add();
+   // 3D
+   k::Specialization<3,L,P,1,3,4>::Add();
+   k::Specialization<3,L,P,1,4,6>::Add();
+   k::Specialization<3,L,P,1,5,8>::Add();
+   k::Specialization<3,L,P,3,3,4>::Add();
+   k::Specialization<3,L,P,3,4,6>::Add();
+   k::Specialization<3,L,P,3,5,8>::Add();
+}
+
+template void InitGradTransposeByNodesKernels<false>();
+template void InitGradTransposeByNodesKernels<true>();
+
+} // namespace quadrature_interpolator
+} // namespace internal
+} // namespace mfem

fem/qinterp/grad_transpose_by_vdim.cpp

@@ -0,0 +1,50 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../quadinterpolator.hpp"
+#include "grad_transpose.hpp"
+
+namespace mfem
+{
+namespace internal
+{
+namespace quadrature_interpolator
+{
+
+template <bool P>
+void InitGradTransposeByVDimKernels()
+{
+   using k = QuadratureInterpolator::GradTransposeKernels;
+   constexpr auto L = QVectorLayout::byVDIM;
+   // 2D
+   k::Specialization<2,L,P,1,3,4>::template Opt<8>::Add();
+   k::Specialization<2,L,P,1,4,6>::template Opt<4>::Add();
+   k::Specialization<2,L,P,1,5,8>::template Opt<2>::Add();
+
+   k::Specialization<2,L,P,2,3,3>::template Opt<8>::Add();
+   k::Specialization<2,L,P,2,3,4>::template Opt<8>::Add();
+   k::Specialization<2,L,P,2,4,6>::template Opt<4>::Add();
+   k::Specialization<2,L,P,2,5,8>::template Opt<2>::Add();
+   // 3D
+   k::Specialization<3,L,P,1,3,4>::Add();
+   k::Specialization<3,L,P,1,4,6>::Add();
+   k::Specialization<3,L,P,1,5,8>::Add();
+   k::Specialization<3,L,P,3,3,4>::Add();
+   k::Specialization<3,L,P,3,4,6>::Add();
+   k::Specialization<3,L,P,3,5,8>::Add();
+}
+
+template void InitGradTransposeByVDimKernels<false>();
+template void InitGradTransposeByVDimKernels<true>();
+
+} // namespace quadrature_interpolator
+} // namespace internal
+} // namespace mfem

fem/quadinterpolator.cpp

@@ -11,6 +11,7 @@
 
 #include "quadinterpolator.hpp"
 #include "qinterp/grad.hpp"
+#include "qinterp/grad_transpose.hpp"
 #include "qinterp/eval.hpp"
 #include "qspace.hpp"
 #include "../general/forall.hpp"
@@ -30,7 +31,10 @@ void InitEvalKernels();
 void InitDetKernels();
 template <bool P> void InitGradByNodesKernels();
 template <bool P> void InitGradByVDimKernels();
+template <bool P> void InitGradTransposeByNodesKernels();
+template <bool P> void InitGradTransposeByVDimKernels();
 void InitTensorEvalHDivKernels();
+void InitEvalTransposeByVDimKernels();
 struct Kernels
 {
    Kernels()
@@ -45,12 +49,19 @@ struct Kernels
       // Phys grad kernels
       InitGradByNodesKernels<true>();
       InitGradByVDimKernels<true>();
+      // Non-phys grad transpose kernels
+      InitGradTransposeByNodesKernels<false>();
+      InitGradTransposeByVDimKernels<false>();
+      // Phys grad transpose kernels
+      InitGradTransposeByNodesKernels<true>();
+      InitGradTransposeByVDimKernels<true>();
       // Determinants
       InitDetKernels();
       // Non-tensor
       InitEvalKernels();
       // Tensor (quad,hex) H(div)
       InitTensorEvalHDivKernels();
+      InitEvalTransposeByVDimKernels();
    }
 };
 }
@@ -509,6 +520,7 @@ void QuadratureInterpolator::Mult(const Vector &e_vec,
                                   Vector &q_der,
                                   Vector &q_det) const
 {
+   NVTX_MARK_FUNCTION;
    using namespace internal::quadrature_interpolator;
 
    const int ne = fespace->GetNE();
@@ -552,11 +564,13 @@ void QuadratureInterpolator::Mult(const Vector &e_vec,
    {
       if (eval_flags & (VALUES | PHYSICAL_VALUES))
       {
+         NVTX_MARK("VALUES");
          TensorEvalKernels::Run(dim, q_layout, vdim, nd, nq, ne, maps.B.Read(),
                                 e_vec.Read(), q_val.Write(), vdim, nd, nq);
       }
       if (eval_flags & (DERIVATIVES | PHYSICAL_DERIVATIVES))
       {
+         NVTX_MARK("DERIVATIVES");
          const bool phys = (eval_flags & PHYSICAL_DERIVATIVES);
          const real_t *J = phys ? geom->J.Read() : nullptr;
          const int s_dim = phys ? sdim : dim;
@@ -566,6 +580,7 @@ void QuadratureInterpolator::Mult(const Vector &e_vec,
       }
       if (eval_flags & DETERMINANTS)
       {
+         NVTX_MARK("DETERMINANTS");
          DetKernels::Run(dim, vdim, nd, nq, ne, maps.B.Read(),
                          maps.G.Read(), e_vec.Read(), q_det.Write(), nd,
                          nq, &d_buffer);
@@ -573,6 +588,7 @@ void QuadratureInterpolator::Mult(const Vector &e_vec,
    }
    else // use_tensor_eval == false
    {
+      NVTX_MARK("!TENSOR");
       EvalKernels::Run(dim, vdim, maps.ndof, maps.nqpt, ne,vdim, q_layout,
                        geom, maps, e_vec, q_val, q_der, q_det, eval_flags);
    }
@@ -646,21 +662,68 @@ void QuadratureInterpolator::MultHDiv(const Vector &e_vec,
    MFEM_CONTRACT_VAR(q_div);
 }
 
-void QuadratureInterpolator::MultTranspose(unsigned eval_flags,
-                                           const Vector &q_val,
-                                           const Vector &q_der,
-                                           Vector &e_vec) const
+void QuadratureInterpolator::AddMultTranspose(unsigned eval_flags,
+                                              const Vector &q_val,
+                                              const Vector &q_der,
+                                              Vector &e_vec) const
 {
-   MFEM_CONTRACT_VAR(eval_flags);
-   MFEM_CONTRACT_VAR(q_val);
-   MFEM_CONTRACT_VAR(q_der);
-   MFEM_CONTRACT_VAR(e_vec);
-   MFEM_ABORT("this method is not implemented yet");
+   NVTX_MARK_FUNCTION;
+   const int ne = fespace->GetNE();
+   if (ne == 0) { return; }
+   const FiniteElement *fe = fespace->GetFE(0);
+   const int vdim = fespace->GetVDim();
+   const int sdim = fespace->GetMesh()->SpaceDimension();
+
+   const bool use_tensor_eval =
+      use_tensor_products &&
+      dynamic_cast<const TensorBasisElement*>(fe) != nullptr;
+   const IntegrationRule *ir =
+      IntRule ? IntRule : &qspace->GetElementIntRule(0);
+   const DofToQuad::Mode mode =
+      use_tensor_eval ? DofToQuad::TENSOR : DofToQuad::FULL;
+   const DofToQuad &maps = fe->GetDofToQuad(*ir, mode);
+   const int dim = maps.FE->GetDim();
+   const int nd = maps.ndof;
+   const int nq = maps.nqpt;
+
+   const GeometricFactors *geom = nullptr;
+   if (eval_flags & PHYSICAL_DERIVATIVES)
+   {
+      const int jacobians = GeometricFactors::JACOBIANS;
+      geom = fespace->GetMesh()->GetGeometricFactors(*ir, jacobians);
+   }
+
+   if (use_tensor_eval)
+   {
+      if (eval_flags & (VALUES | PHYSICAL_VALUES))
+      {
+         NVTX_MARK("VALUES");
+         TensorEvalTransposeKernels::Run(dim, q_layout, vdim, nd, nq, ne,
+                                         maps.B.Read(), q_val.Read(),
+                                         e_vec.ReadWrite(), vdim, nd, nq);
+      }
+      if (eval_flags & (DERIVATIVES | PHYSICAL_DERIVATIVES))
+      {
+         NVTX_MARK("DERIVATIVES");
+         const bool phys = (eval_flags & PHYSICAL_DERIVATIVES);
+         const real_t *J = phys ? geom->J.Read() : nullptr;
+         const int s_dim = phys ? sdim : dim;
+         GradTransposeKernels::Run(dim, q_layout, phys, vdim, nd, nq, ne,
+                                   maps.B.Read(), maps.G.Read(), J,
+                                   q_der.Read(), e_vec.ReadWrite(),
+                                   s_dim, vdim, nd, nq);
+      }
+   }
+   else
+   {
+      MFEM_ABORT("Non-tensor MultTranspose not yet implemented");
+   }
 }
 
 void QuadratureInterpolator::Values(const Vector &e_vec,
                                     Vector &q_val) const
 {
+   NVTX_MARK_FUNCTION;
    Vector empty;
    Mult(e_vec, VALUES, q_val, empty, empty);
 }
@@ -668,6 +731,7 @@ void QuadratureInterpolator::Values(const Vector &e_vec,
 void QuadratureInterpolator::PhysValues(const Vector &e_vec,
                                         Vector &q_val) const
 {
+   NVTX_MARK_FUNCTION;
    Vector empty;
    Mult(e_vec, PHYSICAL_VALUES, q_val, empty, empty);
 }
@@ -675,6 +739,7 @@ void QuadratureInterpolator::PhysValues(const Vector &e_vec,
 void QuadratureInterpolator::Derivatives(const Vector &e_vec,
                                          Vector &q_der) const
 {
+   NVTX_MARK_FUNCTION;
    Vector empty;
    Mult(e_vec, DERIVATIVES, empty, q_der, empty);
 }
@@ -682,6 +747,7 @@ void QuadratureInterpolator::Derivatives(const Vector &e_vec,
 void QuadratureInterpolator::PhysDerivatives(const Vector &e_vec,
                                              Vector &q_der) const
 {
+   NVTX_MARK_FUNCTION;
    Vector empty;
    Mult(e_vec, PHYSICAL_DERIVATIVES, empty, q_der, empty);
 }
@@ -689,6 +755,7 @@ void QuadratureInterpolator::PhysDerivatives(const Vector &e_vec,
 void QuadratureInterpolator::Determinants(const Vector &e_vec,
                                           Vector &q_det) const
 {
+   NVTX_MARK_FUNCTION;
    Vector empty;
    Mult(e_vec, DETERMINANTS, empty, empty, q_det);
 }

fem/quadinterpolator.hpp

@@ -152,8 +152,8 @@ public:
    void Determinants(const Vector &e_vec, Vector &q_det) const;
 
    /// Perform the transpose operation of Mult(). (TODO)
-   void MultTranspose(unsigned eval_flags, const Vector &q_val,
-                      const Vector &q_der, Vector &e_vec) const;
+   void AddMultTranspose(unsigned eval_flags, const Vector &q_val,
+                         const Vector &q_der, Vector &e_vec) const;
 
    /// @brief Returns true if the given finite element space is supported by
    /// QuadratureInterpolator.
@@ -178,6 +178,13 @@ public:
    using TensorEvalHDivKernelType =
       void(*)(const int, const real_t *, const real_t *, const real_t *,
               const real_t *, real_t *, const int, const int);
+   using TensorEvalTransposeKernelType = void(*)(const int, const real_t *,
+                                                 const real_t *,
+                                                 real_t *, const int, const int, const int);
+   using GradTransposeKernelType = void(*)(const int, const real_t *,
+                                           const real_t *, const real_t *,
+                                           const real_t *, real_t *,
+                                           const int, const int, const int, const int);
 
    MFEM_REGISTER_KERNELS(TensorEvalKernels, TensorEvalKernelType,
                          (int, QVectorLayout, int, int, int), (int));
@@ -189,6 +196,10 @@ public:
                          (int, QVectorLayout, bool, int, int), (int));
    MFEM_REGISTER_KERNELS(TensorEvalHDivKernels, TensorEvalHDivKernelType,
                          (int, QVectorLayout, unsigned, int, int));
+   MFEM_REGISTER_KERNELS(TensorEvalTransposeKernels, TensorEvalTransposeKernelType,
+                         (int, QVectorLayout, int, int, int), (int));
+   MFEM_REGISTER_KERNELS(GradTransposeKernels, GradTransposeKernelType,
+                         (int, QVectorLayout, bool, int, int, int), (int));
 };
 
 }

general/enzyme.hpp

@@ -58,6 +58,12 @@ return_type __enzyme_fwddiff(Args...);
 #define MFEM_ENZYME_INACTIVENOFREE
 #define MFEM_ENZYME_INACTIVE
 #define MFEM_ENZYME_FN_LIKE(x)
+
+extern int enzyme_dup;
+extern int enzyme_dupnoneed;
+extern int enzyme_out;
+extern int enzyme_const;
+extern int enzyme_interleave;
 #endif
 
 #define MFEM_ENZYME_FN_LIKE_FREE MFEM_ENZYME_FN_LIKE(free)

general/mem_manager.hpp

@@ -318,6 +318,8 @@ public:
    void UseDevice(bool use_dev) const
    { flags = use_dev ? (flags | USE_DEVICE) : (flags & ~USE_DEVICE); }
 
+   void ValidateDevice(bool valid) { flags = valid ? (flags | VALID_DEVICE) : (flags & ~VALID_DEVICE); }
+
    /// Return the size of the allocated memory.
    int Capacity() const { return capacity; }
 

linalg/CMakeLists.txt

@@ -27,6 +27,7 @@ list(APPEND SRCS
   handle.cpp
   matrix.cpp
   mma.cpp
+  multivector.cpp
   ode.cpp
   operator.cpp
   ordering.cpp
@@ -63,6 +64,7 @@ list(APPEND HDRS
   linalg.hpp
   matrix.hpp
   mma.hpp
+  multivector.hpp
   ode.hpp
   operator.hpp
   ordering.hpp

linalg/linalg.hpp

@@ -15,6 +15,7 @@
 // Linear algebra header file
 
 #include "vector.hpp"
+#include "multivector.hpp"
 #include "operator.hpp"
 #include "matrix.hpp"
 #include "sparsemat.hpp"
@@ -37,6 +38,7 @@
 #include "batched/gpu_blas.hpp"
 #include "batched/solver.hpp"
 #include "tensor.hpp"
+// #include "tensor_arrays.hpp"
 #include "filteredsolver.hpp"
 #include "ordering.hpp"
 #include "particlevector.hpp"

linalg/multivector.cpp

@@ -0,0 +1,60 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "multivector.hpp"
+
+namespace mfem
+{
+
+MultiVector::MultiVector(const Array<int> &vector_sizes)
+{
+   SetSizes(vector_sizes);
+}
+
+MultiVector::MultiVector(const Array<int> &vector_sizes, MemoryType mt)
+{
+   SetSizes(vector_sizes, mt);
+}
+
+MultiVector::MultiVector(Vector &base, const Array<int> &vector_sizes)
+{
+   MakeRef(base, vector_sizes);
+}
+
+void MultiVector::SetSizes(const Array<int> &vector_sizes)
+{
+   blocks.resize(vector_sizes.Size());
+   for (int i = 0; i < vector_sizes.Size(); i++)
+   {
+      operator[](i).SetSize(vector_sizes[i]);
+   }
+}
+
+void MultiVector::SetSizes(const Array<int> &vector_sizes, MemoryType mt)
+{
+   blocks.resize(vector_sizes.Size());
+   for (int i = 0; i < vector_sizes.Size(); i++)
+   {
+      operator[](i).SetSize(vector_sizes[i], mt);
+   }
+}
+
+void MultiVector::MakeRef(Vector &base, const Array<int> &vector_sizes)
+{
+   blocks.resize(vector_sizes.Size());
+   for (int offset = 0, i = 0; i < vector_sizes.Size(); i++)
+   {
+      blocks[i].emplace<0>(base, offset, vector_sizes[i]);
+      offset += vector_sizes[i];
+   }
+}
+
+} // namespace mfem

linalg/multivector.hpp

@@ -0,0 +1,198 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_MULTIVECTOR_HPP
+#define MFEM_MULTIVECTOR_HPP
+
+#include "../general/array.hpp"
+#include "vector.hpp"
+#include <vector>
+#include <array>
+#include <variant>
+
+namespace mfem
+{
+
+/// Class representing an array of Vectors with generally different sizes.
+/** This class is similar to BlockVector with the following two main
+    differences:
+    - the data for the individual Vector blocks does not need to be part of one
+      big contiguous memory allocation;
+    - this class does not inherit from class Vector (as a consequence of the
+      first bullet).
+
+    Internally, each Vector block is represented as either:
+    - (default) a Vector object constructed and owned by this class; this
+      object, in turn, as any Vector object, can own its Memory allocation or
+      refer to a sub-Memory of another Memory object; or
+    - a pointer to an externally allocated Vector or classes derived from
+      Vector. */
+class MultiVector
+{
+private:
+   std::vector<std::variant<Vector,Vector*>> blocks;
+
+public:
+   /// Create an empty MultiVector with zero blocks.
+   MultiVector() = default;
+
+   /** @brief Create a MultiVector with @a num_blocks blocks. The individual
+       Vector blocks are default initialized, i.e. they all have size zero. */
+   MultiVector(int num_blocks)
+      : blocks(num_blocks) { }
+
+   /** @brief Construct a MultiVector with number of blocks and individual block
+       Vector sizes given by @a vector_sizes.
+
+       @note The memory of the individual Vector blocks is NOT initialized. */
+   MultiVector(const Array<int> &vector_sizes);
+
+   /** @brief Construct a MultiVector with number of blocks and individual block
+       Vector sizes given by @a vector_sizes. All Vector blocks use the
+       MemoryType @a mt.
+
+       @note The memory of the individual Vector blocks is NOT initialized. */
+   MultiVector(const Array<int> &vector_sizes, MemoryType mt);
+
+   /** @brief Construct a MultiVector referencing data within a given monolithic
+       Vector @a base.
+
+       With this constructor, the Memory flags of @a base and of the individual
+       Vector blocks may need to be explicitly synchronized when data is moved
+       between host and device. */
+   MultiVector(Vector &base, const Array<int> &vector_sizes);
+
+   /** @brief Construct a MultiVector referencing multiple Vectors given as
+       arguments.
+
+       The VectorTypes reference arguments are expected to be static_cast-able
+       to (Vector &) which is the case if the types are derived from Vector,
+       e.g. HypreParVector, GridFunction, etc.
+
+       With this constructor, operations on individual Vector blocks are
+       performed directly on the objects @a vs. In particular, there is no need
+       to synchronize the Memory flags of @a vs and the ones of the individual
+       Vector blocks when data is moved between host and device. */
+   template <typename... VectorTypes,
+             std::enable_if_t<
+                std::conjunction_v<
+                   std::is_convertible<VectorTypes&,Vector&>...>, bool> = true>
+   MultiVector(VectorTypes &...vs) { MakeRef(vs...); }
+
+   /// Return the number of Vectors in the MultiVector.
+   int NumBlocks() const { return blocks.size(); }
+
+   /** @brief Set the number of Vectors in the MultiVector. Existing Vector
+       blocks will remain unmodified. New Vector blocks will be default
+       initialized, i.e. they all have size zero. */
+   void SetNumBlocks(int num_blocks) { blocks.resize(num_blocks); }
+
+   /// Read-write access to the i-th Vector.
+   inline Vector &operator[](int i);
+
+   /// Read-only access to the i-th Vector.
+   inline const Vector &operator[](int i) const;
+
+   /** @brief Update the MultiVector according to the given @a vector_sizes.
+
+       This method can be used to add or remove blocks. The individual Vector
+       sizes are updated using the method Vector::SetSize(int). */
+   void SetSizes(const Array<int> &vector_sizes);
+
+   /** @brief Update the MultiVector according to the given @a vector_sizes and
+       MemoryType @a mt.
+
+       This method can be used to add or remove blocks. The individual Vector
+       sizes and MemoryType are updated using the method
+       Vector::SetSize(int, MemoryType). */
+   void SetSizes(const Array<int> &vector_sizes, MemoryType mt);
+
+   /** @brief Update the MultiVector to reference data within a given monolithic
+       Vector @a base.
+
+       After calling this method, the Memory flags of @a base and of the
+       individual Vector blocks may need to be explicitly synchronized when data
+       is moved between host and device.*/
+   void MakeRef(Vector &base, const Array<int> &vector_sizes);
+
+   /** @brief Update the @a i-th MultiVector block to reference data within the
+       given monolithic Vector @a base at the given @a offset and with the given
+       @a size.
+
+       After calling this method, the Memory flags of @a base and of the @a i-th
+       Vector block may need to be explicitly synchronized when data is moved
+       between host and device.*/
+   inline void MakeRef(int i, Vector &base, int offset, int size)
+   {
+      blocks[i].emplace<0>(base, offset, size);
+   }
+
+   /** @brief Update the MultiVector to reference multiple Vectors given as
+       arguments.
+
+       The VectorTypes reference arguments are expected to be static_cast-able
+       to (Vector &) which is the case if the types are derived from Vector,
+       e.g. HypreParVector, GridFunction, etc.
+
+       After calling this method, operations on individual Vector blocks are
+       performed directly on the objects @a vs. In particular, there is no need
+       to synchronize the Memory flags of @a vs and the ones of the individual
+       Vector blocks when data is moved between host and device. */
+   template <typename... VectorTypes,
+             std::enable_if_t<
+                std::conjunction_v<
+                   std::is_convertible<VectorTypes&,Vector&>...>, bool> = true>
+   inline void MakeRef(VectorTypes &...vs);
+
+   /** @brief Update the @a i-th MultiVector block to reference the given
+       Vector @a v.
+
+       After calling this method, operations on the @a i-th Vector block are
+       performed directly on the Vector @a v. In particular, there is no need
+       to synchronize the Memory flags of @a v and the ones of the @a i-th
+       Vector blocks when data is moved between host and device. */
+   inline void MakeRef(int i, Vector &v) { blocks[i] = &v; }
+};
+
+// Inline and template methods
+
+inline Vector &MultiVector::operator[](int i)
+{
+   auto &bi = blocks[i];
+   return (bi.index() == 0) ? std::get<0>(bi) : *std::get<1>(bi);
+}
+
+inline const Vector &MultiVector::operator[](int i) const
+{
+   auto &bi = blocks[i];
+   return (bi.index() == 0) ? std::get<0>(bi) : *std::get<1>(bi);
+}
+
+template <typename... VectorTypes,
+          std::enable_if_t<
+             std::conjunction_v<
+                std::is_convertible<VectorTypes&,Vector&>...>, bool>>
+inline void MultiVector::MakeRef(VectorTypes &...vs)
+{
+   blocks.resize(sizeof...(vs));
+   if constexpr (sizeof...(vs) > 0)
+   {
+      const std::array vs_p{&static_cast<Vector&>(vs)...};
+      for (std::size_t i = 0; i < sizeof...(vs); i++)
+      {
+         blocks[i] = vs_p[i];
+      }
+   }
+}
+
+} // namespace mfem
+
+#endif // MFEM_MULTIVECTOR_HPP

linalg/operator.cpp

@@ -111,11 +111,22 @@ void Operator::ArrayAddMultTranspose(const Array<const Vector *> &X,
    }
 }
 
+void Operator::Mult(const MultiVector &, MultiVector &)
+{
+   MFEM_ABORT("this method is not overriden for this class!");
+}
+
+Operator &Operator::GetGradient(const MultiVector &) const
+{
+   MFEM_ABORT("this method is not overriden for this class!");
+}
+
 void Operator::FormLinearSystem(const Array<int> &ess_tdof_list,
                                 Vector &x, Vector &b,
                                 Operator* &Aout, Vector &X, Vector &B,
                                 int copy_interior)
 {
+   NVTX_MARK_FUNCTION;
    const Operator *P = this->GetProlongation();
    const Operator *R = this->GetRestriction();
    InitTVectors(P, R, P, x, b, X, B);
@@ -197,6 +208,7 @@ Operator * Operator::SetupRAP(const Operator *Pi, const Operator *Po)
 void Operator::FormConstrainedSystemOperator(
    const Array<int> &ess_tdof_list, ConstrainedOperator* &Aout)
 {
+   NVTX_MARK_FUNCTION;
    const Operator *P = this->GetProlongation();
    Operator *rap = SetupRAP(P, P);
 
@@ -514,6 +526,7 @@ ConstrainedOperator::ConstrainedOperator(Operator *A, const Array<int> &list,
    : Operator(A->Height(), A->Width()), A(A), own_A(own_A_),
      diag_policy(diag_policy_)
 {
+   NVTX_MARK_FUNCTION;
    // 'mem_class' should work with A->Mult() and mfem::forall():
    mem_class = A->GetMemoryClass()*Device::GetDeviceMemoryClass();
    MemoryType mem_type = GetMemoryType(mem_class);
@@ -527,6 +540,7 @@ ConstrainedOperator::ConstrainedOperator(Operator *A, const Array<int> &list,
 
 void ConstrainedOperator::AssembleDiagonal(Vector &diag) const
 {
+   NVTX_MARK_FUNCTION;
    A->AssembleDiagonal(diag);
 
    if (diag_policy == DIAG_KEEP) { return; }
@@ -558,6 +572,7 @@ void ConstrainedOperator::AssembleDiagonal(Vector &diag) const
 
 void ConstrainedOperator::EliminateRHS(const Vector &x, Vector &b) const
 {
+   NVTX_MARK_FUNCTION;
    w = 0.0;
    const int csz = constraint_list.Size();
    auto idx = constraint_list.Read();
@@ -586,6 +601,7 @@ void ConstrainedOperator::EliminateRHS(const Vector &x, Vector &b) const
 void ConstrainedOperator::ConstrainedMult(const Vector &x, Vector &y,
                                           const bool transpose) const
 {
+   NVTX_MARK_FUNCTION;
    const int csz = constraint_list.Size();
    if (csz == 0)
    {
@@ -600,12 +616,16 @@ void ConstrainedOperator::ConstrainedMult(const Vector &x, Vector &y,
       return;
    }
 
+   NVTX_MARK_INI("z=x");
    z = x;
+   NVTX_MARK_END("z=x");
 
+   NVTX_MARK_INI("z[bc]=0.0");
    auto idx = constraint_list.Read();
    // Use read+write access - we are modifying sub-vector of z
    auto d_z = z.ReadWrite();
    mfem::forall(csz, [=] MFEM_HOST_DEVICE (int i) { d_z[idx[i]] = 0.0; });
+   NVTX_MARK_END("z[bc]=0.0");
 
    if (transpose)
    {
@@ -613,9 +633,11 @@ void ConstrainedOperator::ConstrainedMult(const Vector &x, Vector &y,
    }
    else
    {
+      NVTX_MARK("A->Mult(z, y)");
       A->Mult(z, y);
    }
 
+   NVTX_MARK_INI("DIAG");
    auto d_x = x.Read();
    // Use read+write access - we are modifying sub-vector of y
    auto d_y = y.ReadWrite();
@@ -643,6 +665,7 @@ void ConstrainedOperator::ConstrainedMult(const Vector &x, Vector &y,
          mfem_error("ConstrainedOperator::Mult #2");
          break;
    }
+   NVTX_MARK_END("DIAG");
 }
 
 void ConstrainedOperator::ConstrainedAbsMult(const Vector &x, Vector &y,
@@ -709,6 +732,7 @@ void ConstrainedOperator::ConstrainedAbsMult(const Vector &x, Vector &y,
 
 void ConstrainedOperator::Mult(const Vector &x, Vector &y) const
 {
+   NVTX_MARK_FUNCTION;
    constexpr bool transpose = false;
    ConstrainedMult(x, y, transpose);
 }
@@ -721,6 +745,7 @@ void ConstrainedOperator::AbsMult(const Vector &x, Vector &y) const
 
 void ConstrainedOperator::MultTranspose(const Vector &x, Vector &y) const
 {
+   NVTX_MARK_FUNCTION;
    constexpr bool transpose = true;
    ConstrainedMult(x, y, transpose);
 }
@@ -734,6 +759,7 @@ void ConstrainedOperator::AbsMultTranspose(const Vector &x, Vector &y) const
 void ConstrainedOperator::AddMult(const Vector &x, Vector &y,
                                   const real_t a) const
 {
+   NVTX_MARK_FUNCTION;
    Mult(x, w);
    y.Add(a, w);
 }

linalg/operator.hpp

@@ -13,6 +13,7 @@
 #define MFEM_OPERATOR
 
 #include "vector.hpp"
+#include "multivector.hpp"
 
 namespace mfem
 {
@@ -129,6 +130,16 @@ public:
    virtual void ArrayAddMultTranspose(const Array<const Vector *> &X,
                                       Array<Vector *> &Y, const real_t a = 1.0) const;
 
+   /** @brief Operator application, y = A(x), where the input @a x and the
+       output @a y are MultiVector objects, i.e. they generally use
+       non-contiguous memory representation.
+
+       The typical use case for this method are block operators like
+       DifferentiableOperator.
+
+       The base class implementation for the method is to generate an error. */
+   virtual void Mult(const MultiVector &x, MultiVector &y);
+
    /** @brief Evaluate the gradient operator at the point @a x. The default
        behavior in class Operator is to generate an error. */
    virtual Operator &GetGradient(const Vector &x) const
@@ -137,6 +148,16 @@ public:
       return const_cast<Operator &>(*this);
    }
 
+   /** @brief Evaluate the gradient operator at the point @a x. The input @a x
+       is provided as a MultiVector, i.e. it generally uses non-contiguous
+       memory representation.
+
+       The typical use case for this method are block operators like
+       DifferentiableOperator.
+
+       The base class implementation for the method is to generate an error. */
+   virtual Operator &GetGradient(const MultiVector &x) const;
+
    /** @brief Computes the diagonal entries into @a diag. Typically, this
        operation only makes sense for linear Operator%s. In some cases, only an
        approximation of the diagonal is computed. */

linalg/solvers.cpp

@@ -57,6 +57,7 @@ IterativeSolver::IterativeSolver(MPI_Comm comm_)
 
 real_t IterativeSolver::Dot(const Vector &x, const Vector &y) const
 {
+   NVTX_MARK_FUNCTION;
    if (dot_oper) { return dot_oper->Eval(x,y); } // Use custom inner product (if provided)
 
 #ifndef MFEM_USE_MPI
@@ -868,6 +869,7 @@ void CGSolver::UpdateVectors()
 
 void CGSolver::Mult(const Vector &b, Vector &x) const
 {
+   NVTX_MARK_FUNCTION;
    int i;
    real_t r0, den, nom, nom0, betanom, alpha, beta;
 

linalg/tensor.hpp

@@ -33,17 +33,62 @@ struct tensor;
 /// The implementation can be drastically generalized by using concepts of the
 /// c++17 standard.
 
-template < typename T >
+template <typename T>
 struct tensor<T>
 {
    using type = T;
    static constexpr int ndim      = 1;
    static constexpr int first_dim = 0;
-   MFEM_HOST_DEVICE T& operator[](int /*unused*/) { return values; }
-   MFEM_HOST_DEVICE const T& operator[](int /*unused*/) const { return values; }
-   MFEM_HOST_DEVICE T& operator()(int /*unused*/) { return values; }
-   MFEM_HOST_DEVICE const T& operator()(int /*unused*/) const { return values; }
+
+   MFEM_HOST_DEVICE tensor() = default;
+   MFEM_HOST_DEVICE tensor(T v) : values(v) {}
+
+   MFEM_HOST_DEVICE T& operator[](int) { return values; }
+   MFEM_HOST_DEVICE const T& operator[](int) const { return values; }
+   MFEM_HOST_DEVICE T& operator()(int) { return values; }
+   MFEM_HOST_DEVICE const T& operator()(int) const { return values; }
+   MFEM_HOST_DEVICE T& operator()() { return values; }
+   MFEM_HOST_DEVICE const T& operator()() const { return values; }
+
    MFEM_HOST_DEVICE operator T() const { return values; }
+
+   MFEM_HOST_DEVICE constexpr const T& scalar() const { return values; }
+
+   // A * tensor<T>  ->  A * T
+   template <typename A,
+             std::enable_if_t<!std::is_same_v<std::decay_t<A>, tensor>, int> = 0>
+   MFEM_HOST_DEVICE friend auto operator*(const A& a, const tensor& s)
+   -> decltype(a * s.scalar())
+   {
+      return a * s.scalar();
+   }
+
+   // tensor<T> * A  ->  T * A
+   template <typename A,
+             std::enable_if_t<!std::is_same_v<std::decay_t<A>, tensor>, int> = 0>
+   MFEM_HOST_DEVICE friend auto operator*(const tensor& s, const A& a)
+   -> decltype(s.scalar() * a)
+   {
+      return s.scalar() * a;
+   }
+
+   // A / tensor<T>, tensor<T> / A
+   template <typename A,
+             std::enable_if_t<!std::is_same_v<std::decay_t<A>, tensor>, int> = 0>
+   MFEM_HOST_DEVICE friend auto operator/(const A& a, const tensor& s)
+   -> decltype(a / s.scalar())
+   {
+      return a / s.scalar();
+   }
+
+   template <typename A,
+             std::enable_if_t<!std::is_same_v<std::decay_t<A>, tensor>, int> = 0>
+   MFEM_HOST_DEVICE friend auto operator/(const tensor& s, const A& a)
+   -> decltype(s.scalar() / a)
+   {
+      return s.scalar() / a;
+   }
+
    T values;
 };
 
@@ -541,13 +586,21 @@ MFEM_HOST_DEVICE auto operator*(S scale, const tensor<T, n...>& A) ->
 tensor<decltype(S {} * T{}), n...>
 {
    tensor<decltype(S{} * T{}), n...> C{};
-   for (int i = 0; i < tensor<T, n...>::first_dim; i++)
+   if constexpr (sizeof...(n) == 0)
    {
-      C[i] = scale * A[i];
+      C.values = scale * A.values;
+   }
+   else
+   {
+      for (int i = 0; i < tensor<T, n...>::first_dim; i++)
+      {
+         C[i] = scale * A[i];
+      }
    }
    return C;
 }
 
+
 /**
  * @brief multiply a tensor by a scalar value
  * @tparam S the scalar value type. Must be arithmetic (e.g. float, real_t, int) or a dual number
@@ -563,9 +616,16 @@ MFEM_HOST_DEVICE auto operator*(const tensor<T, n...>& A, S scale) ->
 tensor<decltype(T {} * S{}), n...>
 {
    tensor<decltype(T{} * S{}), n...> C{};
-   for (int i = 0; i < tensor<T, n...>::first_dim; i++)
+   if constexpr (sizeof...(n) == 0)
    {
-      C[i] = A[i] * scale;
+      C.values = A.values * scale;
+   }
+   else
+   {
+      for (int i = 0; i < tensor<T, n...>::first_dim; i++)
+      {
+         C[i] = A[i] * scale;
+      }
    }
    return C;
 }
@@ -729,9 +789,9 @@ auto outer(S A, T B) -> decltype(A * B)
 }
 
 template <typename T, int n, int m> MFEM_HOST_DEVICE
-tensor<T, n + m> flatten(tensor<T, n, m> A)
+tensor<T, n * m> flatten(tensor<T, n, m> A)
 {
-   tensor<T, n + m> B{};
+   tensor<T, n * m> B{};
    for (int i = 0; i < n; i++)
    {
       for (int j = 0; j < m; j++)

linalg/tensor_arrays.hpp

@@ -0,0 +1,380 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_TENSOR_ARRAYS_HPP
+#define MFEM_TENSOR_ARRAYS_HPP
+
+#include "tensor.hpp"
+#include <array>       // std::array, std::size_t (indirectly)
+#include <type_traits> /* std::remove_cv_t, std::remove_reference_t,
+                          std::is_const_v */
+#include <utility>     /* std::forward, std::index_sequence,
+                          std::make_index_sequence */
+#include <algorithm>   // std::min
+#include <tuple>       // std::apply, std::tuple_size_v
+#include <numeric>     // std::iota
+
+namespace mfem
+{
+
+namespace future
+{
+
+template <std::size_t... Is, typename Fn>
+constexpr inline void for_unrolled_simple(std::index_sequence<Is...>, Fn &&fn)
+{
+   (fn(Is), ...);
+}
+
+
+template <int... loop_sizes>
+__attribute__((annotate("enzyme_inactive")))
+constexpr inline auto to_multiindex(std::size_t i)
+{
+   constexpr auto dims = sizeof...(loop_sizes);
+   constexpr std::array<std::size_t,dims> sizes{loop_sizes...};
+   std::array<std::size_t,dims> is{};  // value initialization with zeros
+   for (std::size_t d = 0; d < dims; d++)
+   {
+      is[d] = i%sizes[d];
+      i /= sizes[d];
+   }
+   return is;
+}
+
+
+/// lambda_t:
+/// - input: const std::array<std::size_t,sizeof...(loop_sizes)> &
+/// - output: void
+/// Note: 0D loop executes the lambda one time with an array of dim 0.
+template <int... loop_sizes, typename lambda_t>
+constexpr inline void for_multiindex(lambda_t f)
+{
+   constexpr auto dims = sizeof...(loop_sizes);
+   if constexpr (dims == 0)
+   {
+      f(std::array<std::size_t,0> {});
+   }
+   else
+   {
+      if constexpr (std::min({loop_sizes...}) <= 0) { return; }
+      constexpr auto total_loop_size = (loop_sizes * ...);
+      for_unrolled_simple(std::make_index_sequence<total_loop_size> {},
+                          [&f](std::size_t i)
+      {
+         f(to_multiindex<loop_sizes...>(i));
+      });
+   }
+}
+
+
+/// Extend std::apply to work with 0-size arrays.
+template <typename Fn, typename Tuple>
+inline constexpr decltype(auto) apply(Fn&& f, Tuple&& t)
+{
+   if constexpr (std::tuple_size_v<std::remove_reference_t<Tuple>> == 0)
+   { return f(); }
+   return std::apply(std::forward<Fn>(f), std::forward<Tuple>(t));
+}
+
+
+/// Multi-dimensional array of tensors of the same size.
+/** The array sizes are dynamic while the tensor sizes are static, i.e. template
+    parameters.
+
+    This class provides flexible global data layout where the dynamic (array)
+    dimnsions and the tensor dimnsions are stored in memory using a runtime
+    defined strided layout. */
+template <typename scalar_t, int ndims, int... tensor_sizes>
+class tensor_ndarray
+{
+public:
+   using scalar_type = scalar_t;
+   using tensor_type = tensor<std::remove_cv_t<scalar_t>,tensor_sizes...>;
+
+   static constexpr std::integer_sequence<size_t, tensor_sizes...> tensor_sizes_;
+   static constexpr auto tensor_dims = sizeof...(tensor_sizes);
+   static constexpr auto total_dims = ndims + tensor_dims;
+#ifdef __NVCC__
+   // nvcc does not always emit the device-side symbol
+   static constexpr auto tensor_sizes_array() noexcept
+   {
+      return std::array<std::size_t, tensor_dims> {tensor_sizes...};
+   }
+#else
+   inline static constexpr std::array<std::size_t,tensor_dims>
+   tensor_sizes_array{tensor_sizes...};
+#endif
+
+private:
+   scalar_t *data;  /// Not owned
+public:
+   std::array<std::size_t,ndims> dyn_sizes;
+   mutable std::array<std::size_t,total_dims> strides;
+
+public:
+   /** @brief Constructor with the default, column-major or left, layout where
+       the dynamic dimensions are first, on the left, and the tensor dimensions
+       are second. */
+   tensor_ndarray(scalar_t *ptr, std::array<std::size_t,ndims> dynamic_sizes)
+      : data(ptr), dyn_sizes(dynamic_sizes)
+   {
+      std::array<std::size_t,total_dims> default_perm;
+      std::iota(default_perm.begin(), default_perm.end(), 0); // 0, 1, 2, ...
+      set_layout(default_perm);
+   }
+
+   /// Number of dynamic array dimensions.
+   static constexpr std::size_t rank() { return ndims; }
+
+   scalar_t *get_data() const { return data; }
+
+   /// Array size in the @a k-th dynamic dimension.
+   std::size_t size(int k = 0) const { return dyn_sizes[k]; }
+
+   /// Returns the product of all sizes of the dynamic dimensions.
+   std::size_t total_size() const
+   {
+      std::size_t t = 1;
+      for (int d = 0; d < ndims; d++)
+      {
+         t *= dyn_sizes[d];
+      }
+      return t;
+   }
+
+   /// Number of tensor (static) dimensions.
+   static constexpr std::size_t tensor_rank()
+   { return sizeof...(tensor_sizes); }
+
+   /// Tensor size in the @a k-th tensor (static) dimension.
+   static constexpr std::size_t tensor_size(int k = 0)
+   {
+#ifdef __NVCC__
+      return tensor_sizes_array()[k];
+#else
+      return tensor_sizes_array[k];
+#endif
+   }
+
+   /// Returns the product of all sizes of the static (tensor) dimensions.
+   static constexpr std::size_t total_tensor_size()
+   { return (tensor_sizes * ...); }
+
+   /// Set the global data layout based on the given permutation @a perm.
+   /** The entries of @a perm are numbers identifying either a dynamic or a
+       tensor (static) dimension. Values in the range [0,rank()) identify the
+       dynamic dimensions and values in the range [rank(),rank()+tensor_rank())
+       identify the tensor dymensions. The first entry in @a perm determines
+       which dynamic or tensor dimension will have stride 1. The k-th entry of
+       @a perm determines which dimension will use the next stride which is
+       defined as the product of the sizes of all k-1 previous dimensions from
+       @a perm.
+
+       @note The default layout corresponds to the identity permutation:
+       { 0, 1, ..., rank()+tensor_rank()-1 }.
+
+       @note This method does not permute the global 1D data array. */
+   MFEM_HOST_DEVICE
+   void set_layout(std::array<std::size_t,rank()+tensor_rank()> perm) const
+   {
+      std::size_t stride = 1;
+      for (std::size_t d_g = 0; d_g < total_dims; d_g++)
+      {
+         const auto d_l = perm[d_g];
+         strides[d_l] = stride;
+         stride *= (d_l < ndims) ? dyn_sizes[d_l] :
+#ifdef __NVCC__
+                   this->tensor_sizes_array()[d_l-ndims];
+#else
+                   this->tensor_sizes_array[d_l-ndims];
+#endif
+      }
+   }
+
+   /** @brief Comute the dynamic offset for a given dynamic multi-index @a is.
+       The total offset in the global data array is the sum of the dynamic and
+       static (tensor) offsets. */
+   MFEM_HOST_DEVICE
+   std::size_t get_dynamic_offset(
+      const std::array<std::size_t,rank()> &is) const
+   {
+      std::size_t dynamic_offset = 0;
+      for (std::size_t d = 0; d < ndims; d++)
+      {
+         dynamic_offset += is[d]*strides[d];
+      }
+      return dynamic_offset;
+   }
+
+   /** @brief Comute the static (tensor) offset for a given tensor multi-index
+       @a js. The total offset in the global data array is the sum of the
+       dynamic and static (tensor) offsets. */
+   MFEM_HOST_DEVICE
+   std::size_t get_static_offset(
+      const std::array<std::size_t,tensor_rank()> &js) const
+   {
+      std::size_t static_offset = 0;
+      for (std::size_t d = 0; d < tensor_dims; d++)
+      {
+         static_offset += js[d]*strides[ndims+d];
+      }
+      return static_offset;
+   }
+
+   /** @brief Return a local tensor extracted from the global data array
+       corresponding to the given dynamic multi-index @a is. */
+   /** @note Return a const tensor to prevent attempts to assign to the
+       temporary object which is considered a mistake. */
+   MFEM_HOST_DEVICE
+   const tensor_type get_tensor(std::array<std::size_t,rank()> is) const
+   {
+      tensor_type result;
+      const std::size_t dynamic_offset = get_dynamic_offset(is);
+      for_multiindex<tensor_sizes...>(
+         [&result, this, dynamic_offset](
+            const std::array<std::size_t,tensor_rank()> &js)
+      {
+         ::mfem::future::apply(result, js) =
+            data[dynamic_offset + get_static_offset(js)];
+      });
+      return result;
+   }
+
+   /** @brief Return a local tensor extracted from the global data array
+       corresponding to the given dynamic indices @a is. */
+   /** @note Return a const tensor to prevent attempts to assign to the
+       temporary object which is considered a mistake. */
+   template <typename... index_types> MFEM_HOST_DEVICE
+   const tensor_type get_tensor(index_types... is) const
+   {
+      static_assert(sizeof...(is) == rank(), "invalid number of indices!");
+      return get_tensor(std::array<std::size_t,rank()> {std::size_t(is)...});
+   }
+
+   /** @brief Returns one of the following depending on the type scalar_t:
+       - get_tensor(std::array<std::size_t,rank()>) iff scalar_t is const,
+       - get_accessor(std::array<std::size_t,rank()>) iff scalar_t is not
+         const. */
+   MFEM_HOST_DEVICE
+   decltype(auto) operator()(std::array<std::size_t,rank()> is) const
+   {
+      if constexpr (std::is_const_v<scalar_t>) { return get_tensor(is); }
+      else { return get_accessor(is); }
+   }
+
+   /** @brief Returns one of the following depending on the type scalar_t:
+       - get_tensor(index_types...) iff scalar_t is const,
+       - get_accessor(index_types...) iff scalar_t is not const. */
+   template <typename... index_types> MFEM_HOST_DEVICE
+   decltype(auto) operator()(index_types... is) const
+   {
+      if constexpr (std::is_const_v<scalar_t>) { return get_tensor(is...); }
+      else { return get_accessor(is...); }
+   }
+
+   /** @brief Helper class facilitating the reading/writing of local tensor
+       objects to the global data array of the tensor_ndarray. */
+   class tensor_accessor
+   {
+   private:
+      const tensor_ndarray &base_array;
+      scalar_t *offset_data;  /// Not owned
+
+   public:
+      /** @brief Construct a tensor_accessor to @a base for the given dynamic
+          multi-index @a is.
+
+          During its life time, this object assumes that the @a base object
+          remains unmodified. */
+      MFEM_HOST_DEVICE
+      tensor_accessor(const tensor_ndarray &base,
+                      const std::array<std::size_t,rank()> &is)
+         : base_array(base)
+      {
+         offset_data = base_array.data + base_array.get_dynamic_offset(is);
+      }
+
+      /// Read-write access to a particular entry of the referenced tensor.
+      /** The returned reference points to the corresponding entry in the global
+          data array of the base tensor_ndarray. */
+      MFEM_HOST_DEVICE
+      scalar_t &operator()(const std::array<std::size_t,tensor_rank()> &js)
+      {
+         return offset_data[base_array.get_static_offset(js)];
+      }
+
+      /** @brief Write a tensor to the referenced tensor in the global data
+          array of the base tensor_ndarray. */
+      MFEM_HOST_DEVICE
+      tensor_accessor &operator=(const tensor_type &rhs)
+      {
+         for_multiindex<tensor_sizes...>(
+            [&](const std::array<std::size_t,tensor_rank()> &js)
+         {
+            operator()(js) = ::mfem::future::apply(rhs, js);
+         });
+         return *this;
+      }
+   };
+
+   /** @brief Get a tensor_accessor object referencing the tensor stored at the
+       dynamic multi-index @a is. This object can be used to write tensor
+       objects into the global data array of the tensor_ndarray. */
+   MFEM_HOST_DEVICE
+   tensor_accessor get_accessor(std::array<std::size_t,rank()> is) const
+   {
+      return tensor_accessor(*this, is);
+   }
+
+   /** @brief Get a tensor_accessor object referencing the tensor stored at the
+       dynamic indices @a is. This object can be used to write tensor objects
+       into the global data array of the tensor_ndarray. */
+   template <typename... index_types> MFEM_HOST_DEVICE
+   tensor_accessor get_accessor(index_types... is) const
+   {
+      static_assert(sizeof...(is) == rank(), "invalid number of indices!");
+      return get_accessor(std::array<std::size_t,rank()> {std::size_t(is)...});
+   }
+};
+
+
+/** @brief Construct a tensor_ndarray where only the tensor sizes have to be
+    explicitly given as template parameters, the rest can be deduced from the
+    function call arguments. */
+template <int... tensor_sizes, typename scalar_t, typename... dyn_sizes_t>
+decltype(auto) make_tensor_ndarray(scalar_t *ptr, dyn_sizes_t... dynamic_sizes)
+{
+   NVTX_MARK_FUNCTION;
+   return tensor_ndarray<scalar_t,sizeof...(dynamic_sizes),tensor_sizes...>(
+             ptr, {std::size_t(dynamic_sizes)...});
+}
+
+
+/// Alias for make_tensor_ndarray = make_tensor_array.
+template <int... tensor_sizes, typename scalar_t, typename... dyn_sizes_t>
+decltype(auto) make_tensor_array(scalar_t *ptr, dyn_sizes_t... dynamic_sizes)
+{
+   NVTX_MARK_FUNCTION;
+   return tensor_ndarray<scalar_t,sizeof...(dynamic_sizes),tensor_sizes...>(
+             ptr, {std::size_t(dynamic_sizes)...});
+}
+
+
+/// Short name for one-dimensional tensor_ndarray: tensor_array.
+template <typename scalar_t, int... tensor_sizes>
+using tensor_array = tensor_ndarray<scalar_t, 1, tensor_sizes...>;
+
+} // namespace mfem::future
+
+} // namespace mfem
+
+#endif // MFEM_TENSOR_ARRAYS_HPP

linalg/vector.cpp

@@ -413,6 +413,7 @@ void Vector::Pow(const real_t p)
 
 void add(const Vector &v1, const Vector &v2, Vector &v)
 {
+   NVTX_MARK_FUNCTION;
    MFEM_ASSERT(v.size == v1.size && v.size == v2.size,
                "incompatible Vectors!");
 
@@ -435,6 +436,7 @@ void add(const Vector &v1, const Vector &v2, Vector &v)
 
 void add(const Vector &v1, real_t alpha, const Vector &v2, Vector &v)
 {
+   NVTX_MARK_FUNCTION;
    MFEM_ASSERT(v.size == v1.size && v.size == v2.size,
                "incompatible Vectors!");
 
@@ -1078,6 +1080,8 @@ real_t Vector::Normlp(real_t p) const
 
 real_t Vector::operator*(const Vector &v) const
 {
+   NVTX_MARK_FUNCTION;
+
    MFEM_ASSERT(size == v.size, "incompatible Vectors!");
 
    if (size == 0) { return 0.0; }

makefile

@@ -119,7 +119,7 @@ $(if $(word 2,$(SRC)),$(error Spaces in SRC = "$(SRC)" are not supported))
 MFEM_GIT_STRING = $(shell [ -d $(MFEM_DIR)/.git ] && git -C $(MFEM_DIR) \
    describe --all --long --abbrev=40 --dirty --always 2> /dev/null)
 
-EXAMPLE_SUBDIRS = amgx caliper ginkgo hiop petsc pumi sundials superlu moonolith
+EXAMPLE_SUBDIRS = amgx dfem caliper ginkgo hiop petsc pumi sundials superlu moonolith
 EXAMPLE_DIRS := examples $(addprefix examples/,$(EXAMPLE_SUBDIRS))
 EXAMPLE_TEST_DIRS := examples
 
@@ -807,7 +807,7 @@ FORMAT_EXCLUDE = general/tinyxml2.cpp tests/unit/catch.hpp
 FORMAT_LIST = $(filter-out $(FORMAT_EXCLUDE),$(wildcard $(FORMAT_FILES)))
 
 COUT_CERR_FILES = $(foreach dir,$(DIRS),$(dir)/*.[ch]pp)
-COUT_CERR_EXCLUDE = '^general/error\.cpp' '^general/globals\.[ch]pp'
+COUT_CERR_EXCLUDE = '^general/error\.cpp' '^general/globals\.[ch]pp' '^general/nvtx\.hpp'
 
 DEPRECATION_WARNING := \
 "This feature is planned for removal in the next release."\

mfem.hpp

@@ -52,4 +52,8 @@
 #include "fem/moonolith/transfer.hpp"
 #endif // MFEM_USE_MOONOLITH
 
+#ifdef NVTX_FMT_HPP
+#include NVTX_FMT_HPP
+#endif
+
 #endif

miniapps/hdiv-linear-solver/change_basis.hpp

@@ -80,6 +80,8 @@ public:
    // limitations
    void MultRT_2D(const Vector &x, Vector &y, Mode mode) const;
    void MultRT_3D(const Vector &x, Vector &y, Mode mode) const;
+   // suppress warning about hiding overloaded virtual function:
+   using Operator::Mult;
 };
 
 } // namespace mfem

tests/benchmarks/CMakeLists.txt

@@ -32,7 +32,7 @@ function(add_benchmark name)
     endif(MFEM_USE_CUDA)
 
     add_executable(bench_${name} ${${NAME}_BENCH_SRCS})
-    target_link_libraries(bench_${name} mfem pthread)
+    target_link_libraries(bench_${name} PRIVATE mfem pthread)
     add_dependencies(${MFEM_ALL_BENCHMARKS_TARGET_NAME} bench_${name})
 
     add_test(NAME bench_${name}_cpu
@@ -51,6 +51,7 @@ endfunction(add_benchmark)
 #-------------------------------------------------------------------------------
 add_benchmark(assembly_levels)
 add_benchmark(ceed)
+add_benchmark(dfem)
 add_benchmark(dg_amr)
 add_benchmark(elasticity)
 add_benchmark(tmop)

tests/benchmarks/bench_ceed.cpp

@@ -217,7 +217,7 @@ struct BP : public BakeOff<BFI, VDIM, GLL>
       cg.SetRelTol(0.0);
       cg.SetMaxIter(max_it);
       cg.SetPrintLevel(print_lvl);
-
+      cg.iterative_mode = false;
       benchmark();
       mdofs = 0.0;
    }

tests/benchmarks/bench_dfem.cpp

@@ -0,0 +1,703 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#define NVTX_COLOR ::nvtx::kNvidia
+
+#include "bench.hpp" // IWYU pragma: keep 
+
+#ifdef MFEM_USE_BENCHMARK
+
+#include <memory>
+
+#include "fem/qinterp/det.hpp" // IWYU pragma: keep
+#include "fem/qinterp/grad.hpp" // IWYU pragma: keep
+#include "fem/qinterp/grad_transpose.hpp" // IWYU pragma: keep
+#include "fem/quadinterpolator.hpp" // IWYU pragma: keep
+#include "fem/integ/lininteg_domain_kernels.hpp" // IWYU pragma: keep
+#include "fem/integ/bilininteg_vecdiffusion_pa.hpp" // IWYU pragma: keep 
+
+#include <fem/dfem/backends/devices.hpp>
+using device_backend = mfem::future::DeviceBackend;
+
+#define DFEM_USE_DEFAULT_BACKEND
+#ifdef DFEM_USE_DEFAULT_BACKEND
+#include <fem/dfem/backends/global_qf/prelude.hpp>
+using default_backend = mfem::future::GlobalQFBackend;
+#else
+using default_backend = mfem::future::DeviceBackend;
+#endif
+
+#include <fem/dfem/doperator.hpp>
+#include <linalg/tensor.hpp>
+#include "linalg/tensor_arrays.hpp"
+
+#include "fem/kernels.hpp"
+namespace ker = kernels::internal;
+
+#if defined(__HIP__)
+#include "../usr/src/array/tensor_std_array.hpp"
+#endif
+
+// #include NVTX_FMT_HPP
+
+using namespace mfem;
+
+// using mfem::future::tuple;
+using mfem::future::tensor;
+using mfem::future::tensor_array;
+
+using future::DifferentiableOperator;
+using future::UniformParameterSpace;
+using future::ParameterFunction;
+using future::FieldDescriptor;
+using future::Gradient;
+using future::Value;
+using future::Weight;
+using future::Identity;
+
+/// info //////////////////////////////////////////////////////////////////////
+void info()
+{
+   mfem::out << "\x1b[33m";
+   mfem::out << "version 0: PA std" << std::endl;
+   mfem::out << "version 1: PA new" << std::endl;
+   mfem::out << "version 2: MF ∂fem-global" << std::endl;
+   mfem::out << "version 3: PA ∂fem-global" << std::endl;
+   mfem::out << "version 4: MF ∂fem-global 'devices' backend" << std::endl;
+   mfem::out << "version 5: PA ∂fem-global 'devices' backend" << std::endl;
+   mfem::out << "\x1b[m" << std::endl;
+}
+
+// Custom benchmark arguments generator ///////////////////////////////////////
+static void CustomArguments(bm::Benchmark *b) noexcept
+{
+   constexpr int MAX_NDOFS = 8 * 1024 * (mfem_use_gpu ? 1024 : 8);
+
+   const auto versions = { 0, 1, 2, 3, 4, 5 };
+
+   const auto orders = { 6, 5, 4, 3, 2, 1 };
+
+   constexpr auto ndofs = [](int n) constexpr noexcept -> int
+   {
+      return (n + 1) * (n + 1) * (n + 1);
+   };
+
+   constexpr auto inc = [](int n) constexpr noexcept -> int
+   {
+      return n < 160 ?  4 : n < 240 ?  8 : n < 320 ? 16 : 32;
+   };
+
+   for (auto k : versions)
+   {
+      for (auto p : orders)
+      {
+         for (int n = 4; ndofs(n) <= MAX_NDOFS; n += inc(n))
+         {
+            b->Args({k, p, n});
+         }
+      }
+   }
+}
+
+// Register kernel specializations used in the benchmarks /////////////////////
+static void AddKernelSpecializations()
+{
+   QuadratureInterpolator::DetKernels::Add<3, 3, 2, 2>();
+   QuadratureInterpolator::DetKernels::Add<3, 3, 2, 3>();
+   QuadratureInterpolator::DetKernels::Add<3, 3, 2, 5>();
+   QuadratureInterpolator::DetKernels::Add<3, 3, 2, 6>();
+   QuadratureInterpolator::DetKernels::Add<3, 3, 5, 5>();
+   // Others use too much shared data
+   // uadratureInterpolator::DetKernels::Add<3, 3, 2, 7>();
+
+   using GRAD = QuadratureInterpolator::GradKernels;
+   GRAD::Add<3, QVectorLayout::byNODES, false, 3, 2, 2>();
+   GRAD::Add<3, QVectorLayout::byNODES, false, 3, 2, 7>();
+   GRAD::Add<3, QVectorLayout::byNODES, false, 3, 2, 8>();
+   GRAD::Add<3, QVectorLayout::byNODES, false, 3, 2, 9>();
+
+   GRAD::Add<3, QVectorLayout::byVDIM, false, 3, 2, 3>();
+   GRAD::Add<3, QVectorLayout::byVDIM, false, 3, 2, 4>();
+   GRAD::Add<3, QVectorLayout::byVDIM, false, 3, 2, 5>();
+   GRAD::Add<3, QVectorLayout::byVDIM, false, 3, 2, 6>();
+   GRAD::Add<3, QVectorLayout::byVDIM, false, 3, 2, 7>();
+   GRAD::Add<3, QVectorLayout::byVDIM, false, 3, 2, 8>();
+
+   GRAD::Add<3, QVectorLayout::byVDIM, false, 1, 2, 3>();
+   GRAD::Add<3, QVectorLayout::byVDIM, false, 1, 4, 5>();
+   GRAD::Add<3, QVectorLayout::byVDIM, false, 1, 5, 6>();
+   GRAD::Add<3, QVectorLayout::byVDIM, false, 1, 6, 7>();
+   GRAD::Add<3, QVectorLayout::byVDIM, false, 1, 7, 8>();
+
+   using GRAD_TRANSPOSE = QuadratureInterpolator::GradTransposeKernels;
+   GRAD_TRANSPOSE::Add<3, QVectorLayout::byVDIM, false, 1,2,3>();
+   GRAD_TRANSPOSE::Add<3, QVectorLayout::byVDIM, false, 1,4,5>();
+   GRAD_TRANSPOSE::Add<3, QVectorLayout::byVDIM, false, 1,5,6>();
+   GRAD_TRANSPOSE::Add<3, QVectorLayout::byVDIM, false, 1,6,7>();
+   GRAD_TRANSPOSE::Add<3, QVectorLayout::byVDIM, false, 1,7,8>();
+
+   using LIN = DomainLFIntegrator::AssembleKernels;
+   LIN::Add<3, 7, 7>();
+   LIN::Add<3, 6, 6>();
+   LIN::Add<3, 8, 8>();
+
+   using VDIFF = VectorDiffusionIntegrator::ApplyPAKernels;
+   VDIFF::Add<3, 3, 3, 3>();
+   VDIFF::Add<3, 3, 4, 4>();
+   VDIFF::Add<3, 3, 5, 5>();
+   VDIFF::Add<3, 3, 6, 6>();
+   VDIFF::Add<3, 3, 7, 7>();
+   VDIFF::Add<3, 3, 8, 8>();
+}
+
+/// Globals ///////////////////////////////////////////////////////////////////
+Device *device_ptr = nullptr;
+static int gD1D = 0, gQ1D = 0;
+
+/// StiffnessIntegrator ///////////////////////////////////////////////////////
+struct StiffnessIntegrator : public BilinearFormIntegrator
+{
+   const FiniteElementSpace *fes;
+   const real_t *B, *G, *DX;
+   int ne, d1d, q1d;
+   Vector J0, dx;
+
+public:
+   StiffnessIntegrator()
+   {
+      dbg();
+      NVTX();
+      StiffnessKernels::Add<2, 3>();
+      StiffnessKernels::Add<3, 4>();
+      StiffnessKernels::Add<4, 5>();
+      StiffnessKernels::Add<5, 6>();
+      StiffnessKernels::Add<6, 7>();
+      StiffnessKernels::Add<7, 8>();
+      StiffnessKernels::Add<9, 10>();
+   }
+
+   void AssemblePA(const FiniteElementSpace &fespace) override
+   {
+      dbg();
+      NVTX();
+      fes = &fespace;
+      auto *mesh = fes->GetMesh();
+      const int DIM = mesh->Dimension();
+      ne = mesh->GetNE();
+      const auto p = fes->GetFE(0)->GetOrder();
+      const auto q = 2 * p + mesh->GetElementTransformation(0)->OrderW();
+      const auto type = mesh->GetElementBaseGeometry(0);
+      const IntegrationRule &ir = IntRules.Get(type, q);
+      const int NQPT = ir.GetNPoints();
+      d1d = p + 1;
+      q1d = IntRules.Get(Geometry::SEGMENT, ir.GetOrder()).GetNPoints();
+      MFEM_VERIFY(d1d == gD1D, "D1D mismatch: " << d1d << " != " << gD1D);
+      MFEM_VERIFY(q1d == gQ1D, "Q1D mismatch: " << q1d << " != " << gQ1D);
+      MFEM_VERIFY(NQPT == q1d * q1d * q1d, "");
+      const DofToQuad *maps =
+         &fes->GetFE(0)->GetDofToQuad(ir, DofToQuad::TENSOR);
+      const GridFunction *nodes = (mesh->EnsureNodes(), mesh->GetNodes());
+      const FiniteElementSpace *nfes = nodes->FESpace();
+      const int nVDIM = nfes->GetVDim();
+      dx.SetSize(nVDIM * DIM * NQPT * ne, Device::GetDeviceMemoryType());
+      J0.SetSize(nVDIM * DIM * NQPT * ne, Device::GetDeviceMemoryType());
+      dx.UseDevice(true), J0.UseDevice(true);
+      B = maps->B.Read(), G = maps->G.Read(), DX = dx.Read();
+
+      const Operator *NR =
+         nfes->GetElementRestriction(ElementDofOrdering::LEXICOGRAPHIC);
+      const QuadratureInterpolator *nqi = nfes->GetQuadratureInterpolator(ir);
+      nqi->SetOutputLayout(QVectorLayout::byVDIM);
+      const int nd = nfes->GetFE(0)->GetDof();
+      Vector xe(nVDIM * nd * ne, Device::GetDeviceMemoryType());
+      NR->Mult(*nodes, (xe.UseDevice(true), xe));
+      nqi->Derivatives(xe, J0);
+
+      const int Q1D = q1d;
+      const auto w_r = ir.GetWeights().Read();
+      const auto W = Reshape(w_r, q1d, q1d, q1d);
+      const auto J = Reshape(J0.Read(), 3, 3, q1d, q1d, q1d, ne);
+      auto DX_w = Reshape(dx.Write(), 3, 3, q1d, q1d, q1d, ne);
+
+      mfem::forall_3D(ne, Q1D, Q1D, Q1D,[=] MFEM_HOST_DEVICE(int e)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(qz, z, Q1D)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+               {
+                  const real_t w = W(qx, qy, qz);
+                  const real_t *Jtr = &J(0, 0, qx, qy, qz, e);
+                  const real_t detJ = kernels::Det<3>(Jtr);
+                  const real_t wd = w * detJ;
+                  const real_t D[9] = { wd, 0.0, 0.0,
+                                        0.0, wd, 0.0,
+                                        0.0, 0.0, wd
+                                      };
+                  real_t Jrt[9], A[9];
+                  kernels::CalcInverse<3>(Jtr, Jrt);
+                  kernels::MultABt(3, 3, 3, D, Jrt, A);
+                  kernels::Mult(3, 3, 3, A, Jrt, &DX_w(0, 0, qx, qy, qz, e));
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+      });
+   }
+
+   template <int T_D1D, int T_Q1D>
+   static void StiffnessMult(const int NE, const real_t *b, const real_t *g,
+                             const real_t *dx, const real_t *xe, real_t *ye,
+                             const int d1d, const int q1d)
+   {
+      // NVTX();
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+      constexpr int DIM = 3, VDIM = 1;
+      const auto XE = Reshape(xe, D1D, D1D, D1D, VDIM, NE);
+      const auto DX = Reshape(dx, 3, 3, Q1D, Q1D, Q1D, NE);
+      auto YE = Reshape(ye, D1D, D1D, D1D, VDIM, NE);
+
+      mfem::forall_2D<T_Q1D*T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
+      {
+         constexpr int MD1 = T_D1D > 0 ? kernels::internal::SetMaxOf(T_D1D) : 32;
+         constexpr int MQ1 = T_Q1D > 0 ? kernels::internal::SetMaxOf(T_Q1D) : 32;
+
+         MFEM_SHARED real_t smem[MQ1][MQ1];
+         MFEM_SHARED real_t sB[MD1][MQ1], sG[MD1][MQ1];
+         ker::vd_regs3d_t<VDIM, DIM, MQ1> r0, r1;
+
+         ker::LoadMatrix(D1D, Q1D, b, sB);
+         ker::LoadMatrix(D1D, Q1D, g, sG);
+
+         ker::LoadDofs3d(e, D1D, XE, r0);
+         ker::Grad3d(D1D, Q1D, smem, sB, sG, r0, r1);
+
+         for (int qz = 0; qz < Q1D; qz++)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+               {
+                  real_t v[3], u[3] = { r1[0][0][qz][qy][qx],
+                                        r1[0][1][qz][qy][qx],
+                                        r1[0][2][qz][qy][qx]
+                                      };
+                  const real_t *dx = &DX(0, 0, qx, qy, qz, e);
+                  kernels::Mult(3, 3, dx, u, v);
+                  r0[0][0][qz][qy][qx] = v[0];
+                  r0[0][1][qz][qy][qx] = v[1];
+                  r0[0][2][qz][qy][qx] = v[2];
+               }
+            }
+         }
+         ker::GradTranspose3d(D1D, Q1D, smem, sB, sG, r0, r1);
+         ker::WriteDofs3d(e, D1D, r1, YE);
+      });
+   }
+
+   using StiffnessKernelType = decltype(&StiffnessMult<0, 0>);
+   MFEM_REGISTER_KERNELS(StiffnessKernels, StiffnessKernelType, (int, int));
+
+   void AddMultPA(const Vector &x, Vector &y) const override
+   {
+      StiffnessKernels::Run(d1d, q1d, ne, B, G, DX, x.Read(), y.ReadWrite(),
+                            d1d, q1d);
+   }
+};
+
+template <int D1D, int Q1D>
+StiffnessIntegrator::StiffnessKernelType
+StiffnessIntegrator::StiffnessKernels::Kernel()
+{
+   return StiffnessMult<D1D, Q1D>;
+}
+
+StiffnessIntegrator::StiffnessKernelType
+StiffnessIntegrator::StiffnessKernels::Fallback([[maybe_unused]] int d1d,
+                                                [[maybe_unused]] int q1d)
+{
+   dbg("\x1b[33mFallback d1d:{} q1d:{}", d1d, q1d);
+   MFEM_ABORT("No kernel for d1d=" << d1d << " q1d=" << q1d);
+   return nullptr;
+   // return StiffnessMult;
+}
+
+/// BakeOff ///////////////////////////////////////////////////////////////////
+template <int VDIM, bool GLL>
+struct BakeOff
+{
+   static constexpr int DIM = 3;
+   const int p, c, q, n, nx, ny, nz;
+   const bool check_x, check_y, check_z, checked;
+   Mesh smesh;
+   ParMesh pmesh;
+   H1_FECollection fec;
+   ParFiniteElementSpace pfes;
+   const Geometry::Type geom_type;
+   IntegrationRules irs;
+   const IntegrationRule *ir;
+   ConstantCoefficient one;
+   Vector uvec;
+   VectorConstantCoefficient unit_vec;
+   const int dofs;
+   ParGridFunction &nodes;
+   ParFiniteElementSpace& mfes;
+   ParGridFunction x, y;
+   ParBilinearForm a;
+   std::unique_ptr<DifferentiableOperator> dop;
+   const int elem_size, total_size, d1d, q1d;
+   QuadratureSpace qspace;
+   QuadratureFunction qdata;
+
+   double mdofs{};
+
+   BakeOff(int p, int side):
+      p(p), c(side), q(2 * p + (GLL ? -1 : 3)), n((assert(c >= p), c / p)),
+      nx(n + (p * (n + 1) * p * n * p * n < c * c * c ? 1 : 0)),
+      ny(n + (p * (n + 1) * p * (n + 1) * p * n < c * c * c ? 1 : 0)), nz(n),
+      check_x(p * nx * p * ny * p * nz <= c * c * c),
+      check_y(p * (nx + 1) * p * (ny + 1) * p * nz > c * c * c),
+      check_z(p * (nx + 1) * p * (ny + 1) * p * (nz + 1) > c * c * c),
+      checked((assert(check_x &&check_y && check_z), true)),
+      smesh(Mesh::MakeCartesian3D(nx, ny, nz, Element::HEXAHEDRON)),
+      pmesh(MPI_COMM_WORLD, (smesh.EnsureNodes(), smesh)),
+      fec(p, DIM, BasisType::GaussLobatto),
+      pfes(&pmesh, &fec, VDIM),
+      geom_type(pmesh.GetTypicalElementGeometry()),
+      irs(0, GLL ? Quadrature1D::GaussLobatto : Quadrature1D::GaussLegendre),
+      ir(&irs.Get(geom_type, q)), one(1.0), uvec(DIM),
+      unit_vec((uvec = 1.0, uvec /= uvec.Norml2(), uvec)),
+      dofs(pfes.GetTrueVSize()),
+      nodes(*static_cast<ParGridFunction*>(pmesh.GetNodes())),
+      mfes(*(nodes.ParFESpace())),
+      x(&pfes),
+      y(&pfes),
+      a(&pfes),
+      elem_size(DIM * DIM * ir->GetNPoints()),
+      total_size(elem_size * pmesh.GetNE()),
+      d1d(p + 1),
+      q1d(IntRules.Get(Geometry::SEGMENT, ir->GetOrder()).GetNPoints()),
+      qspace(pmesh, *ir),
+      qdata(qspace, DIM*DIM)
+   {
+      NVTX_MARK_FUNCTION;
+      // pmesh.SetCurvature(p);
+      smesh.Clear();
+      x = 0.0;
+
+      gD1D = d1d, gQ1D = q1d;
+      // dbg("D1D: {}, Q1D: {}", gD1D, gQ1D);
+      assert(q1d*q1d*q1d == ir->GetNPoints());
+   }
+
+   virtual void Benchmark() { MFEM_ABORT("Not implemented."); }
+
+   [[nodiscard]] double SumMdofs() const noexcept { return mdofs; }
+
+   [[nodiscard]] double MDofs() const noexcept { return 1e-6 * dofs; }
+};
+
+/// Q-Functions ///////////////////////////////////////////////////////////////
+template<int DIM>
+struct MFApply
+{
+   void operator()(tensor_array<const real_t, DIM> &Gu,
+                   tensor_array<const real_t, DIM, DIM> &J,
+                   tensor_array<const real_t> &weight,
+                   tensor_array<real_t, DIM> &Gv) const
+   {
+      NVTX_MARK_FUNCTION;
+      mfem::forall(J.size(), [=] MFEM_HOST_DEVICE (int q)
+      {
+         const auto invJ = inv(J(q));
+         const real_t detJ = det(J(q));
+         Gv(q) = ((Gu(q) * invJ)) * transpose(invJ) * detJ * weight(q);
+      });
+   }
+};
+
+template<int DIM>
+struct PASetup
+{
+   void operator()(tensor_array<const real_t, DIM, DIM> &J,
+                   tensor_array<const real_t> &weight,
+                   tensor_array<real_t, DIM, DIM> &D) const
+   {
+      NVTX_MARK_FUNCTION;
+      mfem::forall(J.size(), [=] MFEM_HOST_DEVICE (int q)
+      {
+         const auto invJ = inv(J(q));
+         const real_t detJ = det(J(q));
+         D(q) = invJ * transpose(invJ) * detJ * weight(q);
+      });
+   }
+};
+
+template<int DIM>
+struct PAApply
+{
+   void operator()(tensor_array<const real_t, DIM> &Gu,
+                   tensor_array<const real_t, DIM, DIM> &D,
+                   tensor_array<const real_t> &/*weight*/,
+                   tensor_array<real_t, DIM> &Gv) const
+   {
+      NVTX_MARK_FUNCTION;
+      mfem::forall(Gu.size(), [=] MFEM_HOST_DEVICE (int q) { Gv(q) = D(q) * Gu(q); });
+   }
+};
+
+/// Diffusion /////////////////////////////////////////////////////////////////
+template <int VDIM, bool GLL>
+struct Diffusion : public BakeOff<VDIM, GLL>
+{
+   static constexpr int DIM = 3;
+   static constexpr int U = 0, Ξ = 1, Q = 2;
+
+   const real_t rtol = 0.0;
+   const int max_it = 32, print_lvl = -1;
+
+   const int version;
+   Array<int> ess_tdof_list, ess_bdr, all_domain_attr;
+   ParLinearForm b;
+   OperatorPtr A;
+   Operator *A_ptr;
+   Vector B, X;
+   CGSolver cg;
+   struct WrapOpArg1: public Operator
+   {
+      const std::unique_ptr<DifferentiableOperator> &dop;
+      Vector &arg1;
+
+      WrapOpArg1(const std::unique_ptr<DifferentiableOperator> &dop,
+                 const int height, const int width, Vector &arg1):
+         Operator(height, width), dop(dop), arg1(arg1) { }
+
+      void Mult(const Vector &x, Vector &y) const override
+      {
+         MultiVector M{const_cast<Vector&>(x), arg1}, Y{y};
+         dop->Mult(M, Y);
+      }
+   };
+   std::unique_ptr<WrapOpArg1> wop;
+
+   using BakeOff<VDIM, GLL>::a;
+   using BakeOff<VDIM, GLL>::ir;
+   using BakeOff<VDIM, GLL>::one;
+   using BakeOff<VDIM, GLL>::pmesh;
+   using BakeOff<VDIM, GLL>::pfes;
+   using BakeOff<VDIM, GLL>::mfes;
+   using BakeOff<VDIM, GLL>::x;
+   using BakeOff<VDIM, GLL>::y;
+   using BakeOff<VDIM, GLL>::mdofs;
+   using BakeOff<VDIM, GLL>::dop;
+   using BakeOff<VDIM, GLL>::nodes;
+   using BakeOff<VDIM, GLL>::qdata;
+   using BakeOff<VDIM, GLL>::dofs;
+
+   Diffusion(int version, int order, int side):
+      BakeOff<VDIM, GLL>(order, side),
+      version(version),
+      ess_bdr(pmesh.bdr_attributes.Max()),
+      all_domain_attr(pmesh.bdr_attributes.Max()),
+      b(&pfes),
+      B(pfes.GetVSize()),
+      X(x),
+      cg(MPI_COMM_WORLD)
+   {
+      static_assert(VDIM == 1 && GLL == false);
+
+      ess_bdr = 1;
+      all_domain_attr = 1;
+      pfes.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+
+      b.AddDomainIntegrator(new DomainLFIntegrator(this->one));
+      b.UseFastAssembly(true);
+      b.Assemble();
+
+      // MF setup ///////////////////////////////////////////////////
+      const auto dMFOperatorSetup = [&] (auto backend)
+      {
+         using backend_t = decltype(backend);
+         const auto ifs = std::vector<FieldDescriptor> {{U, &pfes}, {Ξ, &mfes}};
+         const auto ofs = std::vector<FieldDescriptor> {{U, &pfes}};
+         const int height = pfes.GetVSize(), width = pfes.GetVSize();
+         dop = std::make_unique<DifferentiableOperator>(height, width, ifs, ofs, pmesh);
+         MFApply<DIM> mf_apply_qf;
+         dop->template AddDomainIntegrator<backend_t>(mf_apply_qf,
+                                                      std::tuple{Gradient<U>{}, Gradient<Ξ>{}, Weight{}},
+                                                      std::tuple{Gradient<U>{}},
+                                                      *ir, ess_bdr);
+         dop->SetMultLevel(DifferentiableOperator::MultLevel::LVECTOR);
+         wop = std::make_unique<WrapOpArg1>(dop, height, width, nodes);
+         wop->FormLinearSystem(ess_tdof_list, x, b, A_ptr, X, B);
+         A.Reset(A_ptr);
+      };
+
+      // PA setup ///////////////////////////////////////////////////
+      const auto dPAOperatorSetup = [&] (auto backend)
+      {
+         using backend_t = decltype(backend);
+         const int height = pfes.GetVSize(), width = pfes.GetVSize();
+         dbg("height: {} width: {}", height, width);
+         dbg("\x1b[33m PA Setup operator");
+         const auto i0 = std::vector<FieldDescriptor> { {Ξ, &mfes}};
+         const auto o0 = std::vector<FieldDescriptor> { {Q, &qdata}};
+         DifferentiableOperator dSetup(height, width, i0, o0, pmesh);
+         PASetup<DIM> pa_setup_qf;
+         dSetup.AddDomainIntegrator<backend_t>(pa_setup_qf,
+                                               std::tuple{Gradient<Ξ>{}, Weight{}},
+                                               std::tuple{Identity<Q>{}},
+                                               *ir, ess_bdr);
+         dSetup.SetMultLevel(DifferentiableOperator::MultLevel::LVECTOR);
+         MultiVector N{nodes}, D{qdata};
+         dSetup.Mult(N, D);
+         dbg("\x1b[33m PA Apply operator");
+         const auto i1 = std::vector<FieldDescriptor> { {U, &pfes}, {Q, &qdata}};
+         const auto o1 = std::vector<FieldDescriptor> { {U, &pfes}};
+         dop = std::make_unique<DifferentiableOperator>(height, width, i1, o1, pmesh);
+         PAApply<DIM> pa_apply_qf;
+         dop->template AddDomainIntegrator<backend_t>(pa_apply_qf,
+                                                      std::tuple{Gradient<U>{}, Identity<Q>{}, Weight{}},
+                                                      std::tuple{Gradient<U>{}},
+                                                      *ir, ess_bdr);
+         dop->SetMultLevel(DifferentiableOperator::MultLevel::LVECTOR);
+         wop = std::make_unique<WrapOpArg1>(dop, height, width, qdata);
+         wop->FormLinearSystem(ess_tdof_list, x, b, A_ptr, X, B);
+         A.Reset(A_ptr);
+      };
+
+      if (version < 2) // standard, new PA regs
+      {
+         a.SetAssemblyLevel(AssemblyLevel::PARTIAL);
+         if (version == 0) { a.AddDomainIntegrator(new DiffusionIntegrator(ir)); }
+         if (version == 1) { a.AddDomainIntegrator(new StiffnessIntegrator()); }
+         a.Assemble();
+         a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+         if (version == 0)
+         {
+            BilinearFormIntegrator *bfi = a.GetDBFI()->operator[](0);
+            auto *di = dynamic_cast<DiffusionIntegrator*>(bfi);
+            assert(di);
+            const int d1d = di->dofs1D, q1d = di->quad1D;
+            MFEM_VERIFY(d1d == gD1D, "D1D mismatch: " << d1d << " != " << gD1D);
+            MFEM_VERIFY(q1d == gQ1D, "Q1D mismatch: " << q1d << " != " << gQ1D);
+         }
+      }
+      else if (version == 2) // 2: MF ∂FEM ////////////////////////////////////
+      {
+         dbg("\x1b[33m MF ∂FEM");
+         dMFOperatorSetup(default_backend{});
+      }
+      else if (version == 3) // PA ∂FEM ///////////////////////////////////////
+      {
+         dbg("\x1b[33m PA ∂FEM + default backend");
+         dPAOperatorSetup(default_backend{});
+      }
+      else if (version == 4) // MF ∂FEM 'devices' backend /////////////////////
+      {
+         dbg("\x1b[33m MF ∂FEM + devices backend");
+         dMFOperatorSetup(device_backend{});
+      }
+      else if (version == 5) // PA ∂FEM 'devices' backend /////////////////////
+      {
+         dbg("\x1b[33m PA ∂FEM + devices backend");
+         dPAOperatorSetup(device_backend{});
+      }
+      else { MFEM_ABORT("Invalid version"); }
+
+      cg.SetOperator(*A);
+      cg.iterative_mode = false;
+      cg.SetAbsTol(0.0);
+      if (dofs < 128 * 1024)
+      {
+         dbg("check");
+         cg.SetPrintLevel(3/*-1*/);
+         cg.SetMaxIter(2000);
+         cg.SetRelTol(1e-8);
+         cg.Mult(B, X);
+         MFEM_VERIFY(cg.GetConverged(), "❌ CG solver did not converge.");
+         mfem::out << "✅" << std::endl;
+      }
+      cg.SetRelTol(rtol);
+      cg.SetMaxIter(max_it);
+      cg.SetPrintLevel(print_lvl);
+      Benchmark();
+      mdofs = 0.0;
+   }
+
+   void Benchmark() override
+   {
+      cg.Mult(B, X);
+      MFEM_DEVICE_SYNC;
+      mdofs += this->MDofs() * cg.GetNumIterations();
+   }
+};
+
+///////////////////////////////////////////////////////////////////////////////
+#define BakeOff_Problem(i)                                           \
+   static void BP##i(bm::State &state)                               \
+   {                                                                 \
+      const auto version = static_cast<int>(state.range(0));         \
+      const auto order = static_cast<int>(state.range(1));           \
+      const auto side = static_cast<int>(state.range(2));            \
+      Diffusion<1,false> ker(version, order, side);                  \
+      while (state.KeepRunning()) { ker.Benchmark(); }               \
+      bm::Counter::Flags flags = bm::Counter::kIsRate;               \
+      state.counters["MDof/s"] = bm::Counter(ker.SumMdofs(), flags); \
+      state.counters["Dofs"] = bm::Counter(ker.dofs);                \
+      state.counters["p"] = bm::Counter(order);                      \
+      state.counters["version"] = bm::Counter(version);              \
+   }                                                                 \
+   BENCHMARK(BP##i)                                                  \
+      ->Apply(CustomArguments)                                       \
+      ->Unit(bm::kMillisecond)
+
+BakeOff_Problem(3);
+
+/// main //////////////////////////////////////////////////////////////////////
+int main(int argc, char *argv[])
+{
+   dbg();
+   static mfem::MPI_Session mpi(argc, argv);
+
+   bm::ConsoleReporter CR;
+   bm::Initialize(&argc, argv);
+
+   AddKernelSpecializations();
+   info();
+
+   // Device setup, cpu by default
+   std::string device_config = "cpu";
+   const auto global_context = bmi::GetGlobalContext();
+   if (global_context != nullptr)
+   {
+      const auto device = global_context->find("device");
+      if (device != global_context->end())
+      {
+         mfem::out << device->first << " : " << device->second << std::endl;
+         device_config = device->second;
+      }
+   }
+   dbg("device_config: {}", device_config);
+   Device device(device_config.c_str());
+   device_ptr = &device;
+   device.Print();
+
+   if (bm::ReportUnrecognizedArguments(argc, argv)) { return EXIT_FAILURE; }
+
+   bm::RunSpecifiedBenchmarks((bm::BenchmarkReporter*)&CR);
+
+   return EXIT_SUCCESS;
+}
+
+#endif // MFEM_USE_BENCHMARK

tests/benchmarks/makefile

@@ -20,8 +20,8 @@ CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\
 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)
 
-SEQ_TESTS = bench_assembly_levels bench_ceed bench_dg_amr bench_elasticity \
-            bench_tmop bench_vector bench_virtuals
+SEQ_TESTS = bench_assembly_levels bench_ceed bench_dfem bench_dg_amr \
+			bench_elasticity bench_tmop bench_vector bench_virtuals
 PAR_TESTS = 
 ifeq ($(MFEM_USE_MPI),NO)
    TESTS = $(SEQ_TESTS)

tests/unit/CMakeLists.txt

@@ -25,8 +25,8 @@ add_custom_target(copy_test_data
 # Add 'copy_test_data' as a prerequisite for test executables, if the source and the
 # build directories are not the same.
 if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
-  add_dependencies(${MFEM_TEST_EXEC_PREREQUISITES_TARGET_NAME} copy_test_data)
-  add_dependencies(${MFEM_TEST_EXEC_PREREQUISITES_TARGET_NAME} copy_data)
+    add_dependencies(${MFEM_TEST_EXEC_PREREQUISITES_TARGET_NAME} copy_test_data)
+    add_dependencies(${MFEM_TEST_EXEC_PREREQUISITES_TARGET_NAME} copy_data)
 endif()
 
 # Include the source directory for the unit tests - catch.hpp is there.
@@ -35,10 +35,11 @@ include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR})
 # The following list can be updated using (in bash):
 #    for d in dfem general linalg mesh fem enzyme; do ls -1 $d/*.cpp; done
 set(UNIT_TESTS_SRCS
-  dfem/test_diffusion.cpp
-  dfem/test_divergence.cpp
-  dfem/test_lvector_interface.cpp
-  dfem/test_mass.cpp
+  # dfem/test_diffusion.cpp
+  # dfem/test_divergence.cpp
+  # dfem/test_lvector_interface.cpp
+  # dfem/test_mass.cpp
+  dfem/test_multiple_outputs.cpp
   general/test_array.cpp
   general/test_scan.cpp
   general/test_arrays_by_name.cpp
@@ -256,7 +257,7 @@ function(add_serial_miniapp_test name test_uvm)
         set_property(SOURCE ${${NAME}_TESTS_SRCS} PROPERTY LANGUAGE CUDA)
     endif(MFEM_USE_CUDA)
     if (MFEM_USE_HIP)
-       set_property(SOURCE ${${NAME}_TESTS_SRCS} PROPERTY HIP_SOURCE_PROPERTY_FORMAT TRUE)
+        set_property(SOURCE ${${NAME}_TESTS_SRCS} PROPERTY HIP_SOURCE_PROPERTY_FORMAT TRUE)
     endif(MFEM_USE_HIP)
 
     mfem_add_executable(${name}_tests_cpu ${${NAME}_TESTS_SRCS})

tests/unit/dfem/test_multiple_outputs.cpp

@@ -0,0 +1,410 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../unit_tests.hpp"
+#include "mfem.hpp"
+#include "../fem/dfem/doperator.hpp"
+#include "../fem/dfem/backends/local_qf/prelude.hpp"
+#include "linalg/tensor_arrays.hpp"
+
+#ifdef NVTX_DEBUG_HPP
+#undef NVTX_COLOR
+#define NVTX_COLOR ::nvtx::kCyan
+#include NVTX_DEBUG_HPP
+#else
+#define dbg(...)
+#endif
+#include <proteus/JitInterface.h>
+
+#ifdef MFEM_USE_MPI
+
+using namespace mfem;
+using namespace mfem::future;
+using mfem::future::tensor;
+
+#ifdef MFEM_USE_ENZYME
+using dscalar_t = real_t;
+#else
+using mfem::future::dual;
+using dscalar_t = dual<real_t, real_t>;
+#endif
+
+constexpr int DIM = 2;
+
+class DummyParameterSpace : public ParameterSpace
+{
+public:
+   class Bimpl : public Operator
+   {
+      virtual void Mult(const Vector &x, Vector &y) const
+      {
+         for (int i = 0; i < y.Size(); i++)
+         {
+            y(i) = x(0);
+         }
+      }
+   };
+
+   class Btimpl : public Operator
+   {
+      virtual void Mult(const Vector &x, Vector &y) const
+      {
+         y(0) = x(0);
+      }
+   };
+
+   DummyParameterSpace() : ParameterSpace(1) {}
+
+   virtual int GetTrueVSize() const override
+   {
+      return 1;
+   }
+
+   virtual int GetVSize() const override
+   {
+      return 1;
+   }
+
+   virtual const Operator* GetB() const override
+   {
+      if (!B)
+      {
+         B.reset(new Bimpl());
+      }
+      return B.get();
+   }
+
+   virtual const Operator* GetBt() const override
+   {
+      if (!Bt)
+      {
+         Bt.reset(new Btimpl());
+      }
+      return Bt.get();
+   }
+};
+
+struct massqf
+{
+   inline MFEM_HOST_DEVICE
+   void operator()(
+      tensor_array<const real_t> &u,
+      tensor_array<const real_t, DIM, DIM> &J,
+      tensor_array<const real_t> &w,
+      tensor_array<real_t> &out1,
+      tensor_array<real_t> &out2) const
+   {
+      for (size_t q = 0; q < u.size(); q++)
+      {
+         const auto v = u(q) * det(J(q)) * w(q);
+         out1(q) = v;
+         out2(q) = v;
+      }
+   }
+};
+
+struct mass_diffusion_qdata_qf
+{
+   inline MFEM_HOST_DEVICE
+   void operator()(
+      tensor_array<const real_t> &u,
+      tensor_array<const real_t, DIM> &dudxi,
+      tensor_array<const real_t, DIM, DIM> &J,
+      tensor_array<const real_t, DIM, DIM> &qdata,
+      tensor_array<const real_t> &w,
+      tensor_array<const real_t> &dummy_parameter,
+      tensor_array<real_t> &out1,
+      tensor_array<real_t, DIM> &out2,
+      tensor_array<real_t, DIM, DIM> &out3) const
+   {
+      for (size_t q = 0; q < u.size(); q++)
+      {
+         const auto invJq = inv(J(q));
+         const auto detJq = det(J(q));
+
+         out1(q) = u(q) * detJq * w(q);
+         // out2(q) = (dudxi(q) * invJq) * transpose(invJq) * (detJq * w(q));
+         out3(q) = J(q);
+      }
+
+      jit_bounds(dudxi, J, w, out2, u.size());
+   }
+
+   // XXX: Attribute instrumentation does not work due to ABI differences that
+   // change the argument number.
+   //__attribute__((annotate("jit", 5)))
+   void jit_bounds(
+      tensor_array<const real_t, DIM> &dudxi,
+      tensor_array<const real_t, DIM, DIM> &J,
+      tensor_array<const real_t> &w,
+      tensor_array<real_t, DIM> &out,
+      size_t NQ) const
+   {
+      for (size_t q = 0; q < NQ; q++)
+      {
+         const auto invJq = inv(J(q));
+         const auto detJq = det(J(q));
+         out(q) = (dudxi(q) * invJq) * transpose(invJq) * (detJq * w(q));
+      }
+   }
+};
+
+TEST_CASE("dFEM Multiple Outputs", "[Parallel][dFEM][Outputs]")
+{
+   const bool all_tests = launch_all_non_regression_tests;
+
+   const auto p = !all_tests ? 2 : GENERATE(1, 2, 3);
+   const char *filename = "../../data/inline-quad.mesh";
+   CAPTURE(filename, DIM, p);
+
+   Mesh smesh(filename);
+   MFEM_ASSERT(smesh.Dimension() == DIM, "DIM and mesh dimension have to match");
+
+   ParMesh pmesh(MPI_COMM_WORLD, smesh);
+   pmesh.EnsureNodes();
+   auto* nodes = static_cast<ParGridFunction*>(pmesh.GetNodes());
+   smesh.Clear();
+
+   H1_FECollection fec(p, DIM);
+   ParFiniteElementSpace fes(&pmesh, &fec);
+
+   const auto *ir = &IntRules.Get(pmesh.GetTypicalElementGeometry(), 2 * p);
+
+   ParGridFunction x(&fes), y(&fes), z(&fes);
+
+   ConstantCoefficient one(1.0);
+
+   Array<int> all_domain_attr;
+   if (pmesh.attributes.Size() > 0)
+   {
+      all_domain_attr.SetSize(pmesh.attributes.Max());
+      all_domain_attr = 1;
+   }
+
+   // {
+   //    Array<int> inoffsets(3);
+   // inoffsets[0] = 0;
+   // inoffsets[1] = fes.GetTrueVSize();
+   // inoffsets[2] = nodes->ParFESpace()->GetTrueVSize();
+   // inoffsets.PartialSum();
+
+   //    BlockVector X(inoffsets);
+   //    X.GetBlock(0).Randomize(1);
+   //    X.GetBlock(1) = *nodes;
+   //    x.SetFromTrueDofs(X.GetBlock(0));
+
+   //    Array<int> outoffsets(2);
+   //    outoffsets[0] = 0;
+   //    outoffsets[1] = fes.GetTrueVSize();
+   //    outoffsets.PartialSum();
+   //    BlockVector Z(outoffsets);
+
+   //    ParBilinearForm blf(&fes);
+   //    blf.AddDomainIntegrator(new MassIntegrator(one, ir));
+   //    blf.SetAssemblyLevel(AssemblyLevel::PARTIAL);
+   //    blf.Assemble();
+   //    blf.Mult(x, y);
+   //    Vector Y(fes.GetTrueVSize());
+   //    fes.GetProlongationMatrix()->MultTranspose(y, Y);
+
+   //    static constexpr int U = 0, COORDINATES = 1, V = 2;
+   //    const std::vector<FieldDescriptor> in
+   //    {
+   //       {U, &fes},
+   //       {COORDINATES, nodes->ParFESpace()}
+   //    };
+
+   //    const std::vector<FieldDescriptor> out // test spaces?
+   //    {
+   //       {V, &fes},
+   //    };
+   //    DifferentiableOperator dop(in, out, pmesh);
+
+   //    auto derivatives = std::integer_sequence<size_t, U> {};
+   //    auto mass_qfunc = massqf{};
+   //    dop.AddDomainIntegrator(mass_qfunc,
+   //                            tuple{ Value<U>{}, Gradient<COORDINATES>{}, Weight{} },
+   //                            tuple{ Value<V>{}, Value<V>{} },
+   //                            *ir, all_domain_attr, derivatives);
+
+   //    fes.GetRestrictionMatrix()->Mult(x, X.GetBlock(0));
+   //    dop.Mult(X, Z);
+
+   //    Vector Y0(Y);
+   //    Y0 *= 2.0;
+   //    Y0 -= Z.GetBlock(0);
+
+   //    real_t norm_g, norm_l = Y0.Normlinf();
+   //    MPI_Allreduce(&norm_l, &norm_g, 1, MPI_DOUBLE, MPI_MAX, pmesh.GetComm());
+   //    REQUIRE(norm_g == MFEM_Approx(0.0));
+   //    MPI_Barrier(MPI_COMM_WORLD);
+
+   //    auto ddop = dop.GetDerivative(U, X);
+
+   //    ddop->Mult(X.GetBlock(0), Z);
+   //    Y0 = Y;
+   //    Y0 *= 2.0;
+   //    Y0 -= Z.GetBlock(0);
+
+   //    norm_l = Y0.Normlinf();
+   //    MPI_Allreduce(&norm_l, &norm_g, 1, MPI_DOUBLE, MPI_MAX, pmesh.GetComm());
+   //    REQUIRE(norm_g == MFEM_Approx(0.0));
+   //    MPI_Barrier(MPI_COMM_WORLD);
+   // }
+
+   {
+      QuadratureSpace qs(pmesh, *ir);
+      QuadratureFunction qdata(qs, DIM*DIM);
+
+      DummyParameterSpace dps;
+      ParameterFunction dpf(dps);
+      dpf = 9.12345;
+
+      auto coef_func = [](const Vector &coords)
+      {
+         return coords[0] * coords[1] * (DIM == 3 ? coords[2] : 1.0);
+      };
+      FunctionCoefficient coef(coef_func);
+      x.ProjectCoefficient(coef);
+
+      Vector xtvec, ytvec, ytvecmfem;
+      x.GetTrueDofs(xtvec);
+      ytvec.SetSize(xtvec.Size());
+      ytvecmfem.SetSize(xtvec.Size());
+
+      Vector nodestvec;
+      nodes->GetTrueDofs(nodestvec);
+
+      qdata = 123.0;
+      Vector yqdata(qdata.Size());
+
+      MultiVector X{xtvec, nodestvec, qdata, dpf};
+      MultiVector Z{ytvec, yqdata};
+
+      ParBilinearForm blf(&fes);
+      blf.AddDomainIntegrator(new MassIntegrator(ir));
+      blf.AddDomainIntegrator(new DiffusionIntegrator(ir));
+      blf.SetAssemblyLevel(AssemblyLevel::PARTIAL);
+      blf.Assemble();
+      blf.Mult(x, y);
+      fes.GetProlongationMatrix()->MultTranspose(y, ytvecmfem);
+
+      std::cout << "mfem: ";
+      pretty_print(ytvecmfem);
+
+      static constexpr int U = 0, COORDINATES = 1, V = 2, S = 3, L = 4;
+      const std::vector<FieldDescriptor> din
+      {
+         {U, &fes},
+         {COORDINATES, nodes->ParFESpace()},
+         {S, &qdata},
+         {L, &dps}
+      };
+
+      const std::vector<FieldDescriptor> dout
+      {
+         {V, &fes},
+         {S, &qdata}
+      };
+
+      <<<<<<< HEAD
+      DifferentiableOperator dop(din, dout, pmesh);
+      =======
+      {
+         DifferentiableOperator dop(in, out, pmesh);
+         >>>>>>> dfem-multiple-outputs
+
+         dop.SetQLayouts({{Value<U>{}, {1, 0}}}, {});
+
+         <<<<<<< HEAD
+         // auto derivatives = std::integer_sequence<size_t, U> {};
+         auto mass_diffusion_qfunc = mass_diffusion_qdata_qf{};
+         dop.AddDomainIntegrator(mass_diffusion_qfunc,
+                                 tuple{Value<U>{}, Gradient<U>{}, Gradient<COORDINATES>{}, Identity<S>{}, Weight{}, Value<L>{}},
+                                 tuple{Value<V>{}, Gradient<V>{}, Identity<S>{}},
+                                 *ir, all_domain_attr);//, derivatives);
+         =======
+            auto derivatives = std::integer_sequence<size_t, U> {};
+         auto mass_diffusion_qfunc = mass_diffusion_qdata_qf{};
+         dop.AddDomainIntegrator(mass_diffusion_qfunc,
+                                 tuple{Value<U>{}, Gradient<U>{}, Gradient<COORDINATES>{}, Identity<S>{}, Weight{}, Value<L>{}},
+                                 tuple{Value<V>{}, Gradient<V>{}, Identity<S>{}},
+                                 *ir, all_domain_attr, derivatives);
+         >>>>>>> dfem-multiple-outputs
+
+         fes.GetRestrictionMatrix()->Mult(x, xtvec);
+         dop.Mult(X, Z);
+
+         std::cout << "dfem: ";
+         pretty_print(Z[0]);
+
+         Vector Y0(ytvecmfem);
+         Y0 -= Z[0];
+
+         real_t norm_l = Y0.Normlinf();
+         real_t norm_g = norm_l;
+         MPI_Allreduce(&norm_l, &norm_g, 1, MPI_DOUBLE, MPI_MAX, pmesh.GetComm());
+         REQUIRE(norm_g == MFEM_Approx(0.0));
+         MPI_Barrier(MPI_COMM_WORLD);
+
+         <<<<<<< HEAD
+         dbg("🔥🔥🔥");
+         return;
+
+         auto ddop = dop.GetDerivative(U, X);
+         =======
+            auto ddop = dop.GetDerivative(U, X);
+         >>>>>>> dfem-multiple-outputs
+
+         ddop->Mult(X[0], Z);
+         Y0 = ytvecmfem;
+         Y0 -= Z[0];
+
+         std::cout << "∂dfem: ";
+         pretty_print(Z[0]);
+
+         norm_l = Y0.Normlinf();
+         norm_g = norm_l;
+         MPI_Allreduce(&norm_l, &norm_g, 1, MPI_DOUBLE, MPI_MAX, pmesh.GetComm());
+         REQUIRE(norm_g == MFEM_Approx(0.0));
+         MPI_Barrier(MPI_COMM_WORLD);
+      }
+      {
+         DifferentiableOperator dop(in, out, pmesh);
+
+         dop.SetQLayouts({{Value<U>{}, {1, 0}}}, {});
+
+         auto mass_diffusion_qfunc = mass_diffusion_qdata_qf{};
+         dop.AddDomainIntegrator<LocalQFBackend>(
+            mass_diffusion_qfunc,
+            tuple{Value<U>{}, Gradient<U>{}, Gradient<COORDINATES>{}, Identity<S>{}, Weight{}, Value<L>{}},
+            tuple{Value<V>{}, Gradient<V>{}, Identity<S>{}},
+            *ir, all_domain_attr);
+
+         fes.GetRestrictionMatrix()->Mult(x, xtvec);
+         dop.Mult(X, Z);
+
+         std::cout << "dfem: ";
+         pretty_print(Z[0]);
+
+         Vector Y0(ytvecmfem);
+         Y0 -= Z[0];
+
+         real_t norm_l = Y0.Normlinf();
+         real_t norm_g = norm_l;
+         MPI_Allreduce(&norm_l, &norm_g, 1, MPI_DOUBLE, MPI_MAX, pmesh.GetComm());
+         REQUIRE(norm_g == MFEM_Approx(0.0));
+         MPI_Barrier(MPI_COMM_WORLD);
+      }
+   }
+}
+
+#endif // MFEM_USE_MPI